Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
138 changes: 127 additions & 11 deletions .github/workflows/sync-cloud-run-env.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,40 @@ name: Deploy Cloud Run

on:
workflow_dispatch:
inputs:
approved_ref:
description: Exact Git ref approved for this operation, for example refs/heads/main.
required: true
type: string
expected_sha:
description: Exact 40-character commit SHA approved for this operation.
required: true
type: string
approve_image_deploy:
description: Create a new Cloud Run revision without traffic.
default: false
required: true
type: boolean
approve_env_secret_sync:
description: Apply Cloud Run environment and Secret Manager reference changes.
default: false
required: true
type: boolean
approve_scheduler_iam_sync:
description: Apply Cloud Scheduler and Cloud Run IAM changes.
default: false
required: true
type: boolean
approve_traffic_shift:
description: Shift Cloud Run traffic to the latest revision.
default: false
required: true
type: boolean
approve_cleanup:
description: Delete old Cloud Run revisions, images, or legacy Scheduler jobs.
default: false
required: true
type: boolean

permissions:
contents: read
Expand Down Expand Up @@ -33,6 +67,13 @@ jobs:
ENABLE_GITHUB_CLOUD_RUN_DEPLOY: ${{ vars.ENABLE_GITHUB_CLOUD_RUN_DEPLOY }}
ENABLE_GITHUB_ENV_SYNC: ${{ vars.ENABLE_GITHUB_ENV_SYNC }}
CLOUD_RUN_CLEANUP_ENABLED: ${{ vars.CLOUD_RUN_CLEANUP_ENABLED }}
APPROVED_REF: ${{ inputs.approved_ref }}
EXPECTED_SHA: ${{ inputs.expected_sha }}
APPROVE_IMAGE_DEPLOY: ${{ inputs.approve_image_deploy }}
APPROVE_ENV_SECRET_SYNC: ${{ inputs.approve_env_secret_sync }}
APPROVE_SCHEDULER_IAM_SYNC: ${{ inputs.approve_scheduler_iam_sync }}
APPROVE_TRAFFIC_SHIFT: ${{ inputs.approve_traffic_shift }}
APPROVE_CLEANUP: ${{ inputs.approve_cleanup }}
ENABLE_MAIN_PUSH_CLOUD_RUN_AUTOMATION: ${{ vars.ENABLE_MAIN_PUSH_CLOUD_RUN_AUTOMATION }}
QSL_ENABLE_CLOUD_RUN_AUTOMATION: ${{ vars.QSL_ENABLE_CLOUD_RUN_AUTOMATION }}
GCP_ARTIFACT_REGISTRY_HOSTNAME: ${{ vars.GCP_ARTIFACT_REGISTRY_HOSTNAME }}
Expand Down Expand Up @@ -133,6 +174,9 @@ jobs:

deploy_enabled=false
env_sync_enabled=false
scheduler_iam_sync_enabled=false
traffic_shift_enabled=false
cleanup_enabled=false

# QSL_ENABLE_CLOUD_RUN_AUTOMATION overrides ENABLE_MAIN_PUSH_CLOUD_RUN_AUTOMATION
ENABLE_MAIN_PUSH_CLOUD_RUN_AUTOMATION="${QSL_ENABLE_CLOUD_RUN_AUTOMATION:-$ENABLE_MAIN_PUSH_CLOUD_RUN_AUTOMATION}"
Expand All @@ -145,21 +189,36 @@ jobs:
exit 0
fi

if [ "${ENABLE_GITHUB_CLOUD_RUN_DEPLOY:-}" = "true" ]; then
if [ "${ENABLE_GITHUB_CLOUD_RUN_DEPLOY:-}" = "true" ] && [ "${APPROVE_IMAGE_DEPLOY:-false}" = "true" ]; then
deploy_enabled=true
fi

if [ "${ENABLE_GITHUB_ENV_SYNC:-}" = "true" ]; then
if [ "${ENABLE_GITHUB_ENV_SYNC:-}" = "true" ] && [ "${APPROVE_ENV_SECRET_SYNC:-false}" = "true" ]; then
env_sync_enabled=true
fi

if [ "${ENABLE_GITHUB_ENV_SYNC:-}" = "true" ] && [ "${APPROVE_SCHEDULER_IAM_SYNC:-false}" = "true" ]; then
scheduler_iam_sync_enabled=true
fi

if [ "${ENABLE_GITHUB_ENV_SYNC:-}" = "true" ] && [ "${APPROVE_TRAFFIC_SHIFT:-false}" = "true" ]; then
traffic_shift_enabled=true
fi

if [ "${deploy_enabled}" = "true" ] && [ "${CLOUD_RUN_CLEANUP_ENABLED:-}" = "true" ] && [ "${APPROVE_CLEANUP:-false}" = "true" ]; then
cleanup_enabled=true
fi

write_github_output \
"deploy_enabled=${deploy_enabled}" \
"env_sync_enabled=${env_sync_enabled}"
"env_sync_enabled=${env_sync_enabled}" \
"scheduler_iam_sync_enabled=${scheduler_iam_sync_enabled}" \
"traffic_shift_enabled=${traffic_shift_enabled}" \
"cleanup_enabled=${cleanup_enabled}"

if [ "${deploy_enabled}" != "true" ] && [ "${env_sync_enabled}" != "true" ]; then
if [ "${deploy_enabled}" != "true" ] && [ "${env_sync_enabled}" != "true" ] && [ "${scheduler_iam_sync_enabled}" != "true" ] && [ "${traffic_shift_enabled}" != "true" ] && [ "${cleanup_enabled}" != "true" ]; then
write_github_output "enabled=false"
echo "Skipping Cloud Run automation because ENABLE_GITHUB_CLOUD_RUN_DEPLOY and ENABLE_GITHUB_ENV_SYNC are not true." >&2
echo "Skipping Cloud Run automation because no matching explicit approval is true." >&2
exit 0
fi

Expand All @@ -171,6 +230,26 @@ jobs:
with:
ref: ${{ github.sha }}

- name: Verify approved source
if: steps.config.outputs.enabled == 'true'
run: |
set -euo pipefail

if [[ ! "${EXPECTED_SHA}" =~ ^[0-9a-f]{40}$ ]]; then
echo "expected_sha must be a 40-character lowercase commit SHA." >&2
exit 1
fi
if [ "${APPROVED_REF}" != "${GITHUB_REF}" ]; then
echo "approved_ref does not match the dispatched Git ref." >&2
exit 1
fi

checked_out_sha="$(git rev-parse HEAD)"
if [ "${EXPECTED_SHA}" != "${GITHUB_SHA}" ] || [ "${EXPECTED_SHA}" != "${checked_out_sha}" ]; then
echo "expected_sha does not match the dispatched and checked-out commit." >&2
exit 1
fi

- name: Set up Python for strategy requirement resolution
if: steps.config.outputs.enabled == 'true'
uses: actions/setup-python@v6
Expand All @@ -186,7 +265,7 @@ jobs:

- name: Resolve Cloud Run sync targets
id: strategy_requirements
if: steps.config.outputs.env_sync_enabled == 'true'
if: steps.config.outputs.env_sync_enabled == 'true' || steps.config.outputs.scheduler_iam_sync_enabled == 'true' || steps.config.outputs.traffic_shift_enabled == 'true'
run: |
set -euo pipefail
sync_plan_json="$(uv run --no-sync python scripts/build_cloud_run_env_sync_plan.py --json)"
Expand Down Expand Up @@ -279,6 +358,19 @@ jobs:
project_id: ${{ env.GCP_PROJECT_ID }}
version: ">= 416.0.0"

- name: Capture no-traffic deployment baseline
if: steps.config.outputs.deploy_enabled == 'true'
env:
DEPLOY_READBACK_FILE: ${{ runner.temp }}/cloud-run-no-traffic-baseline.json
run: |
set -euo pipefail
python3 scripts/verify_cloud_run_no_traffic_deploy.py capture \
--project="${GCP_PROJECT_ID}" \
--region="${CLOUD_RUN_REGION}" \
--service="${CLOUD_RUN_SERVICE}" \
--scheduler-location="${CLOUD_SCHEDULER_LOCATION:-${CLOUD_RUN_REGION}}" \
--output="${DEPLOY_READBACK_FILE}"

- name: Verify deployed runtime target admission before traffic shift
if: steps.config.outputs.deploy_enabled == 'true'
run: |
Expand All @@ -290,6 +382,7 @@ jobs:


- name: Build, push, and deploy Cloud Run image
id: deploy
if: steps.config.outputs.deploy_enabled == 'true'
run: |
set -euo pipefail
Expand All @@ -301,6 +394,12 @@ jobs:
gcloud auth configure-docker "${artifact_registry_hostname}" --quiet
docker build --pull -t "${image}" .
docker push "${image}"
image_digest="$(gcloud artifacts docker images describe "${image}" --project="${GCP_PROJECT_ID}" --format='value(image_summary.digest)')"
if [[ ! "${image_digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; then
echo "Artifact Registry did not return a valid image digest." >&2
exit 1
fi
echo "image_digest=${image_digest}" >> "${GITHUB_OUTPUT}"

gcloud run deploy "${CLOUD_RUN_SERVICE}" \
--project="${GCP_PROJECT_ID}" \
Expand All @@ -316,8 +415,25 @@ jobs:
--cpu=1 \
--timeout=300s \
--labels="managed-by=github-actions,commit-sha=${GITHUB_SHA},github-run-id=${GITHUB_RUN_ID}" \
--no-traffic \
--quiet

- name: Verify no-traffic deployment readback
if: steps.config.outputs.deploy_enabled == 'true'
env:
DEPLOY_READBACK_FILE: ${{ runner.temp }}/cloud-run-no-traffic-baseline.json
EXPECTED_IMAGE_DIGEST: ${{ steps.deploy.outputs.image_digest }}
run: |
set -euo pipefail
python3 scripts/verify_cloud_run_no_traffic_deploy.py verify \
--project="${GCP_PROJECT_ID}" \
--region="${CLOUD_RUN_REGION}" \
--service="${CLOUD_RUN_SERVICE}" \
--scheduler-location="${CLOUD_SCHEDULER_LOCATION:-${CLOUD_RUN_REGION}}" \
--before="${DEPLOY_READBACK_FILE}" \
--expected-sha="${EXPECTED_SHA}" \
--expected-image-digest="${EXPECTED_IMAGE_DIGEST}"

- name: Sync Cloud Run environment
if: steps.config.outputs.env_sync_enabled == 'true'
env:
Expand Down Expand Up @@ -681,7 +797,7 @@ jobs:
gcloud "${gcloud_args[@]}"

- name: Reconcile Cloud Run traffic
if: steps.config.outputs.env_sync_enabled == 'true'
if: steps.config.outputs.traffic_shift_enabled == 'true'
env:
SYNC_PLAN_JSON: ${{ steps.strategy_requirements.outputs.sync_plan_json }}
run: |
Expand All @@ -698,7 +814,7 @@ jobs:

- name: Sync Cloud Scheduler schedule
id: scheduler_sync
if: steps.config.outputs.env_sync_enabled == 'true'
if: steps.config.outputs.scheduler_iam_sync_enabled == 'true'
env:
SYNC_PLAN_JSON: ${{ steps.strategy_requirements.outputs.sync_plan_json }}
DIRECT_MONITOR_MIGRATION_COMPLETE: ${{ vars.DIRECT_MONITOR_MIGRATION_COMPLETE }}
Expand Down Expand Up @@ -1024,7 +1140,7 @@ jobs:
fi

- name: Remove legacy Cloud Scheduler jobs
if: steps.config.outputs.env_sync_enabled == 'true'
if: steps.config.outputs.cleanup_enabled == 'true'
env:
SYNC_PLAN_JSON: ${{ steps.strategy_requirements.outputs.sync_plan_json }}
DIRECT_MONITOR_MIGRATION_COMPLETE: ${{ vars.DIRECT_MONITOR_MIGRATION_COMPLETE }}
Expand All @@ -1035,7 +1151,7 @@ jobs:
python3 scripts/reconcile_cloud_runtime.py cleanup-schedulers

- name: Prune old Cloud Run revisions
if: steps.config.outputs.deploy_enabled == 'true' && env.CLOUD_RUN_CLEANUP_ENABLED == 'true'
if: steps.config.outputs.cleanup_enabled == 'true'
run: |
set -euo pipefail

Expand Down Expand Up @@ -1084,7 +1200,7 @@ jobs:
done

- name: Clean up old Cloud Run images
if: steps.config.outputs.deploy_enabled == 'true' && env.CLOUD_RUN_CLEANUP_ENABLED == 'true'
if: steps.config.outputs.cleanup_enabled == 'true'
run: |
set -euo pipefail

Expand Down
143 changes: 143 additions & 0 deletions scripts/verify_cloud_run_no_traffic_deploy.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
#!/usr/bin/env python3
"""Read back one no-traffic Cloud Run deploy without exposing secret values."""

from __future__ import annotations

import argparse
import hashlib
import json
import subprocess
import sys
from pathlib import Path


SERVICE_FORMAT = (
"json(status.traffic,spec.template.spec.serviceAccountName,"
"spec.template.spec.containerConcurrency,spec.template.spec.timeoutSeconds,"
"spec.template.spec.containers.resources,spec.template.spec.containers.env.name,"
"spec.template.spec.containers.env.valueFrom.secretKeyRef)"
)
IAM_FORMAT = "json(bindings.role,bindings.members,bindings.condition)"
SCHEDULER_FORMAT = "json(name,state,schedule,timeZone,httpTarget.uri,httpTarget.oidcToken)"
REVISION_FORMAT = "json(metadata.name,metadata.labels,spec.containers.image)"


def _run_json(command: list[str]) -> object:
result = subprocess.run(command, text=True, capture_output=True, check=False)
if result.returncode:
raise RuntimeError("read-only gcloud readback command failed")
try:
return json.loads(result.stdout or "null")
except json.JSONDecodeError as exc:
raise RuntimeError("gcloud readback returned invalid JSON") from exc


def _canonical(value: object) -> str:
return json.dumps(value, sort_keys=True, separators=(",", ":"))


def _digest(value: object) -> str:
return hashlib.sha256(_canonical(value).encode("utf-8")).hexdigest()


def _snapshot(args: argparse.Namespace) -> dict[str, object]:
service = _run_json([
"gcloud", "run", "services", "describe", args.service,
f"--project={args.project}", f"--region={args.region}", f"--format={SERVICE_FORMAT}",
])
if not isinstance(service, dict):
raise RuntimeError("Cloud Run service readback returned a non-object payload")
iam = _run_json([
"gcloud", "run", "services", "get-iam-policy", args.service,
f"--project={args.project}", f"--region={args.region}", f"--format={IAM_FORMAT}",
])
scheduler = _run_json([
"gcloud", "scheduler", "jobs", "list", f"--project={args.project}",
f"--location={args.scheduler_location}", f"--format={SCHEDULER_FORMAT}",
])
status = service.get("status") if isinstance(service.get("status"), dict) else {}
# Keep only digests in the on-runner baseline. Service-account identities,
# endpoint URIs, and secret-reference names are needed for comparison but
# must not be persisted or printed by this verification helper.
return {
"traffic": _digest(status.get("traffic")),
"configuration": _digest(service.get("spec")),
"iam": _digest(iam),
"scheduler": _digest(scheduler),
}


def _created_revision(args: argparse.Namespace) -> dict[str, object]:
result = subprocess.run([
"gcloud", "run", "services", "describe", args.service,
f"--project={args.project}", f"--region={args.region}",
"--format=value(status.latestCreatedRevisionName)",
], text=True, capture_output=True, check=False)
revision_name = result.stdout.strip() if result.returncode == 0 else ""
if not revision_name:
raise RuntimeError("Cloud Run service did not report a created revision")
revision = _run_json([
"gcloud", "run", "revisions", "describe", revision_name,
f"--project={args.project}", f"--region={args.region}", f"--format={REVISION_FORMAT}",
])
if not isinstance(revision, dict):
raise RuntimeError("Cloud Run revision readback returned a non-object payload")
return revision


def _verify(args: argparse.Namespace) -> None:
try:
before = json.loads(args.before.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise RuntimeError("deployment baseline is unreadable") from exc
after = _snapshot(args)
if not isinstance(before, dict):
raise RuntimeError("deployment baseline is malformed")
for key in ("traffic", "scheduler", "iam", "configuration"):
if _canonical(before.get(key)) != _canonical(after.get(key)):
raise RuntimeError(f"{key} changed during no-traffic deployment")

revision = _created_revision(args)
metadata = revision.get("metadata") if isinstance(revision.get("metadata"), dict) else {}
labels = metadata.get("labels") if isinstance(metadata.get("labels"), dict) else {}
containers = revision.get("spec", {}).get("containers", []) if isinstance(revision.get("spec"), dict) else []
image = containers[0].get("image", "") if containers and isinstance(containers[0], dict) else ""
if labels.get("commit-sha") != args.expected_sha:
raise RuntimeError("created revision commit SHA does not match expected SHA")
if f"@{args.expected_image_digest}" not in str(image):
raise RuntimeError("created revision image digest does not match the pushed image")
print(
"Verified no-traffic deployment: commit SHA and image digest match; "
"traffic, scheduler, IAM, and configuration digests are unchanged."
)


def main() -> int:
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(dest="command", required=True)
for name in ("capture", "verify"):
command = subparsers.add_parser(name)
command.add_argument("--project", required=True)
command.add_argument("--region", required=True)
command.add_argument("--service", required=True)
command.add_argument("--scheduler-location", required=True)
subparsers.choices["capture"].add_argument("--output", required=True, type=Path)
verify = subparsers.choices["verify"]
verify.add_argument("--before", required=True, type=Path)
verify.add_argument("--expected-sha", required=True)
verify.add_argument("--expected-image-digest", required=True)
args = parser.parse_args()
try:
if args.command == "capture":
args.output.write_text(_canonical(_snapshot(args)), encoding="utf-8")
print("Captured non-secret Cloud Run deployment baseline.")
else:
_verify(args)
except RuntimeError as exc:
print(f"No-traffic deployment readback failed: {exc}", file=sys.stderr)
return 1
return 0


if __name__ == "__main__":
raise SystemExit(main())
Loading