From 133717723547381eaa3b24dd99f5e458cf664725 Mon Sep 17 00:00:00 2001 From: Shaurya K Sharma Date: Wed, 8 Jul 2026 09:28:39 +0100 Subject: [PATCH 1/2] infra: add deterministic Render deploy pipeline --- .github/workflows/deploy.yml | 124 +++++++++++++----- docs/deployment/render.md | 239 +++++++++++++++++++++++++++++++++++ render.yaml | 152 ++++++++++++++++++++++ scripts/render_deploy.py | 184 +++++++++++++++++++++++++++ startup.sh | 16 +-- 5 files changed, 673 insertions(+), 42 deletions(-) create mode 100644 docs/deployment/render.md create mode 100644 render.yaml create mode 100644 scripts/render_deploy.py diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5d63a23..e2e9d1d 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,8 +1,20 @@ name: Deploy API to Render on: + push: + branches: + - dev + - main workflow_dispatch: # allows manual trigger from GitHub UI +# Never let two deploys of the same branch race each other. +concurrency: + group: deploy-${{ github.ref }} + cancel-in-progress: false + +permissions: + contents: read + jobs: deploy: name: Deploy to Render @@ -31,31 +43,70 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt - # ── Secret check (Determines if smoke tests should run) ─────────── - - name: Check for JWT_SECRET - id: check_config + # ── Branch → environment routing ─────────────────────────────────── + # dev -> staging, main -> production. Any other ref (e.g. a manual + # dispatch from a feature branch) is treated as non-protected: we do not + # have a Render environment for it, so deploy/health/smoke are skipped. + - name: Select target environment + id: target run: | - if [ -n "${{ secrets.JWT_SECRET }}" ]; then - echo "is_configured=true" >> $GITHUB_OUTPUT - else - echo "is_configured=false" >> $GITHUB_OUTPUT - fi + case "${{ github.ref_name }}" in + dev) + echo "env_name=staging" >> "$GITHUB_OUTPUT" + echo "is_protected=true" >> "$GITHUB_OUTPUT" + ;; + main) + echo "env_name=production" >> "$GITHUB_OUTPUT" + echo "is_protected=true" >> "$GITHUB_OUTPUT" + ;; + *) + echo "env_name=none" >> "$GITHUB_OUTPUT" + echo "is_protected=false" >> "$GITHUB_OUTPUT" + ;; + esac + echo "Target environment: $(grep env_name "$GITHUB_OUTPUT" | cut -d= -f2)" - # ── Wait for Render auto-deployment ──────────────────────────────── - # Render handles the actual physical deployment when you push. - # We just pause the Action to let Render's servers finish building. - - name: Wait for app to initialise + # ── Deterministic Render deploy ──────────────────────────────────── + # Triggers a deploy of the correct service pinned to this commit SHA and + # blocks until Render reports that specific deploy live (or fails). No + # fixed sleeps: the step's exit code is Render's real deploy status. + - name: Deploy to Render and wait for it to go live + if: steps.target.outputs.is_protected == 'true' + env: + RENDER_API_KEY: ${{ secrets.RENDER_API_KEY }} + # Both ids are exposed; the correct one is chosen strictly by branch + # below (avoids empty-string fallback selecting the wrong environment). + RENDER_STAGING_SERVICE_ID: ${{ secrets.RENDER_STAGING_SERVICE_ID }} + RENDER_PRODUCTION_SERVICE_ID: ${{ secrets.RENDER_PRODUCTION_SERVICE_ID }} + GITHUB_SHA: ${{ github.sha }} run: | - echo "Waiting 120 seconds for Render to build and start the app..." - sleep 120 + if [ "${{ github.ref_name }}" = "main" ]; then + export RENDER_SERVICE_ID="$RENDER_PRODUCTION_SERVICE_ID" + else + export RENDER_SERVICE_ID="$RENDER_STAGING_SERVICE_ID" + fi + if [ -z "$RENDER_API_KEY" ] || [ -z "$RENDER_SERVICE_ID" ]; then + echo "ERROR: RENDER_API_KEY and the ${{ steps.target.outputs.env_name }} service id must be configured as GitHub secrets." + exit 1 + fi + python scripts/render_deploy.py - # ── Health gate ──────────────────────────────────────────────────── + # ── Health gate (against the branch-specific URL) ────────────────── - name: Health gate check - id: health_gate + if: steps.target.outputs.is_protected == 'true' env: - # Use secret URL if provided, otherwise fallback to default - API_URL: ${{ secrets.API_URL || 'https://openshield-api.onrender.com' }} + STAGING_API_URL: ${{ secrets.STAGING_API_URL }} + PRODUCTION_API_URL: ${{ secrets.PRODUCTION_API_URL }} run: | + if [ "${{ github.ref_name }}" = "main" ]; then + API_URL="$PRODUCTION_API_URL" + else + API_URL="$STAGING_API_URL" + fi + if [ -z "$API_URL" ]; then + echo "ERROR: the ${{ steps.target.outputs.env_name }} API URL secret is not configured." + exit 1 + fi MAX_RETRIES=5 RETRY_DELAY=15 URL="${API_URL}/health" @@ -74,22 +125,15 @@ jobs: sleep $RETRY_DELAY done - echo "HEALTH GATE FAILED after $MAX_RETRIES attempts" - echo "Note: If you haven't set up Render for this fork, this is expected." - # Only allow failure on feature branches; fail on main/dev - if [[ "${{ github.ref }}" == "refs/heads/main" || "${{ github.ref }}" == "refs/heads/dev" ]]; then - echo "ERROR: Health check failed on protected branch. Deployment verification required." - exit 1 - else - echo "Allowing health check failure on feature branch (infra may not be set up)" - exit 0 - fi + echo "ERROR: Health gate failed after $MAX_RETRIES attempts on the ${{ steps.target.outputs.env_name }} environment." + exit 1 - # ── Smoke tests ──────────────────────────────────────────────────── + # ── Smoke tests (against the same branch-specific URL) ───────────── - name: Run smoke tests against live deployment - if: steps.check_config.outputs.is_configured == 'true' || github.event_name == 'workflow_dispatch' + if: steps.target.outputs.is_protected == 'true' env: - API_URL: ${{ secrets.API_URL || 'https://openshield-api.onrender.com' }} + STAGING_API_URL: ${{ secrets.STAGING_API_URL }} + PRODUCTION_API_URL: ${{ secrets.PRODUCTION_API_URL }} JWT_SECRET: ${{ secrets.JWT_SECRET }} AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} @@ -97,9 +141,21 @@ jobs: AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} RUN_REAL_SCAN: "true" run: | - if [[ "${{ github.ref }}" == "refs/heads/main" && -z "${{ secrets.JWT_SECRET }}" ]]; then - echo "ERROR: Cannot run smoke tests on main branch without JWT_SECRET configured" + if [ "${{ github.ref_name }}" = "main" ]; then + export API_URL="$PRODUCTION_API_URL" + else + export API_URL="$STAGING_API_URL" + fi + if [ -z "$JWT_SECRET" ]; then + echo "ERROR: JWT_SECRET is required to run smoke tests against a protected environment." exit 1 fi echo "Running smoke tests against: $API_URL" - python tests/smoke_test.py \ No newline at end of file + python tests/smoke_test.py + + # ── Non-protected refs ───────────────────────────────────────────── + - name: Skip deploy for non-protected ref + if: steps.target.outputs.is_protected != 'true' + run: | + echo "Ref '${{ github.ref_name }}' has no Render environment (only dev->staging and main->production deploy)." + echo "Skipping deploy, health gate, and smoke tests." diff --git a/docs/deployment/render.md b/docs/deployment/render.md new file mode 100644 index 0000000..719b5dc --- /dev/null +++ b/docs/deployment/render.md @@ -0,0 +1,239 @@ +# OpenShield — Render Deployment + +This document describes how OpenShield is deployed to [Render](https://render.com) +using a **deterministic, GitHub Actions-controlled** pipeline. Deploys succeed or +fail based on Render's *actual* deploy status, never on a fixed timer. + +- Blueprint: [`render.yaml`](../../render.yaml) +- Deploy script: [`scripts/render_deploy.py`](../../scripts/render_deploy.py) +- Workflow: [`.github/workflows/deploy.yml`](../../.github/workflows/deploy.yml) + +--- + +## 1. Service layout + +The blueprint declares four services across two environments: + +| Service | Type | Branch | Start command | Purpose | +|---|---|---|---|---| +| `openshield-api-staging` | web | `dev` | `./startup.sh` | Staging API (Gunicorn) | +| `openshield-worker-staging` | worker | `dev` | `python -m scanner.worker` | Staging scan worker | +| `openshield-api` | web | `main` | `./startup.sh` | Production API (Gunicorn) | +| `openshield-worker` | worker | `main` | `python -m scanner.worker` | Production scan worker | + +### Staging / production branch mapping + +- `dev` → **staging** (`openshield-api-staging`, `openshield-worker-staging`) +- `main` → **production** (`openshield-api`, `openshield-worker`) + +The staging and production APIs are served from **separate URLs** and CI health-checks +and smoke-tests each branch against its own URL — they no longer share a fallback +production URL. + +### The worker is a separate service + +The scan worker runs as its own Render **background worker** service, not as a +subshell inside the web container (`startup.sh` no longer launches it). Consequences: + +- Restarting, redeploying, or scaling the **web** service does **not** kill the + worker process. +- Interrupted scans are recovered by the worker itself — `scanner.worker` calls + `db.recover_stale_scans(timeout_minutes=60)` on every poll loop. +- Web and worker in the same environment **must share the same `DATABASE_URL` and + `JWT_SECRET`**. These are declared per service with `sync: false`, so set them to + identical values in the dashboard for the web and worker of the same environment. + (Render does not allow `sync: false` inside env var groups, so the values cannot + be centralised in the blueprint — this pairing is enforced by convention.) + +--- + +## 2. Deploy runtime (native Python) + +Each service uses Render's native Python runtime: + +- **Build command:** `pip install -r requirements.txt` +- **Web start command:** `./startup.sh` (DB init, then Gunicorn) +- **Worker start command:** `python -m scanner.worker` + +> The repository also ships a `Dockerfile` for local `docker-compose`. If you +> prefer to deploy on Render with Docker instead, set each service's `runtime` to +> `docker` and provide a `dockerCommand` (`./startup.sh` for web, +> `python -m scanner.worker` for the worker). The native-Python setup above matches +> the currently documented Render configuration. + +### Validating the blueprint + +`render.yaml` is checked in CI only for YAML validity — there is no offline Render +schema validator (the Render CLI does not provide a `blueprint validate` command). +**Maintainers must validate the blueprint against Render's live schema before +relying on it**, by either: + +1. Opening the repo as a **Blueprint** in the Render dashboard + (*New → Blueprint*), which parses `render.yaml` and previews the four services + before creating anything; or +2. Running the blueprint sync in a throwaway Render workspace first. + +Render treats `autoDeployTrigger` as the current field and ignores the deprecated +`autoDeploy`; the values used here are `"off"` (web) and `commit` (worker). + +--- + +## 3. GitHub Actions secrets + +Configure these under **Settings → Secrets and variables → Actions**. No values +are stored in the repo; `render.yaml` declares secrets with `sync: false` so +Render never overwrites the values you set in its dashboard. + +| Secret | Used for | +|---|---| +| `RENDER_API_KEY` | Authenticating Render API deploy calls | +| `RENDER_STAGING_SERVICE_ID` | Service id of `openshield-api-staging` | +| `RENDER_PRODUCTION_SERVICE_ID` | Service id of `openshield-api` | +| `STAGING_API_URL` | Staging base URL (health gate + smoke tests) | +| `PRODUCTION_API_URL` | Production base URL (health gate + smoke tests) | +| `JWT_SECRET` | Signing smoke-test JWTs (must match the value set in Render) | +| `AZURE_SUBSCRIPTION_ID` | Real-scan smoke tests | +| `AZURE_CLIENT_ID` | Real-scan smoke tests | +| `AZURE_CLIENT_SECRET` | Real-scan smoke tests | +| `AZURE_TENANT_ID` | Real-scan smoke tests | + +The workflow selects the service id and API URL automatically from the branch: +`dev` uses the staging pair, `main` uses the production pair. + +--- + +## 4. How deterministic deploy polling works + +On push to `dev`/`main` (or manual `workflow_dispatch`), the workflow runs +[`scripts/render_deploy.py`](../../scripts/render_deploy.py) with `RENDER_API_KEY`, +the branch's `RENDER_SERVICE_ID`, and `GITHUB_SHA`. The script: + +1. **Triggers a deploy** pinned to the commit CI built: + `POST https://api.render.com/v1/services/{serviceId}/deploys` with + `{"commitId": ""}`. +2. **Captures the returned deploy id.** +3. **Polls that specific deploy:** + `GET https://api.render.com/v1/services/{serviceId}/deploys/{deployId}`. +4. Continues while the status is in progress (`created`, `queued`, + `build_in_progress`, `update_in_progress`, `pre_deploy_in_progress`). +5. **Exits 0 only when the deploy becomes `live`.** +6. **Exits non-zero** if the deploy is `build_failed`, `update_failed`, + `pre_deploy_failed`, `canceled`, or `deactivated`, or if the overall timeout + (`RENDER_DEPLOY_TIMEOUT_SECONDS`, default 1800s) is exceeded. + +Poll interval and timeout are tunable via `RENDER_DEPLOY_POLL_SECONDS` +(default 15) and `RENDER_DEPLOY_TIMEOUT_SECONDS` (default 1800). The API key is +never printed. + +Only after the deploy is live does the workflow run the **health gate** +(`GET {API_URL}/health` must return 200) and the **smoke tests** +(`tests/smoke_test.py`) against the branch-specific URL. On `dev`/`main`, any of +these failing fails the workflow. + +### Deploy control (`autoDeployTrigger`) and avoiding duplicate deploys + +`render.yaml` uses `autoDeployTrigger` (the current field; it replaces the +deprecated `autoDeploy`): + +- **Web services → `autoDeployTrigger: "off"`.** GitHub Actions is the single + source of truth for web deploys, so Render's own auto-deploy is turned off. If it + were left on, every push would trigger *both* a Render auto-deploy and the + Actions-driven deploy of the same service — duplicate, racing deploys. `"off"` is + quoted because unquoted `off` is parsed as the boolean `false` by YAML. +- **Worker services → `autoDeployTrigger: commit`.** The workers **auto-deploy on + every push** to their branch. Nothing else deploys them, so there is no + duplication, and they stay in sync with the web service. The health gate and + smoke tests do **not** cover the worker; if you want a worker deployed + deterministically (e.g. to gate or roll it back precisely), run + `scripts/render_deploy.py` with `RENDER_SERVICE_ID` set to the worker's service id. + +In short: **web = deployed by the workflow; worker = auto-deployed by Render on +push** (and optionally by the same script). + +--- + +## 5. Manual verification + +### Staging + +```bash +# Health +curl -fsS "$STAGING_API_URL/health" # expect {"status":"ok"} + +# Smoke suite +API_URL="$STAGING_API_URL" JWT_SECRET="" \ + python tests/smoke_test.py +``` + +Confirm `dev` is serving staging: push a change to `dev`, watch the +"Deploy API to Render" workflow reach *live*, then re-run the health check. + +### Production + +```bash +# Health +curl -fsS "$PRODUCTION_API_URL/health" # expect {"status":"ok"} + +# Smoke suite +API_URL="$PRODUCTION_API_URL" JWT_SECRET="" \ + python tests/smoke_test.py +``` + +Confirm the worker is separate: restart the web service in the Render dashboard — +the worker service stays running and continues processing the scan queue. + +--- + +## 6. Rollback — redeploy a previous commit via the Render API + +Because deploys are pinned to a `commitId`, rolling back is just deploying an +older commit. Use the same mechanism CI uses. + +**Option A — the deploy script (recommended):** + +```bash +export RENDER_API_KEY="" +export RENDER_SERVICE_ID="" +export GITHUB_SHA="" +python scripts/render_deploy.py +``` + +The script triggers the deploy, waits for it to go live, and exits non-zero if the +rollback deploy fails. + +**Option B — raw Render API:** + +```bash +# Trigger a deploy of a known-good commit +curl -X POST "https://api.render.com/v1/services/$RENDER_SERVICE_ID/deploys" \ + -H "Authorization: Bearer $RENDER_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"commitId": ""}' +# -> note the returned "id" (dep-...), then poll it: + +curl "https://api.render.com/v1/services/$RENDER_SERVICE_ID/deploys/" \ + -H "Authorization: Bearer $RENDER_API_KEY" +# repeat until "status": "live" +``` + +You can also **Rollback** to a prior successful deploy from the Render dashboard +(service → *Deploys* → previous deploy → *Rollback*). + +Roll back the matching **worker** service the same way (with its own service id) if +a bad commit changed worker behaviour. + +--- + +## 7. What NOT to do + +- ❌ **Do not reintroduce a fixed `sleep` to "wait for the deploy"** in the + workflow. Gate on Render's real deploy status via `scripts/render_deploy.py`. +- ❌ **Do not run the scan worker inside the web container** (no background + subshell in `startup.sh`). It must stay a separate Render worker service. +- ❌ **Do not set the web services to `autoDeployTrigger: commit`** (or re-add the + deprecated `autoDeploy: true`) while GitHub Actions also deploys them — that + causes duplicate/racing deploys. Keep web services on `autoDeployTrigger: "off"`. +- ❌ **Do not put secret values or real service ids** in `render.yaml` or the + workflow. Use GitHub secrets and `sync: false` env var declarations. +- ❌ **Do not point `dev` and `main` at the same URL.** Staging and production are + distinct services with distinct URLs. diff --git a/render.yaml b/render.yaml new file mode 100644 index 0000000..611bc1d --- /dev/null +++ b/render.yaml @@ -0,0 +1,152 @@ +# Render blueprint for OpenShield. +# +# Declares the staging and production environments as four services: +# openshield-api-staging web (branch: dev) -> serves the staging API +# openshield-worker-staging worker (branch: dev) -> runs scanner.worker +# openshield-api web (branch: main) -> serves the production API +# openshield-worker worker (branch: main) -> runs scanner.worker +# +# The scan worker runs as its own background worker service, NOT inside the web +# container. Restarting/scaling a web service therefore does not kill the worker. +# +# Deploy control (autoDeployTrigger — replaces the deprecated `autoDeploy`): +# * Web services use `autoDeployTrigger: "off"` (quoted — unquoted `off` is +# parsed as boolean false by YAML). Their deploys are driven +# deterministically by GitHub Actions +# (.github/workflows/deploy.yml -> scripts/render_deploy.py), so Render's own +# auto-deploy is disabled to avoid duplicate/racing deploys of the same service. +# * Worker services use `autoDeployTrigger: commit` (auto-deploy on every push to +# their branch). Nothing else deploys them, so there is no duplication. They can +# also be deployed deterministically with scripts/render_deploy.py by passing the +# worker's service id as RENDER_SERVICE_ID. +# +# No secret values, real service ids, or credentials are stored here. Secrets are +# declared per service with `sync: false`, meaning Render will NOT overwrite the +# values you set in the dashboard on each blueprint sync — set them once per service. +# (`sync: false` is intentionally declared on the services, not in an env var group: +# Render does not allow `sync: false` inside envVarGroups.) +# +# IMPORTANT: within one environment the web and worker services MUST be given the +# SAME DATABASE_URL and JWT_SECRET values (they share one database). The blueprint +# cannot enforce this for sync:false vars, so set them identically in the dashboard. + +services: + # ── Staging (dev branch) ───────────────────────────────────────────── + - type: web + name: openshield-api-staging + runtime: python + branch: dev + autoDeployTrigger: "off" + buildCommand: pip install -r requirements.txt + startCommand: ./startup.sh + healthCheckPath: /health + envVars: + - key: OPENSHIELD_ENV + value: production + - key: DATABASE_URL + sync: false + - key: JWT_SECRET + sync: false + - key: AZURE_SUBSCRIPTION_ID + sync: false + - key: AZURE_CLIENT_ID + sync: false + - key: AZURE_CLIENT_SECRET + sync: false + - key: AZURE_TENANT_ID + sync: false + - key: NVD_API_KEY + sync: false + - key: ANTHROPIC_API_KEY + sync: false + - key: SENTRY_DSN + sync: false + + - type: worker + name: openshield-worker-staging + runtime: python + branch: dev + autoDeployTrigger: commit + buildCommand: pip install -r requirements.txt + startCommand: python -m scanner.worker + envVars: + - key: OPENSHIELD_ENV + value: production + - key: DATABASE_URL + sync: false + - key: JWT_SECRET + sync: false + - key: AZURE_SUBSCRIPTION_ID + sync: false + - key: AZURE_CLIENT_ID + sync: false + - key: AZURE_CLIENT_SECRET + sync: false + - key: AZURE_TENANT_ID + sync: false + - key: NVD_API_KEY + sync: false + - key: ANTHROPIC_API_KEY + sync: false + - key: SENTRY_DSN + sync: false + + # ── Production (main branch) ───────────────────────────────────────── + - type: web + name: openshield-api + runtime: python + branch: main + autoDeployTrigger: "off" + buildCommand: pip install -r requirements.txt + startCommand: ./startup.sh + healthCheckPath: /health + envVars: + - key: OPENSHIELD_ENV + value: production + - key: DATABASE_URL + sync: false + - key: JWT_SECRET + sync: false + - key: AZURE_SUBSCRIPTION_ID + sync: false + - key: AZURE_CLIENT_ID + sync: false + - key: AZURE_CLIENT_SECRET + sync: false + - key: AZURE_TENANT_ID + sync: false + - key: NVD_API_KEY + sync: false + - key: ANTHROPIC_API_KEY + sync: false + - key: SENTRY_DSN + sync: false + + - type: worker + name: openshield-worker + runtime: python + branch: main + autoDeployTrigger: commit + buildCommand: pip install -r requirements.txt + startCommand: python -m scanner.worker + envVars: + - key: OPENSHIELD_ENV + value: production + - key: DATABASE_URL + sync: false + - key: JWT_SECRET + sync: false + - key: AZURE_SUBSCRIPTION_ID + sync: false + - key: AZURE_CLIENT_ID + sync: false + - key: AZURE_CLIENT_SECRET + sync: false + - key: AZURE_TENANT_ID + sync: false + - key: NVD_API_KEY + sync: false + - key: ANTHROPIC_API_KEY + sync: false + - key: SENTRY_DSN + sync: false diff --git a/scripts/render_deploy.py b/scripts/render_deploy.py new file mode 100644 index 0000000..bbadfe7 --- /dev/null +++ b/scripts/render_deploy.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +""" +scripts/render_deploy.py + +Trigger a Render deploy for a single service and block until that *specific* +deploy reaches a terminal state. This replaces the previous "sleep then hope" +approach in the deploy workflow: the exit code reflects Render's real deploy +status, never a timer. + +Behaviour +--------- +1. POST https://api.render.com/v1/services/{serviceId}/deploys with the GitHub + commit SHA as ``commitId`` (so Render deploys exactly the commit CI built). +2. Capture the returned deploy id. +3. Poll GET https://api.render.com/v1/services/{serviceId}/deploys/{deployId} + until the deploy becomes live (exit 0) or fails/cancels/times out (exit != 0). + +Environment +----------- +Required: + RENDER_API_KEY Render API key (Bearer token). Never printed. + RENDER_SERVICE_ID Target service id, e.g. "srv-xxxxxxxx". + GITHUB_SHA Commit SHA to deploy. Provided automatically by GitHub Actions. + +Optional: + RENDER_DEPLOY_TIMEOUT_SECONDS Overall poll budget (default 1800). + RENDER_DEPLOY_POLL_SECONDS Seconds between polls (default 15). + +Only the Python standard library is used so the script runs with no extra +dependencies installed. +""" + +import json +import os +import sys +import time +import urllib.error +import urllib.request + +API_ROOT = "https://api.render.com/v1" + +# Render deploy statuses. See https://api-docs.render.com/reference/get-deploy +LIVE_STATUS = "live" +IN_PROGRESS_STATUSES = { + "created", + "queued", + "build_in_progress", + "update_in_progress", + "pre_deploy_in_progress", +} +FAILED_STATUSES = { + "build_failed", + "update_failed", + "pre_deploy_failed", + "canceled", + "deactivated", +} + + +def _fail(message: str) -> "None": + """Print an error to stderr and exit non-zero.""" + print(f"ERROR: {message}", file=sys.stderr) + sys.exit(1) + + +def _env(name: str, required: bool = True, default: str = "") -> str: + value = os.environ.get(name, default) + if required and not value: + _fail(f"{name} environment variable is not set") + return value + + +def _request(method: str, url: str, api_key: str, payload: "dict | None" = None) -> dict: + """Make a Render API call and return the parsed JSON object. + + Raises SystemExit (via _fail) on transport errors or malformed responses. + """ + data = json.dumps(payload).encode("utf-8") if payload is not None else None + req = urllib.request.Request(url, data=data, method=method) + req.add_header("Authorization", f"Bearer {api_key}") + req.add_header("Accept", "application/json") + if data is not None: + req.add_header("Content-Type", "application/json") + + try: + with urllib.request.urlopen(req, timeout=30) as resp: + body = resp.read().decode("utf-8") + except urllib.error.HTTPError as exc: + # Read the error body for context but never echo the API key. + detail = "" + try: + detail = exc.read().decode("utf-8")[:500] + except Exception: + pass + _fail(f"{method} {url} returned HTTP {exc.code}. {detail}".strip()) + except urllib.error.URLError as exc: + _fail(f"{method} {url} failed: {exc.reason}") + + try: + parsed = json.loads(body) + except json.JSONDecodeError: + _fail(f"{method} {url} returned a non-JSON response: {body[:200]!r}") + + return parsed + + +def trigger_deploy(service_id: str, api_key: str, commit_sha: str) -> str: + """Create a deploy for ``commit_sha`` and return the new deploy id.""" + url = f"{API_ROOT}/services/{service_id}/deploys" + payload = {"commitId": commit_sha} if commit_sha else {} + print(f"Triggering Render deploy for service {service_id} at commit {commit_sha or '(latest)'}...") + + result = _request("POST", url, api_key, payload=payload) + if not isinstance(result, dict): + _fail(f"Unexpected deploy-create response (not an object): {result!r}") + + deploy_id = result.get("id") + if not deploy_id: + _fail(f"Deploy-create response did not contain a deploy id: {result!r}") + + print(f"Created deploy {deploy_id} (initial status: {result.get('status', 'unknown')})") + return deploy_id + + +def poll_deploy(service_id: str, api_key: str, deploy_id: str, timeout_s: int, poll_s: int) -> None: + """Poll a specific deploy until it is live, fails, or times out.""" + url = f"{API_ROOT}/services/{service_id}/deploys/{deploy_id}" + deadline = time.monotonic() + timeout_s + last_status = None + + while True: + result = _request("GET", url, api_key) + # The get-deploy endpoint may return the deploy object directly or + # wrapped as {"deploy": {...}}; handle both shapes. + if isinstance(result, dict) and "deploy" in result and isinstance(result["deploy"], dict): + result = result["deploy"] + if not isinstance(result, dict): + _fail(f"Unexpected deploy-status response (not an object): {result!r}") + + status = result.get("status") + if not status: + _fail(f"Deploy-status response did not contain a status: {result!r}") + + if status != last_status: + print(f"Deploy {deploy_id} status: {status}") + last_status = status + + if status == LIVE_STATUS: + print(f"Deploy {deploy_id} is live. Deployment succeeded.") + return + + if status in FAILED_STATUSES: + _fail(f"Deploy {deploy_id} ended in non-live status '{status}'.") + + if status not in IN_PROGRESS_STATUSES: + # Unknown/new status: keep polling but make it visible. + print(f"Deploy {deploy_id} reported unrecognised status '{status}'; continuing to poll.") + + if time.monotonic() >= deadline: + _fail(f"Timed out after {timeout_s}s waiting for deploy {deploy_id} (last status: {status}).") + + time.sleep(poll_s) + + +def main() -> None: + api_key = _env("RENDER_API_KEY") + service_id = _env("RENDER_SERVICE_ID") + commit_sha = _env("GITHUB_SHA", required=False) + + try: + timeout_s = int(_env("RENDER_DEPLOY_TIMEOUT_SECONDS", required=False, default="1800")) + poll_s = int(_env("RENDER_DEPLOY_POLL_SECONDS", required=False, default="15")) + except ValueError: + _fail("RENDER_DEPLOY_TIMEOUT_SECONDS and RENDER_DEPLOY_POLL_SECONDS must be integers") + + if poll_s <= 0: + _fail("RENDER_DEPLOY_POLL_SECONDS must be a positive integer") + + deploy_id = trigger_deploy(service_id, api_key, commit_sha) + poll_deploy(service_id, api_key, deploy_id, timeout_s, poll_s) + + +if __name__ == "__main__": + main() diff --git a/startup.sh b/startup.sh index 6b3e0df..dd03262 100755 --- a/startup.sh +++ b/startup.sh @@ -24,13 +24,13 @@ except Exception as e: sys.exit(1) " -echo "Startup complete. Starting background worker and Gunicorn..." -# Start the background worker process with a simple restart loop -( - until python3 -m scanner.worker; do - echo "Worker process crashed with exit code $?. Respawning in 5 seconds..." >&2 - sleep 5 - done -) & +echo "Startup complete. Starting Gunicorn web server..." +# NOTE: The scan worker is intentionally NOT started here. It runs as a +# separate Render background worker service (see render.yaml: +# openshield-worker / openshield-worker-staging, start command +# `python -m scanner.worker`). Keeping the worker out of the web container +# means restarting or scaling the web service does not kill the worker +# process; the standalone worker recovers stale scans on its own. Do not +# reintroduce a background worker subshell here. exec gunicorn --bind=0.0.0.0:$PORT --timeout 120 --workers 2 api.app:application \ No newline at end of file From ee40ed63c65e01a746839cb768c1b0f59830231c Mon Sep 17 00:00:00 2001 From: Shaurya K Sharma Date: Wed, 8 Jul 2026 16:48:13 +0100 Subject: [PATCH 2/2] infra: make Render deploy workflow manual --- .github/workflows/deploy.yml | 97 +++++++++++++++--------------------- docs/deployment/render.md | 73 +++++++++++++++++---------- 2 files changed, 88 insertions(+), 82 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e2e9d1d..39083c6 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,15 +1,31 @@ name: Deploy API to Render +# Manual, deterministic deploys only. +# +# Automatic push-triggered deploys are intentionally NOT enabled: the staging/ +# production Render services, database, and secrets must exist first. Deploying on +# every push/merge before that infrastructure is provisioned would fail. Once +# maintainers confirm the Render staging/prod setup, a `push:` trigger can be +# re-added if desired. on: - push: - branches: - - dev - - main - workflow_dispatch: # allows manual trigger from GitHub UI - -# Never let two deploys of the same branch race each other. + workflow_dispatch: + inputs: + environment: + description: Target Render environment + required: true + type: choice + options: + - staging + - production + run_smoke_tests: + description: Run smoke tests after deploy + required: true + default: true + type: boolean + +# Never let two deploys of the same environment race each other. concurrency: - group: deploy-${{ github.ref }} + group: deploy-${{ inputs.environment }} cancel-in-progress: false permissions: @@ -17,7 +33,7 @@ permissions: jobs: deploy: - name: Deploy to Render + name: Deploy to Render (${{ inputs.environment }}) runs-on: ubuntu-latest steps: @@ -43,68 +59,44 @@ jobs: python -m pip install --upgrade pip pip install -r requirements.txt - # ── Branch → environment routing ─────────────────────────────────── - # dev -> staging, main -> production. Any other ref (e.g. a manual - # dispatch from a feature branch) is treated as non-protected: we do not - # have a Render environment for it, so deploy/health/smoke are skipped. - - name: Select target environment - id: target - run: | - case "${{ github.ref_name }}" in - dev) - echo "env_name=staging" >> "$GITHUB_OUTPUT" - echo "is_protected=true" >> "$GITHUB_OUTPUT" - ;; - main) - echo "env_name=production" >> "$GITHUB_OUTPUT" - echo "is_protected=true" >> "$GITHUB_OUTPUT" - ;; - *) - echo "env_name=none" >> "$GITHUB_OUTPUT" - echo "is_protected=false" >> "$GITHUB_OUTPUT" - ;; - esac - echo "Target environment: $(grep env_name "$GITHUB_OUTPUT" | cut -d= -f2)" - # ── Deterministic Render deploy ──────────────────────────────────── - # Triggers a deploy of the correct service pinned to this commit SHA and - # blocks until Render reports that specific deploy live (or fails). No - # fixed sleeps: the step's exit code is Render's real deploy status. + # Triggers a deploy of the chosen environment's service pinned to the + # dispatched commit SHA and blocks until Render reports that specific + # deploy live (or fails). No fixed sleeps: the step's exit code is Render's + # real deploy status. - name: Deploy to Render and wait for it to go live - if: steps.target.outputs.is_protected == 'true' env: RENDER_API_KEY: ${{ secrets.RENDER_API_KEY }} - # Both ids are exposed; the correct one is chosen strictly by branch - # below (avoids empty-string fallback selecting the wrong environment). + # Both ids are exposed; the correct one is chosen strictly by the + # selected environment below (no cross-environment fallback). RENDER_STAGING_SERVICE_ID: ${{ secrets.RENDER_STAGING_SERVICE_ID }} RENDER_PRODUCTION_SERVICE_ID: ${{ secrets.RENDER_PRODUCTION_SERVICE_ID }} GITHUB_SHA: ${{ github.sha }} run: | - if [ "${{ github.ref_name }}" = "main" ]; then + if [ "${{ inputs.environment }}" = "production" ]; then export RENDER_SERVICE_ID="$RENDER_PRODUCTION_SERVICE_ID" else export RENDER_SERVICE_ID="$RENDER_STAGING_SERVICE_ID" fi if [ -z "$RENDER_API_KEY" ] || [ -z "$RENDER_SERVICE_ID" ]; then - echo "ERROR: RENDER_API_KEY and the ${{ steps.target.outputs.env_name }} service id must be configured as GitHub secrets." + echo "ERROR: RENDER_API_KEY and the ${{ inputs.environment }} service id must be configured as GitHub secrets." exit 1 fi python scripts/render_deploy.py - # ── Health gate (against the branch-specific URL) ────────────────── + # ── Health gate (against the selected environment's URL) ─────────── - name: Health gate check - if: steps.target.outputs.is_protected == 'true' env: STAGING_API_URL: ${{ secrets.STAGING_API_URL }} PRODUCTION_API_URL: ${{ secrets.PRODUCTION_API_URL }} run: | - if [ "${{ github.ref_name }}" = "main" ]; then + if [ "${{ inputs.environment }}" = "production" ]; then API_URL="$PRODUCTION_API_URL" else API_URL="$STAGING_API_URL" fi if [ -z "$API_URL" ]; then - echo "ERROR: the ${{ steps.target.outputs.env_name }} API URL secret is not configured." + echo "ERROR: the ${{ inputs.environment }} API URL secret is not configured." exit 1 fi MAX_RETRIES=5 @@ -125,12 +117,12 @@ jobs: sleep $RETRY_DELAY done - echo "ERROR: Health gate failed after $MAX_RETRIES attempts on the ${{ steps.target.outputs.env_name }} environment." + echo "ERROR: Health gate failed after $MAX_RETRIES attempts on the ${{ inputs.environment }} environment." exit 1 - # ── Smoke tests (against the same branch-specific URL) ───────────── + # ── Smoke tests (optional; against the same environment URL) ─────── - name: Run smoke tests against live deployment - if: steps.target.outputs.is_protected == 'true' + if: inputs.run_smoke_tests env: STAGING_API_URL: ${{ secrets.STAGING_API_URL }} PRODUCTION_API_URL: ${{ secrets.PRODUCTION_API_URL }} @@ -141,21 +133,14 @@ jobs: AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} RUN_REAL_SCAN: "true" run: | - if [ "${{ github.ref_name }}" = "main" ]; then + if [ "${{ inputs.environment }}" = "production" ]; then export API_URL="$PRODUCTION_API_URL" else export API_URL="$STAGING_API_URL" fi if [ -z "$JWT_SECRET" ]; then - echo "ERROR: JWT_SECRET is required to run smoke tests against a protected environment." + echo "ERROR: JWT_SECRET is required to run smoke tests against a deployed environment." exit 1 fi echo "Running smoke tests against: $API_URL" python tests/smoke_test.py - - # ── Non-protected refs ───────────────────────────────────────────── - - name: Skip deploy for non-protected ref - if: steps.target.outputs.is_protected != 'true' - run: | - echo "Ref '${{ github.ref_name }}' has no Render environment (only dev->staging and main->production deploy)." - echo "Skipping deploy, health gate, and smoke tests." diff --git a/docs/deployment/render.md b/docs/deployment/render.md index 719b5dc..796f37d 100644 --- a/docs/deployment/render.md +++ b/docs/deployment/render.md @@ -8,6 +8,14 @@ fail based on Render's *actual* deploy status, never on a fixed timer. - Deploy script: [`scripts/render_deploy.py`](../../scripts/render_deploy.py) - Workflow: [`.github/workflows/deploy.yml`](../../.github/workflows/deploy.yml) +> **Deploys are currently MANUAL only.** The workflow runs solely via +> `workflow_dispatch` (Actions → *Deploy API to Render* → *Run workflow*), where you +> pick the target environment (`staging` or `production`). There is **no automatic +> deploy on push or merge**: the staging/production Render services, database, and +> secrets must exist first, so auto-deploying before that infrastructure is +> provisioned would fail. Automatic push deploys can be re-enabled later once +> maintainers confirm the staging/prod Render setup. + --- ## 1. Service layout @@ -21,14 +29,18 @@ The blueprint declares four services across two environments: | `openshield-api` | web | `main` | `./startup.sh` | Production API (Gunicorn) | | `openshield-worker` | worker | `main` | `python -m scanner.worker` | Production scan worker | -### Staging / production branch mapping +### Staging / production mapping + +The blueprint tracks branches so that, once synced, each Render service builds from +the right branch: - `dev` → **staging** (`openshield-api-staging`, `openshield-worker-staging`) - `main` → **production** (`openshield-api`, `openshield-worker`) -The staging and production APIs are served from **separate URLs** and CI health-checks -and smoke-tests each branch against its own URL — they no longer share a fallback -production URL. +When you run the manual deploy workflow you choose the environment explicitly +(`staging` or `production`); the workflow deploys that environment's service and +health-checks/smoke-tests it against its own URL. Staging and production are served +from **separate URLs** — they do not share a fallback production URL. ### The worker is a separate service @@ -97,16 +109,19 @@ Render never overwrites the values you set in its dashboard. | `AZURE_CLIENT_SECRET` | Real-scan smoke tests | | `AZURE_TENANT_ID` | Real-scan smoke tests | -The workflow selects the service id and API URL automatically from the branch: -`dev` uses the staging pair, `main` uses the production pair. +The workflow selects the service id and API URL from the environment you pick when +running it: `staging` uses the staging pair, `production` uses the production pair. --- ## 4. How deterministic deploy polling works -On push to `dev`/`main` (or manual `workflow_dispatch`), the workflow runs +The deploy workflow is triggered **manually** (`workflow_dispatch`) with an +`environment` input (`staging` or `production`) and an optional `run_smoke_tests` +toggle. It runs [`scripts/render_deploy.py`](../../scripts/render_deploy.py) with `RENDER_API_KEY`, -the branch's `RENDER_SERVICE_ID`, and `GITHUB_SHA`. The script: +the selected environment's `RENDER_SERVICE_ID`, and `GITHUB_SHA` (the commit the +workflow was dispatched on). The script: 1. **Triggers a deploy** pinned to the commit CI built: `POST https://api.render.com/v1/services/{serviceId}/deploys` with @@ -126,28 +141,29 @@ Poll interval and timeout are tunable via `RENDER_DEPLOY_POLL_SECONDS` never printed. Only after the deploy is live does the workflow run the **health gate** -(`GET {API_URL}/health` must return 200) and the **smoke tests** -(`tests/smoke_test.py`) against the branch-specific URL. On `dev`/`main`, any of -these failing fails the workflow. +(`GET {API_URL}/health` must return 200) and, when `run_smoke_tests` is enabled, the +**smoke tests** (`tests/smoke_test.py`) against the selected environment's URL. Any +of these failing fails the workflow run. ### Deploy control (`autoDeployTrigger`) and avoiding duplicate deploys `render.yaml` uses `autoDeployTrigger` (the current field; it replaces the deprecated `autoDeploy`): -- **Web services → `autoDeployTrigger: "off"`.** GitHub Actions is the single - source of truth for web deploys, so Render's own auto-deploy is turned off. If it - were left on, every push would trigger *both* a Render auto-deploy and the - Actions-driven deploy of the same service — duplicate, racing deploys. `"off"` is - quoted because unquoted `off` is parsed as the boolean `false` by YAML. -- **Worker services → `autoDeployTrigger: commit`.** The workers **auto-deploy on - every push** to their branch. Nothing else deploys them, so there is no - duplication, and they stay in sync with the web service. The health gate and - smoke tests do **not** cover the worker; if you want a worker deployed - deterministically (e.g. to gate or roll it back precisely), run - `scripts/render_deploy.py` with `RENDER_SERVICE_ID` set to the worker's service id. - -In short: **web = deployed by the workflow; worker = auto-deployed by Render on +- **Web services → `autoDeployTrigger: "off"`.** The manual deploy workflow is the + single source of truth for web deploys, so Render's own auto-deploy is turned off. + This also avoids duplicate/racing deploys if a `push:` trigger is added later + (Render auto-deploy *and* the Actions deploy firing for the same commit). `"off"` + is quoted because unquoted `off` is parsed as the boolean `false` by YAML. +- **Worker services → `autoDeployTrigger: commit`.** Once the blueprint is synced + and the worker infrastructure exists, the workers auto-deploy on push to their + branch; nothing else deploys them, so there is no duplication. Until that + infrastructure exists, no worker deploys happen at all. The health gate and smoke + tests do **not** cover the worker; to deploy a worker deterministically (e.g. to + gate or roll it back precisely), run `scripts/render_deploy.py` with + `RENDER_SERVICE_ID` set to the worker's service id. + +In short: **web = deployed by the manual workflow; worker = auto-deployed by Render on push** (and optionally by the same script). --- @@ -165,8 +181,9 @@ API_URL="$STAGING_API_URL" JWT_SECRET="" \ python tests/smoke_test.py ``` -Confirm `dev` is serving staging: push a change to `dev`, watch the -"Deploy API to Render" workflow reach *live*, then re-run the health check. +Confirm staging is deployable: in **Actions → *Deploy API to Render* → *Run +workflow***, choose `environment: staging` (from the `dev` branch), watch the deploy +step reach *live*, then re-run the health check. ### Production @@ -230,6 +247,10 @@ a bad commit changed worker behaviour. workflow. Gate on Render's real deploy status via `scripts/render_deploy.py`. - ❌ **Do not run the scan worker inside the web container** (no background subshell in `startup.sh`). It must stay a separate Render worker service. +- ❌ **Do not add a `push:` trigger to the deploy workflow** until maintainers have + confirmed the staging/prod Render services, database, and secrets exist. Auto- + deploying before the infrastructure is provisioned will fail. Deploys stay manual + (`workflow_dispatch`) until then. - ❌ **Do not set the web services to `autoDeployTrigger: commit`** (or re-add the deprecated `autoDeploy: true`) while GitHub Actions also deploys them — that causes duplicate/racing deploys. Keep web services on `autoDeployTrigger: "off"`.