deploy-aws.yml

name: Deploy to AWS

# Replaces `daily-deploy.yml`, which was written for a Proxmox host: it rsynced
# research artifacts into a directory on a self-hosted runner and restarted a
# docker-compose service. The deployment is AWS now — Terraform owns the
# infrastructure, Ansible owns the host — and that workflow had no executor
# anyway, because the repository has no registered self-hosted runner.
#
# **This workflow is not what makes a deployment correct.** The refusals that
# do that live in `infra/` and run wherever the deployment runs: Terraform
# refuses an image not pinned by digest, Ansible refuses a database behind the
# code's migration head, and the host refuses to start without a model key. See
# `infra/README.md` for the full table. What this adds is that the same
# procedure runs from a commit rather than from somebody's shell, and that its
# result is *proved* rather than assumed.
#
# **The proof is the last step and it is not optional.** A deployment job that
# ends when Ansible exits has reported that a playbook ran, not that the
# service is serving the revision it was given. Those differ every time an
# image fails to pull, a container restarts into an old layer, or a health
# check passes against the previous process.

on:
  workflow_dispatch:
    inputs:
      environment:
        description: 'tfvars file under infra/terraform/environments'
        required: false
        default: 'test'

# Least privilege, and `id-token` for OIDC when the account is wired for it.
permissions:
  contents: read
  id-token: write

concurrency:
  group: deploy-aws
  cancel-in-progress: false

env:
  AWS_REGION: us-east-1
  ECR_REPOSITORY: quantify
  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
  AWS_DEFAULT_REGION: us-east-1

jobs:
  deploy:
    runs-on: ubuntu-latest
    timeout-minutes: 60

    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
        with:
          # `vendor/discovery-runtime` is a submodule and pip installs
          # from it. Without this the directory is empty in CI and the
          # build fails on a path that exists locally — which is how the
          # vendoring fix worked on a developer machine and moved the
          # problem into the pipeline rather than solving it.
          submodules: recursive

      - name: Fetch the submodule's tags (the build gate requires an exact tag)
        run: |
          # actions/checkout populates the submodule at its gitlink commit but does
          # NOT fetch that submodule's tags, so build_image.sh's
          # `git describe --tags --exact-match` sees no tag and refuses a correctly
          # pinned release (the gitlink IS v0.1.12 — proven locally). Fetch them; the
          # checkout above already persisted credentials for the submodule.
          git -C vendor/discovery-runtime fetch --tags --force origin

      - name: Refuse to deploy a dirty or ambiguous revision
        run: |
          # The commit is the deployment's identity, so it must be exactly the
          # one being built. A tag or a branch name would let two deployments
          # claim one identity.
          echo "COMMIT=${GITHUB_SHA}" >> "$GITHUB_ENV"
          echo "SHORT=${GITHUB_SHA::7}" >> "$GITHUB_ENV"
          echo "deploying ${GITHUB_SHA} from ${GITHUB_REF_NAME}"

      # Credentials through the environment rather than through
      # `aws-actions/configure-aws-credentials`. The repository's Actions
      # policy is `selected` with one named third party, and this workflow was
      # blocked by it on the first attempt — correctly. Widening the allowlist
      # to run a deployment is the wrong direction, and the AWS CLI and
      # Terraform both read these variables natively, so the action buys
      # nothing here but another supply chain.

      - name: Install pytest for the serving-image contract gate
        run: |
          # build_image.sh's step 5 runs `python3 -m pytest
          # tests/test_serving_image_contract.py` ON THE RUNNER (the test probes the
          # built image via `docker run`). The runner's python3 has no pytest, so the
          # gate failed with "No module named pytest". Install it (the only host dep the
          # contract test needs); --break-system-packages covers a PEP 668 runner.
          python3 -m pip install --quiet pytest \
            || python3 -m pip install --quiet --break-system-packages pytest

      - name: Build and push the image, pinned by digest
        id: image
        env:
          AWS_REGION: ${{ env.AWS_REGION }}
          ECR_REPOSITORY: ${{ env.ECR_REPOSITORY }}
        run: |
          set -euo pipefail
          # One implementation, invoked rather than repeated.
          #
          # This was thirteen lines of inline shell, which is invisible to
          # everything that is not this workflow — so the moment anything else
          # needed to build the image it would have got a second procedure.
          # That is how the two requirements files came to answer "which
          # Discovery Runtime?" differently.
          #
          # The script also verifies things this step never did: that the
          # submodule is at the gitlink this commit records, that the image
          # holds that runtime rather than a cached one, and that the ten
          # serving-image gates pass — including that a container without the
          # Stanza model refuses to serve instead of quietly becoming a
          # MODEL_ONLY server.
          IMAGE="$(scripts/build_image.sh --push | tail -1)"
          echo "image=${IMAGE}" >> "$GITHUB_OUTPUT"
          echo "pinned ${IMAGE}"

      - name: Verify the release manifest binds this image (freeze §5 promotion gate)
        env:
          IMAGE: ${{ steps.image.outputs.image }}
        run: |
          set -euo pipefail
          # build_image.sh wrote release-manifest.json for the image it just
          # pushed. The promotion gate refuses if the image about to be applied is
          # not the one the manifest binds — so a stale digest can never select
          # production code independently of the release identity (the incident).
          DEPLOYED_DIGEST="${IMAGE##*@}"
          python3 -m deploy.release.manifest verify \
            --manifest release-manifest.json \
            --actual "{\"image_digest\":\"${DEPLOYED_DIGEST}\",\"app_commit\":\"${COMMIT}\"}"

      - name: Install Terraform
        run: |
          # Pinned to the version that wrote the state (`terraform state pull` ->
          # terraform_version). A newer terraform would rewrite the state to its own
          # version and an older one would refuse it, so this must match, not float.
          # Installed by direct download rather than a marketplace action — the repo's
          # Actions allowlist is `selected`, and a run step needs no addition to it.
          ver=1.14.9
          curl -fsSL -o /tmp/terraform.zip \
            "https://releases.hashicorp.com/terraform/${ver}/terraform_${ver}_linux_amd64.zip"
          mkdir -p "$HOME/bin"
          unzip -o /tmp/terraform.zip -d "$HOME/bin"
          echo "$HOME/bin" >> "$GITHUB_PATH"
          "$HOME/bin/terraform" version

      - name: Materialize the tfvars + backend config
        working-directory: infra/terraform
        env:
          TFVARS: ${{ github.event.inputs.environment || 'test' }}
          CF_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
          CF_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }}
        run: |
          set -euo pipefail
          # backend.hcl and environments/*.tfvars are both gitignored, so a fresh
          # checkout has neither and `terraform init`/`apply` have nothing to read.
          # Rebuild both here from the non-secret config inline (the S3 state bucket +
          # the env-scoped key; the two Cloudflare ids from secrets). build_commit /
          # build_release_ref are NOT written — the apply step passes them by -var.
          cat > backend.hcl <<EOF
          bucket       = "quantify-tfstate-388062344663"
          key          = "quantify/${TFVARS}/terraform.tfstate"
          region       = "us-east-1"
          use_lockfile = true
          encrypt      = true
          EOF
          : "${CF_ACCOUNT_ID:?set the CLOUDFLARE_ACCOUNT_ID secret}"
          : "${CF_ZONE_ID:?set the CLOUDFLARE_ZONE_ID secret}"
          cat > "environments/${TFVARS}.tfvars" <<EOF
          project     = "quantify"
          environment = "test"
          region      = "us-east-1"
          domain_name           = "quantify.club"
          identity_domain_name  = "auth.quantify.club"
          workspace_domain_name = "workspace.quantify.club"
          cloudflare_account_id = "${CF_ACCOUNT_ID}"
          cloudflare_zone_id    = "${CF_ZONE_ID}"
          registry_host     = "388062344663.dkr.ecr.us-east-1.amazonaws.com"
          build_snapshot_id = "syn-2026-08"
          instance_type           = "t3.small"
          db_instance_class       = "db.t4g.micro"
          db_allocated_storage_gb = 20
          db_multi_az             = false
          pilot_data_policy = "market-data-egress/pilot-vendor-approved@1"
          parser_mode       = "RUNTIME"
          parser_provider   = "OPENAI"
          parser_model      = "gpt-5.4-2026-03-05"
          parser_fallback   = "REFUSE"
          alert_email = "redevops@redevops.io"
          enable_kubernetes  = true
          cluster_albs_ready = true
          EOF
          echo "wrote environments/${TFVARS}.tfvars (Cloudflare ids from secrets)"

      - name: Terraform apply
        working-directory: infra/terraform
        env:
          IMAGE: ${{ steps.image.outputs.image }}
          TFVARS: ${{ github.event.inputs.environment || 'test' }}
          # The Cloudflare provider reads this from the environment (providers.tf keeps
          # it out of the plan/state on purpose); the AWS creds come from the job env.
          CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
        run: |
          set -euo pipefail
          terraform init -backend-config=backend.hcl -input=false
          # `application_image` is deliberately NOT passed. It used to be, and
          # that is the two-variable design that let a deploy declare one commit
          # while running another image: nothing tied them together, and on
          # 2026-08-12 they drifted. Terraform now resolves the digest from the
          # commit's own tag, so this run cannot supply a mismatched pair even
          # by mistake. The push step above still verifies the tag exists.
          terraform apply -input=false -auto-approve \
            -var-file="environments/${TFVARS}.tfvars" \
            -var "build_commit=${COMMIT}" \
            -var "build_release_ref=${GITHUB_REF_NAME}"

      - name: Install the Kubernetes client for Ansible
        run: |
          # kubernetes.core.k8s imports the `kubernetes` python library, and its modules
          # run under ansible-core's OWN interpreter. On the runner ansible-core is a
          # pipx app (/opt/pipx/venvs/ansible-core), so a system `pip install` is invisible
          # to it — the play failed with "Failed to import the required Python library
          # (kubernetes)". Inject it into the ansible-core venv; fall back to a
          # system / PEP-668 install if ansible-core is not a pipx app.
          pipx inject ansible-core kubernetes \
            || python3 -m pip install --quiet kubernetes \
            || python3 -m pip install --quiet --break-system-packages kubernetes

      - name: Ansible deploy
        working-directory: infra/ansible
        env:
          # services.yml writes the parser's model key into the quantify-web Secret from
          # lookup('env', 'OPENAI_API_KEY') and refuses a blank one; the AWS creds (for
          # eks/kubectl/secretsmanager) come from the job env.
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          set -euo pipefail
          ansible-galaxy collection install -r requirements.yml
          # The application services only. `identity` and `tunnel` are
          # deliberately not deployed here: Zitadel restarts on a Recreate
          # strategy and the connector is the sole ingress path, so a routine
          # application deploy has no business interrupting either. Both are
          # deployed on purpose, by name, when they change.
          ansible-playbook services.yml -e service=data,evaluate,web

      - name: Prove the running service is this revision
        env:
          # Read back from Terraform rather than carried from the build step, so
          # the artifact records the image the deployment actually resolved.
          IMAGE: ${{ steps.image.outputs.image }}
        run: |
          set -euo pipefail
          # Both halves, and neither substitutes for the other: the service must
          # be able to identify itself, and the identity it offers must be the
          # commit this run deployed. Anything else means a user is handed a
          # different program than the one answering them.
          python scripts/verify_deployment_identity.py \
            --url "https://quantify.club" --expect-commit "${COMMIT}" \
            --release-ref "${GITHUB_REF_NAME}" --image "${IMAGE}" \
            | tee deployment-proof.txt

      - name: Preserve the proof
        # Kept because a cohort observation joins to code through it:
        #   cohort event -> serving_commit -> this proof -> repository revision
        # The running service is the perishable half; this file is the durable
        # one, and it must outlive the deployment it describes.
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: deployment-proof-${{ github.sha }}
          path: deployment-proof.txt
          retention-days: 90