From 2c3082a3edfde01351c1031c9ef9791250d9e233 Mon Sep 17 00:00:00 2001 From: Niels Poensgen Date: Tue, 4 Aug 2026 11:39:50 +0200 Subject: [PATCH] Add CF log bridge, workflows and drain infra --- .github/workflows/cf-log-bridge-deploy.yml | 178 +++++++++++++++++++++ .github/workflows/cf-log-bridge-image.yml | 70 ++++++++ apps/backend/manifest.yml | 3 + infra/cf-log-bridge/Dockerfile | 10 ++ infra/cf-log-bridge/README.md | 103 ++++++++++++ infra/cf-log-bridge/manifest.yml | 15 ++ infra/cf-log-bridge/vector.yaml | 136 ++++++++++++++++ infra/terraform/cloud-foundry/README.md | 31 ++++ infra/terraform/cloud-foundry/log-drain.tf | 31 ++++ infra/terraform/cloud-foundry/variables.tf | 34 ++++ 10 files changed, 611 insertions(+) create mode 100644 .github/workflows/cf-log-bridge-deploy.yml create mode 100644 .github/workflows/cf-log-bridge-image.yml create mode 100644 infra/cf-log-bridge/Dockerfile create mode 100644 infra/cf-log-bridge/README.md create mode 100644 infra/cf-log-bridge/manifest.yml create mode 100644 infra/cf-log-bridge/vector.yaml create mode 100644 infra/terraform/cloud-foundry/log-drain.tf diff --git a/.github/workflows/cf-log-bridge-deploy.yml b/.github/workflows/cf-log-bridge-deploy.yml new file mode 100644 index 000000000..27fbabd88 --- /dev/null +++ b/.github/workflows/cf-log-bridge-deploy.yml @@ -0,0 +1,178 @@ +name: CF log bridge deploy to STACKIT Cloud Foundry + +on: + push: + branches: [staging, main] + paths: ["infra/cf-log-bridge/**", ".github/workflows/cf-log-bridge-deploy.yml"] + workflow_dispatch: + +permissions: + contents: read + packages: read + +env: + IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/baergpt-log-bridge + # Suffix must match the CF space names and the drain URL built in + # infra/terraform/cloud-foundry/log-drain.tf. + APP_NAME: baergpt-log-bridge-${{ github.ref_name == 'main' && 'prod' || 'staging' }} + +jobs: + deploy: + runs-on: ubuntu-latest + concurrency: + group: cf-log-bridge-deploy-${{ github.ref }} + cancel-in-progress: false + # workflow_dispatch has no branches filter, so guard the ref here. + if: ${{ github.ref_name == 'staging' || github.ref_name == 'main' }} + environment: ${{ github.ref_name == 'main' && 'Production' || 'Staging' }} + + steps: + - name: Checkout code ⬇️ + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0, pinned to commit sha to avoid sha-hulud like attacks + with: + persist-credentials: "false" + + - name: Setup environment ⚙️ + uses: ./.github/actions/setup + with: + install-1password: "true" + + # Same formula as cf-log-bridge-image.yml, so it resolves to the same tag. + - name: Compute image tag 🏷️ + id: tag + run: | + set -euo pipefail + echo "tag=content-sha-${{ hashFiles('infra/cf-log-bridge/vector.yaml', 'infra/cf-log-bridge/Dockerfile') }}" >> "$GITHUB_OUTPUT" + + - name: Set 1Password item IDs based on branch 🌿 + id: env-config + run: | + set -euo pipefail + if [[ "${{ github.ref_name }}" == "main" ]]; then + echo "cf_item_id=${{ secrets.OP_CF_DEPLOY_PRODUCTION_ENV_ITEM_ID }}" >> "$GITHUB_OUTPUT" + echo "app_item_id=${{ secrets.OP_LOG_BRIDGE_PRODUCTION_ENV_ITEM_ID }}" >> "$GITHUB_OUTPUT" + else + echo "cf_item_id=${{ secrets.OP_CF_DEPLOY_STAGING_ENV_ITEM_ID }}" >> "$GITHUB_OUTPUT" + echo "app_item_id=${{ secrets.OP_LOG_BRIDGE_STAGING_ENV_ITEM_ID }}" >> "$GITHUB_OUTPUT" + fi + + - name: Fetch STACKIT Cloud Foundry credentials from 1Password 🔑 + run: | + set -euo pipefail + op read "op://${{ secrets.OP_VAULT_ID }}/${{ steps.env-config.outputs.cf_item_id }}/notesPlain" > cf.env + while IFS= read -r line || [ -n "$line" ]; do + [[ -z "$line" || "$line" == \#* ]] && continue + key="${line%%=*}" + value="${line#*=}" + echo "::add-mask::$value" + echo "$key=$value" >> "$GITHUB_ENV" + done < cf.env + rm cf.env + env: + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + + # Both workflows fire on the same push, so the build may still be running. + - name: Wait for image to exist in GHCR 🔍 + run: | + set -euo pipefail + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin + image="${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}" + timeout=600 + counter=0 + until docker manifest inspect "$image" >/dev/null 2>&1; do + if [ $counter -ge $timeout ]; then + echo "::error::Image $image not found in GHCR after ${timeout}s." \ + "Ensure 'CF log bridge image build and push' ran for this commit." + exit 1 + fi + echo "Waiting for $image to be available on GHCR... (${counter}s)" + sleep 10 + counter=$((counter + 10)) + done + echo "Image $image found." + + - name: Install cf CLI ⚙️ + run: | + set -euo pipefail + curl -sL "https://packages.cloudfoundry.org/stable?release=linux64-binary&version=v8&source=github" \ + | tar -zx + sudo mv cf8 /usr/local/bin/cf + + - name: Target STACKIT Cloud Foundry 🎯 + run: | + set -euo pipefail + cf api "$CF_API_ENDPOINT" + cf auth + cf target -o "$CF_ORG" -s "$CF_SPACE" + + # KEY=VALUE lines in the item's notes. See infra/cf-log-bridge/README.md. + - name: Fetch bridge env vars from 1Password 🔑 + run: | + set -euo pipefail + op read "op://${{ secrets.OP_VAULT_ID }}/${{ steps.env-config.outputs.app_item_id }}/notesPlain" > bridge.env + : > infra/cf-log-bridge/app.env + while IFS= read -r line || [ -n "$line" ]; do + [[ -z "$line" || "$line" == \#* ]] && continue + key="${line%%=*}" + value="${line#*=}" + echo "::add-mask::$value" + echo "$key=$value" >> infra/cf-log-bridge/app.env + done < bridge.env + rm bridge.env + env: + OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }} + + # cf set-env needs the app to exist, so the first deploy creates it with + # --no-start. Unsetting first means keys removed from 1Password do not linger. + - name: Sync bridge env vars 🔄 + id: sync + working-directory: infra/cf-log-bridge + env: + CF_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + if ! cf app "${{ env.APP_NAME }}" >/dev/null 2>&1; then + echo "bootstrapped=true" >> "$GITHUB_OUTPUT" + cf push \ + -f manifest.yml \ + --no-start \ + --var app_name="${{ env.APP_NAME }}" \ + --var image="${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}" \ + --var docker_username="${{ github.actor }}" + fi + + guid=$(cf app "${{ env.APP_NAME }}" --guid) + # Assigned first: a substitution inside `for ... in` discards its exit status, so + # a failed cf curl would silently skip the unset and leave stale vars behind. + keys=$(cf curl "/v3/apps/$guid/environment_variables" | jq -r '.var | keys[]') + for key in $keys; do + cf unset-env "${{ env.APP_NAME }}" "$key" >/dev/null + done + + while IFS= read -r line || [ -n "$line" ]; do + key="${line%%=*}" + value="${line#*=}" + cf set-env "${{ env.APP_NAME }}" "$key" "$value" >/dev/null + done < app.env + + rm -f app.env + + # Rolling keeps the old version serving the drain until the new one is up. The + # first deploy has nothing to roll over, hence the flag from the sync step. + - name: Push image to Cloud Foundry 🚀 + working-directory: infra/cf-log-bridge + env: + CF_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + cf push \ + -f manifest.yml \ + ${{ steps.sync.outputs.bootstrapped != 'true' && '--strategy rolling' || '' }} \ + --var app_name="${{ env.APP_NAME }}" \ + --var image="${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }}" \ + --var docker_username="${{ github.actor }}" \ + || { cf cancel-deployment "${{ env.APP_NAME }}" 2>/dev/null || true; exit 1; } + + - name: Dump recent app logs on failure 🪵 + if: ${{ failure() && !cancelled() }} + run: cf logs "${{ env.APP_NAME }}" --recent diff --git a/.github/workflows/cf-log-bridge-image.yml b/.github/workflows/cf-log-bridge-image.yml new file mode 100644 index 000000000..500c29358 --- /dev/null +++ b/.github/workflows/cf-log-bridge-image.yml @@ -0,0 +1,70 @@ +name: CF log bridge image build and push to registry + +on: + pull_request: + branches: [staging, main] + paths: ["infra/cf-log-bridge/**", ".github/workflows/cf-log-bridge-image.yml"] + push: + branches: [staging, main] + paths: ["infra/cf-log-bridge/**", ".github/workflows/cf-log-bridge-image.yml"] + workflow_dispatch: + +permissions: + contents: read + packages: write + +env: + IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/baergpt-log-bridge + +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + - name: Checkout code ⬇️ + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: "2" + + # The image is the upstream Vector release plus this directory's config, so the + # config alone determines whether a rebuild is needed. Fixtures and the local + # harness are excluded: they never enter the image. + - name: Compute image tag 🏷️ + id: tag + run: | + echo "tag=content-sha-${{ hashFiles('infra/cf-log-bridge/vector.yaml', 'infra/cf-log-bridge/Dockerfile') }}" >> "$GITHUB_OUTPUT" + + - name: Log in to GitHub Container Registry 🔑 + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Runner's default docker driver can't export cache (type=gha); switches + # to the docker-container driver, which supports --cache-to. + - name: Set up Docker Buildx ⚙️ + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + + - name: Check if image already exists in GHCR 🔍 + id: check + run: | + if docker manifest inspect ${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }} >/dev/null 2>&1; then + echo "exists=true" >> "$GITHUB_OUTPUT" + else + echo "exists=false" >> "$GITHUB_OUTPUT" + fi + + - name: Build and push image 🐳 + if: steps.check.outputs.exists != 'true' + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: infra/cf-log-bridge + file: infra/cf-log-bridge/Dockerfile + platforms: linux/amd64 + push: true + cache-from: type=gha + cache-to: type=gha,mode=max + tags: | + ${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.tag }} + ${{ env.IMAGE_NAME }}:commit-${{ github.sha }} diff --git a/apps/backend/manifest.yml b/apps/backend/manifest.yml index 00c0ffe98..4484d4cd2 100644 --- a/apps/backend/manifest.yml +++ b/apps/backend/manifest.yml @@ -17,6 +17,9 @@ applications: health-check-http-endpoint: /health health-check-invocation-timeout: 10 default-route: true + # Created by infra/terraform/cloud-foundry/log-drain.tf + services: + - stackit-drain # env vars (API keys, DB URL, etc.) aren't set here — the deployment workflow # syncs them onto the app via `cf set-env` from a 1Password item on every # deploy, so the same image runs across environments — see diff --git a/infra/cf-log-bridge/Dockerfile b/infra/cf-log-bridge/Dockerfile new file mode 100644 index 000000000..5562ba1d7 --- /dev/null +++ b/infra/cf-log-bridge/Dockerfile @@ -0,0 +1,10 @@ +FROM --platform=linux/amd64 timberio/vector:0.57.0-debian + +COPY vector.yaml /etc/vector/vector.yaml + +EXPOSE 8080 + +# Vector does not substitute env vars into its config unless this flag is set: an +# endpoint written as ${SOME_URL} would stay that literal text instead of becoming the +# value. Baked in here rather than set as an app env var so a deploy cannot omit it. +CMD ["--config", "/etc/vector/vector.yaml", "--dangerously-allow-env-var-interpolation"] diff --git a/infra/cf-log-bridge/README.md b/infra/cf-log-bridge/README.md new file mode 100644 index 000000000..96d63a717 --- /dev/null +++ b/infra/cf-log-bridge/README.md @@ -0,0 +1,103 @@ +# CF log bridge + +Cloud Foundry can forward an app's logs to a URL you nominate, as syslog over HTTPS. +STACKIT Observability does not accept syslog, so this Vector app sits in between: +it receives the drain, and pushes logs to Loki and container metrics to Prometheus +remote-write. + +One instance runs per CF space, named `baergpt-log-bridge-`. `default-route: true` +turns that name into the route, and a route belongs to one space only, so the suffix is +what keeps staging and prod apart. The deploy workflow and +`infra/terraform/cloud-foundry/log-drain.tf` build the name separately and must agree. + +The drain itself is a `stackit-drain` service defined in +`infra/terraform/cloud-foundry/log-drain.tf`, and apps opt in by naming it in their own +manifest. + +**Never bind this app to `stackit-drain`.** Its own output would be drained back into +itself, and the resulting amplification trips the platform's limit of 1000 log-lines per +second per instance, after which lines are dropped for every app on that cell. + +Design notes: `docs/superpowers/specs/2026-07-30-cloud-foundry-observability-design.md`. + +## Environment + +Set by `.github/workflows/cf-log-bridge-deploy.yml` from a 1Password item, which is the +single source of truth: the workflow unsets every existing variable before re-setting. + +| Variable | Notes | +| ------------------------------------------------ | ------------------------------------------------------------------------------- | +| `BAERGPT_ENV` | `staging` or `production`. Becomes the `source` label as `cf-` | +| `DRAIN_USERNAME`, `DRAIN_PASSWORD` | Basic auth on the inbound endpoint. Must match the credentials in the drain URL | +| `STACKIT_OBSERVABILITY_USERNAME`, `..._PASSWORD` | From the Observability instance's Credentials tab | +| `STACKIT_OBSERVABILITY_LOGS_ENDPOINT` | Base URL only. Vector appends `/loki/api/v1/push` | +| `STACKIT_OBSERVABILITY_METRICS_ENDPOINT` | Used verbatim, including `/api/v1/receive` | +| `VECTOR_LOG` | `warn` in deployed environments | + +The two endpoints are handled in opposite ways. Copying the portal's logs URL verbatim +gives a doubled path and a 404. + +## Output + +Loki labels: `source`, `app_name`, `space_name`, `source_type`, `instance_id`, `level`. + +`process_instance_id` and the cell `index` stay on the event and are deliberately not +promoted. They are per-container and per-cell, and unbounded label values break Loki. +Same reasoning as `container.id` in `infra/supabase/MONITORING.md`. + +`source` is load-bearing. Alert rules select on it, and a rule matching no series applies +cleanly and never fires. + +Metrics are named `cf_`: `cf_cpu`, `cf_cpu_entitlement`, `cf_memory`, +`cf_memory_quota`, `cf_disk`, `cf_disk_quota`, `cf_rx_bytes`, `cf_tx_bytes`, +`cf_log_rate`, `cf_log_rate_limit`, `cf_container_age`. All are gauges. `rx_bytes` and +`tx_bytes` are cumulative, so `rate()` works, but a container restart appears as a drop +rather than a counter reset. + +STACKIT's ingest adds its own `receive` and `tenant_id` labels. Do not select on them. + +## Frame shape + +Useful when changing the remap in `vector.yaml`. Everything CF sends is an RFC5424 +syslog frame: + +``` +<14>1 2026-07-30T12:08:43Z baergpt.staging.log-emitter 6c5662d9-... [APP/PROC/WEB/0] - [tags@47450 ...] drain-probe +│ │ │ │ │ │ │ │ │ +│ │ │ │ │ │ │ │ └─ message +│ │ │ │ │ │ │ └─ structured data +│ │ │ │ │ │ └─ msgid: unused by CF +│ │ │ │ │ └─ procid: source type and instance +│ │ │ │ └─ app-name: the app GUID +│ │ │ └─ hostname: org.space.app +│ │ └─ timestamp +│ └─ version +└─ PRI: facility x 8 + severity +``` + +PRI packs two numbers into one: `14` is facility 1 (user) times 8 plus severity 6 (info). +Vector decodes it to a `severity` string, which becomes the `level` label — a line written +to stderr arrives as `<11>`, severity 3, and is labelled `error` with no work from the app. + +Structured data is exposed under its full element id: `."tags@47450"`, `."gauge@47450"`, +`."counter@47450"`. Vector also sets its own top-level `source_type` (`http_server`), so +CF's is carried as `cf_source_type` and renamed in the Loki sink's label map. + +Logs and metrics share one connection and differ in four fields: + +| | Log frame | Metric frame | +| ------------------------------------- | ------------------------------------------- | ---------------------- | +| `message` | the line as written | empty | +| `procid` | source type (`[APP/PROC/WEB/0]`, `[RTR/5]`) | instance index (`[0]`) | +| `."tags@47450".source_type` | the clean source type | absent | +| `."gauge@47450"` / `."counter@47450"` | absent | the value, as a string | + +## Access control + +CF authenticates using credentials in the drain URL's userinfo +(`https://user:pass@/`). That is its only mechanism, so the same pair must +be set on this app and in the drain URL. Rotating means changing both together. + +This is load-bearing rather than defence in depth: Loki labels are built from fields in +the received frame, so an open endpoint would let anyone forge structured data and mint +arbitrary label values in an instance shared by every environment. diff --git a/infra/cf-log-bridge/manifest.yml b/infra/cf-log-bridge/manifest.yml new file mode 100644 index 000000000..7f8cda093 --- /dev/null +++ b/infra/cf-log-bridge/manifest.yml @@ -0,0 +1,15 @@ +# ((image)), ((docker_username)) and ((app_name)) are injected at push time via +# `cf push --var ...`. Runtime env is set by the deploy workflow from 1Password, not +# here. +applications: + - name: ((app_name)) + docker: + image: ((image)) + username: ((docker_username)) + instances: 1 + memory: 256M + disk_quota: 512M + # A port check is a TCP connect. An http check would send a request, get 401 from + # Vector's basic auth, and mark the app crashed. + health-check-type: port + default-route: true diff --git a/infra/cf-log-bridge/vector.yaml b/infra/cf-log-bridge/vector.yaml new file mode 100644 index 000000000..16ffab52b --- /dev/null +++ b/infra/cf-log-bridge/vector.yaml @@ -0,0 +1,136 @@ +# CF syslog drain to STACKIT Observability: logs to Loki, container metrics to +# Prometheus remote-write. +# +# Env: BAERGPT_ENV, DRAIN_{USERNAME,PASSWORD}, +# STACKIT_OBSERVABILITY_{USERNAME,PASSWORD,LOGS_ENDPOINT,METRICS_ENDPOINT} +# +# Env vars below are substituted only because the Dockerfile passes +# --dangerously-allow-env-var-interpolation. That substitution runs over the whole file, +# comments included, so a reference written in a comment is resolved too and aborts +# startup if the name is unset. Escape with a double dollar to mention one: $${EXAMPLE}. + +sources: + drain: + type: http_server + address: 0.0.0.0:${PORT:-8080} + # CF authenticates with the userinfo from the drain URL. + auth: + strategy: basic + username: ${DRAIN_USERNAME:?drain username is required} + password: ${DRAIN_PASSWORD:?drain password is required} + decoding: + codec: syslog + +transforms: + # Metric frames carry a gauge/counter element and an empty message. Log frames carry + # neither. Both arrive over the same connection. + logs_only: + type: filter + inputs: [drain] + condition: '!exists(."gauge@47450") && !exists(."counter@47450")' + + metrics_only: + type: filter + inputs: [drain] + condition: 'exists(."gauge@47450") || exists(."counter@47450")' + + label: + type: remap + inputs: [logs_only] + source: | + tags = object(."tags@47450") ?? {} + + .source = "cf-${BAERGPT_ENV:?BAERGPT_ENV is required}" + .app_name = string(tags.app_name) ?? "unknown" + .space_name = string(tags.space_name) ?? "unknown" + .instance_id = string(tags.instance_id) ?? "0" + .level = string(.severity) ?? "info" + + # Vector owns the top-level source_type field, so CF's is renamed in the sink. + .cf_source_type = string(tags.source_type) ?? "unknown" + + # Not promoted to labels: per-container and per-cell, unbounded cardinality. + .process_instance_id = string(tags.process_instance_id) ?? "" + .cell_index = string(tags.index) ?? "" + + del(."tags@47450") + + metric_fields: + type: remap + inputs: [metrics_only] + source: | + tags = object(."tags@47450") ?? {} + gauge = object(."gauge@47450") ?? {} + counter = object(."counter@47450") ?? {} + + .metric_name = string(gauge.name) ?? string(counter.name) ?? "unknown" + + # Values live in the structured-data element as strings; log_to_metric does not + # coerce. to_float! drops the event on failure rather than falling back to 0, + # which would read as a healthy container and never alert. + raw = string(gauge.value) ?? string(counter.total) ?? "" + .metric_value = to_float!(raw) + + .source = "cf-${BAERGPT_ENV:?BAERGPT_ENV is required}" + .app_name = string(tags.app_name) ?? "unknown" + .space_name = string(tags.space_name) ?? "unknown" + .instance_id = string(tags.instance_id) ?? "0" + + to_metric: + type: log_to_metric + inputs: [metric_fields] + metrics: + # Gauges throughout, including the cumulative rx_bytes/tx_bytes. rate() works over + # those, but a container restart appears as a drop, not a counter reset. + - type: gauge + field: metric_value + name: "cf_{{ metric_name }}" + tags: + source: "{{ source }}" + app_name: "{{ app_name }}" + space_name: "{{ space_name }}" + instance_id: "{{ instance_id }}" + +sinks: + loki: + type: loki + inputs: [label] + # Base URL only. Vector appends /loki/api/v1/push. + endpoint: ${STACKIT_OBSERVABILITY_LOGS_ENDPOINT:?logs endpoint is required} + auth: + strategy: basic + user: ${STACKIT_OBSERVABILITY_USERNAME:?observability username is required} + password: ${STACKIT_OBSERVABILITY_PASSWORD:?observability password is required} + # Label values come wholly from received fields, which Vector rejects by default. + # Acceptable only while the drain endpoint is authenticated. + dangerously_allow_unconfined_template_resolution: true + labels: + source: "{{ source }}" + app_name: "{{ app_name }}" + space_name: "{{ space_name }}" + source_type: "{{ cf_source_type }}" + instance_id: "{{ instance_id }}" + level: "{{ level }}" + encoding: + codec: text + out_of_order_action: accept + request: + retry_attempts: 5 + buffer: + type: memory + max_events: 5000 + when_full: drop_newest + + metrics: + type: prometheus_remote_write + inputs: [to_metric] + # Used verbatim, including /api/v1/receive. + endpoint: ${STACKIT_OBSERVABILITY_METRICS_ENDPOINT:?metrics endpoint is required} + # STACKIT answers 405 to Vector's probe, so the check can never pass. Writes are + # unaffected, and the Loki healthcheck validates the same credentials. + healthcheck: + enabled: false + auth: + strategy: basic + user: ${STACKIT_OBSERVABILITY_USERNAME:?observability username is required} + password: ${STACKIT_OBSERVABILITY_PASSWORD:?observability password is required} diff --git a/infra/terraform/cloud-foundry/README.md b/infra/terraform/cloud-foundry/README.md index 44763ba9c..d67c16282 100644 --- a/infra/terraform/cloud-foundry/README.md +++ b/infra/terraform/cloud-foundry/README.md @@ -14,6 +14,7 @@ Provisions one CF org (`baergpt`) with `staging` and `prod` spaces in the | `spaces.tf` | spaces, space quotas, org/space roles | | `service-accounts.tf` | per-space CI service accounts + their keys | | `autoscaler.tf` | per-space App-Autoscaler service instance (`-autoscaler`) | +| `log-drain.tf` | per-space syslog drain pointing at that space's log bridge | | `outputs.tf` | api_url, CI credentials | ## Quotas @@ -93,6 +94,36 @@ space only. Read a login out and put it in the pipeline's secret store: $OP terraform output -json deployer_credentials | jq '.prod' ``` +## Log drain + +Each space gets a `stackit-drain` user-provided service whose URL points at that space's +log bridge (`infra/cf-log-bridge`), with `?drain-type=all` so container metrics arrive +alongside logs. Apps bind to it by naming it in their own manifest (see +`apps/backend/manifest.yml`), so a recreated app re-binds itself. + +The URL host is `-.`. A route belongs to one space +only, so the suffix is what keeps the environments apart. +`.github/workflows/cf-log-bridge-deploy.yml` builds the same name from the branch — change +one, change both. + +The service holds only a string; Cloud Controller never resolves it, so the drain can be +created before the bridge exists. Delivery obviously requires the bridge to be up. + +`var.drain_credentials` carries the bridge's basic-auth credentials per space, embedded +in the URL because CF's syslog agent has no other way to authenticate. Supply it from +the op env-file: + +```sh +TF_VAR_drain_credentials='{"staging":{"username":"...","password":"..."},"prod":{"username":"...","password":"..."}}' +``` + +**These must match `DRAIN_USERNAME` / `DRAIN_PASSWORD` in that space's bridge 1Password +item.** A mismatch produces 401s inside CF's syslog agent and telemetry stops with no +error surfaced anywhere. Rotate both together. + +> CI runs `terraform fmt` over this module but only validates `observability`, so run +> `$OP terraform validate` here yourself before opening a PR. + ## SSH access ### Cloud Foundry app containers — `cf ssh` diff --git a/infra/terraform/cloud-foundry/log-drain.tf b/infra/terraform/cloud-foundry/log-drain.tf new file mode 100644 index 000000000..29d357bf5 --- /dev/null +++ b/infra/terraform/cloud-foundry/log-drain.tf @@ -0,0 +1,31 @@ +# Per-space syslog drain pointing at that space's log bridge (infra/cf-log-bridge). +# Holds nothing but a URL; CF forwards a bound app's logs there. Binding is declared in +# each app's manifest so a redeployed app re-binds itself, see apps/backend/manifest.yml. +resource "cloudfoundry_service_instance" "log_drain" { + for_each = cloudfoundry_space.this + + name = "stackit-drain" + type = "user-provided" + space = each.value.id + + # Credentials sit in the URL's userinfo because that is CF's only way to authenticate + # to a drain. They must match DRAIN_USERNAME/DRAIN_PASSWORD on the bridge app, or + # delivery 401s with no error surfaced anywhere. + # + # drain-type=all adds container metrics alongside logs. The route is derived rather + # than looked up: Cloud Controller stores this string without resolving it, so the + # bridge need not exist yet. + # + # Host is . and a route belongs to one space, so the app name + # carries the space. The deploy workflow builds it the same way. + syslog_drain_url = format( + "https://%s:%s@%s-%s.%s/?drain-type=all", + var.drain_credentials[each.key].username, + var.drain_credentials[each.key].password, + var.log_bridge_app_name, + each.key, + var.apps_domain, + ) + + depends_on = [cloudfoundry_space_role.manager] +} diff --git a/infra/terraform/cloud-foundry/variables.tf b/infra/terraform/cloud-foundry/variables.tf index 77a5d8275..73b7ee12a 100644 --- a/infra/terraform/cloud-foundry/variables.tf +++ b/infra/terraform/cloud-foundry/variables.tf @@ -26,6 +26,40 @@ variable "operators" { description = "CF usernames granted organization_user + space_developer on both spaces (the operating/CI account). Each must already exist in UAA (logged in once)." } +variable "apps_domain" { + type = string + default = "apps.01.cf.eu01.stackit.cloud" + description = "Shared CF apps domain. App routes are .; used to build the log drain URL." +} + +variable "log_bridge_app_name" { + type = string + default = "baergpt-log-bridge" + description = "Base name of the Vector app receiving the drain; the space is appended (-staging/-prod). Must match APP_NAME in .github/workflows/cf-log-bridge-deploy.yml." +} + +# Must match DRAIN_USERNAME / DRAIN_PASSWORD in that space's bridge 1Password item; a +# mismatch stops delivery with no error anywhere. Supplied as TF_VAR_drain_credentials +# from the op env-file. +variable "drain_credentials" { + type = map(object({ + username = string + password = string + })) + sensitive = true + description = "Per-space drain basic-auth credentials, keyed by space name." + + validation { + condition = alltrue([for c in var.drain_credentials : length(c.password) >= 24]) + error_message = "Drain passwords must be at least 24 characters." + } + + validation { + condition = toset(keys(var.drain_credentials)) == toset(keys(var.spaces)) + error_message = "drain_credentials needs exactly one entry per space: ${join(", ", keys(var.spaces))}." + } +} + # Per-space settings. Keys must be exactly "staging" and "prod". variable "spaces" { type = map(object({