diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 11088bcf..eaa71551 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -240,4 +240,14 @@ jobs: "*=${{ env.IMAGE_PREFIX }}/$(echo "$d" | sed 's/-service//;s/frontend/web/'):${TAG}" \ -n spectra-lab || true done + # Workers share the service images (celery-worker/beat run analysis, + # pc-worker runs process-control) — pin them to the same tag or they + # drift onto :latest. + for d in celery-worker celery-beat; do + kubectl set image "deployment/$d" \ + "*=${{ env.IMAGE_PREFIX }}/analysis:${TAG}" -n spectra-lab || true + done + kubectl set image deployment/pc-worker \ + "*=${{ env.IMAGE_PREFIX }}/process-control:${TAG}" -n spectra-lab || true kubectl rollout status deployment/analysis-service -n spectra-lab --timeout=10m + kubectl rollout status deployment/celery-worker -n spectra-lab --timeout=10m diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3e1e098..cc30dee8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -460,6 +460,23 @@ jobs: - name: Run backup-script smoke test run: make backup-test + # ============================================================================ + # k8s manifests must render — the CD deploy job runs `kubectl apply -k` + # on these exact overlays, and its skip-notice claims they are "validated + # in CI". This job makes that claim true: a broken kustomization or YAML + # typo fails the PR instead of the production deploy. + # ============================================================================ + k8s-manifests: + name: k8s manifests render + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: kustomize build both overlays + run: | + kubectl kustomize k8s/base > /dev/null + kubectl kustomize k8s/overlays/production > /dev/null + echo "both overlays render" + # ============================================================================ # Status Check # ============================================================================ @@ -570,7 +587,7 @@ jobs: status: name: CI Status runs-on: ubuntu-latest - needs: [lint, frontend-typecheck, test, integration-test, security, docker-build, migration-test, web-build, db-guardrail, secrets-guardrail, backup-script-sync, e2e-smoke] + needs: [lint, frontend-typecheck, test, integration-test, security, docker-build, migration-test, web-build, db-guardrail, secrets-guardrail, backup-script-sync, e2e-smoke, k8s-manifests] if: always() steps: - name: Check CI status diff --git a/k8s/base/analysis-deployment.yaml b/k8s/base/analysis-deployment.yaml index e44b153f..f3431f7b 100644 --- a/k8s/base/analysis-deployment.yaml +++ b/k8s/base/analysis-deployment.yaml @@ -117,6 +117,24 @@ spec: - name: PYTHONUNBUFFERED value: "1" + # Object storage (compose parity — exports + telemetry blobs) + - name: OBJECT_STORE_ENDPOINT + value: "http://minio:9000" + - name: OBJECT_STORE_BUCKET + value: "spectra-lab" + - name: OBJECT_STORE_REGION + value: "us-east-1" + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: minio-credentials + key: root-user + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: root-password + resources: requests: memory: "512Mi" diff --git a/k8s/base/kustomization.yaml b/k8s/base/kustomization.yaml index 1156ce12..1cfe4823 100644 --- a/k8s/base/kustomization.yaml +++ b/k8s/base/kustomization.yaml @@ -8,6 +8,8 @@ resources: - serviceaccounts.yaml - analysis-deployment.yaml - services.yaml # lims, process_control, web (frontend), redis — Phase 5.2 + - workers.yaml # celery-worker, celery-beat, pc-worker — prod parity + - minio.yaml # object storage + bucket/lifecycle init — prod parity - postgresql.yaml - ingress.yaml - backup # Session 6.3 — pg_dump + mc-mirror CronJobs @@ -37,6 +39,11 @@ secretGenerator: literals: - redis-url=redis://redis:6379/0 + - name: minio-credentials + literals: + - root-user=spectra-minio-admin + - root-password=CHANGE_ME_IN_PRODUCTION + commonLabels: app.kubernetes.io/managed-by: kustomize app.kubernetes.io/part-of: spectra-lab diff --git a/k8s/base/minio.yaml b/k8s/base/minio.yaml new file mode 100644 index 00000000..a3851236 --- /dev/null +++ b/k8s/base/minio.yaml @@ -0,0 +1,186 @@ +# MinIO object storage (k8s parity for the compose `minio` + `minio-init` +# services, Session 6.1). analysis, lims, and celery-worker write CSV +# exports + telemetry blobs here via OBJECT_STORE_* env. +# +# Single-replica StatefulSet: MinIO in standalone mode. For HA object +# storage, swap OBJECT_STORE_ENDPOINT for a managed S3 bucket instead of +# scaling this — the app speaks plain S3 either way. +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: minio + namespace: spectra-lab + labels: + app: minio + app.kubernetes.io/part-of: spectra-lab +spec: + serviceName: minio + replicas: 1 + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + fsGroup: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: minio + # Tracks the compose stack (minio/minio:latest). Pin a RELEASE.* + # digest here once a cluster exists and upgrades become deliberate. + image: minio/minio:latest + args: ["server", "/data"] + ports: + - name: s3 + containerPort: 9000 + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: minio-credentials + key: root-user + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: minio-credentials + key: root-password + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "1Gi" + cpu: "500m" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + livenessProbe: + httpGet: + path: /minio/health/live + port: s3 + initialDelaySeconds: 15 + periodSeconds: 15 + readinessProbe: + httpGet: + path: /minio/health/ready + port: s3 + initialDelaySeconds: 5 + periodSeconds: 5 + volumeMounts: + - name: data + mountPath: /data + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 20Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: spectra-lab + labels: + app: minio +spec: + type: ClusterIP + ports: + - port: 9000 + targetPort: s3 + name: s3 + selector: + app: minio +--- +# Bucket + lifecycle bootstrap (k8s twin of the compose `minio-init` +# one-shot). `mc ilm import` REPLACES the bucket's lifecycle config, so +# re-running the Job is idempotent. The JSON below mirrors the canonical +# policy in infra/minio/lifecycle.json — keep the two in sync. +apiVersion: batch/v1 +kind: Job +metadata: + name: minio-init + namespace: spectra-lab + labels: + app: minio-init + app.kubernetes.io/part-of: spectra-lab +spec: + backoffLimit: 6 + template: + metadata: + labels: + app: minio-init + spec: + restartPolicy: OnFailure + securityContext: + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: mc + image: minio/mc:latest + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: minio-credentials + key: root-user + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: minio-credentials + key: root-password + command: + - /bin/sh + - -ec + - | + mc alias set spectra http://minio:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" + mc mb --ignore-existing spectra/spectra-lab + mc ilm import spectra/spectra-lab <<'POLICY' + { + "Rules": [ + { + "ID": "expire-telemetry-blobs-after-90d", + "Status": "Enabled", + "Filter": { "Prefix": "telemetry/" }, + "Expiration": { "Days": 90 } + }, + { + "ID": "expire-export-files-after-7d", + "Status": "Enabled", + "Filter": { "Prefix": "exports/" }, + "Expiration": { "Days": 7 } + }, + { + "ID": "abort-incomplete-multipart-after-1d", + "Status": "Enabled", + "Filter": { "Prefix": "" }, + "AbortIncompleteMultipartUpload": { "DaysAfterInitiation": 1 } + } + ] + } + POLICY + echo "bucket + lifecycle ready" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + capabilities: + drop: ["ALL"] + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "128Mi" + cpu: "200m" diff --git a/k8s/base/secrets/ghcr-pull-secret-template.yaml b/k8s/base/secrets/ghcr-pull-secret-template.yaml new file mode 100644 index 00000000..f213ae53 --- /dev/null +++ b/k8s/base/secrets/ghcr-pull-secret-template.yaml @@ -0,0 +1,24 @@ +# TEMPLATE — do not apply as-is. The ghcr.io/alovladi007/spectra-lab/* +# images are private, so every pod needs registry credentials to pull. +# +# Create the real secret once per cluster with a GitHub PAT that has +# read:packages (a fine-grained token scoped to nothing else): +# +# kubectl create secret docker-registry ghcr-pull-secret \ +# --namespace spectra-lab \ +# --docker-server=ghcr.io \ +# --docker-username= \ +# --docker-password= \ +# --docker-email=unused@example.com +# +# The ServiceAccounts in serviceaccounts.yaml reference this name; if the +# secret is absent, Kubernetes ignores the reference and private pulls +# fail with ImagePullBackOff — that is the signal you skipped this step. +apiVersion: v1 +kind: Secret +metadata: + name: ghcr-pull-secret + namespace: spectra-lab +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: REPLACE_WITH_BASE64_DOCKERCONFIGJSON diff --git a/k8s/base/secrets/minio-credentials-template.yaml b/k8s/base/secrets/minio-credentials-template.yaml new file mode 100644 index 00000000..9f559c0c --- /dev/null +++ b/k8s/base/secrets/minio-credentials-template.yaml @@ -0,0 +1,16 @@ +# TEMPLATE — do not apply as-is. Real values come from the kustomize +# secretGenerator (see ../kustomization.yaml) or an external secret +# manager. The same pair is consumed two ways: +# - MinIO itself: MINIO_ROOT_USER / MINIO_ROOT_PASSWORD +# - analysis, lims, celery-worker: AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY +# If you point OBJECT_STORE_ENDPOINT at managed S3 instead of in-cluster +# MinIO, put the IAM key pair here and skip deploying minio.yaml. +apiVersion: v1 +kind: Secret +metadata: + name: minio-credentials + namespace: spectra-lab +type: Opaque +stringData: + root-user: spectra-minio-admin + root-password: CHANGE_ME_IN_PRODUCTION diff --git a/k8s/base/serviceaccounts.yaml b/k8s/base/serviceaccounts.yaml index 5f1a9b04..89614093 100644 --- a/k8s/base/serviceaccounts.yaml +++ b/k8s/base/serviceaccounts.yaml @@ -3,7 +3,9 @@ # referenced `analysis-sa` but it was never defined — the pod would fail # admission. Minimal SAs (no RBAC bindings) so token automounting is # explicit and each workload has a distinct identity for audit/network -# policy later. +# policy later. All SAs carry the ghcr-pull-secret reference (the images +# are private on ghcr) — see secrets/ghcr-pull-secret-template.yaml for +# how to provision the actual secret per cluster. apiVersion: v1 kind: ServiceAccount metadata: @@ -12,6 +14,8 @@ metadata: labels: app.kubernetes.io/part-of: spectra-lab automountServiceAccountToken: false +imagePullSecrets: + - name: ghcr-pull-secret --- apiVersion: v1 kind: ServiceAccount @@ -21,6 +25,8 @@ metadata: labels: app.kubernetes.io/part-of: spectra-lab automountServiceAccountToken: false +imagePullSecrets: + - name: ghcr-pull-secret --- apiVersion: v1 kind: ServiceAccount @@ -30,6 +36,8 @@ metadata: labels: app.kubernetes.io/part-of: spectra-lab automountServiceAccountToken: false +imagePullSecrets: + - name: ghcr-pull-secret --- apiVersion: v1 kind: ServiceAccount @@ -39,3 +47,5 @@ metadata: labels: app.kubernetes.io/part-of: spectra-lab automountServiceAccountToken: false +imagePullSecrets: + - name: ghcr-pull-secret diff --git a/k8s/base/services.yaml b/k8s/base/services.yaml index a79e20a8..6b558338 100644 --- a/k8s/base/services.yaml +++ b/k8s/base/services.yaml @@ -135,6 +135,24 @@ spec: key: secret-key - name: ENVIRONMENT value: "production" + + # Object storage (compose parity — exports + telemetry blobs) + - name: OBJECT_STORE_ENDPOINT + value: "http://minio:9000" + - name: OBJECT_STORE_BUCKET + value: "spectra-lab" + - name: OBJECT_STORE_REGION + value: "us-east-1" + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: minio-credentials + key: root-user + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: root-password resources: requests: memory: "256Mi" diff --git a/k8s/base/workers.yaml b/k8s/base/workers.yaml new file mode 100644 index 00000000..9c8339e9 --- /dev/null +++ b/k8s/base/workers.yaml @@ -0,0 +1,286 @@ +# Background workers (k8s parity for the compose `celery-worker`, +# `celery-beat`, and `pc-worker` services). Without these, async analysis +# runs (CVD/diffusion/oxidation), scheduled tasks, and process-control job +# execution simply never happen in a cluster — the APIs would accept jobs +# that no one consumes. +# +# Identity: the analysis-side workers run the analysis image and reuse +# analysis-sa; pc-worker runs the process-control image and reuses +# process-control-sa. Broker DB 2 matches compose (DB 0 is cache). +--- +# ============================================================================ +# Celery worker — analysis run queues +# ============================================================================ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: celery-worker + namespace: spectra-lab + labels: + app: celery-worker + app.kubernetes.io/part-of: spectra-lab +spec: + replicas: 2 + selector: + matchLabels: + app: celery-worker + template: + metadata: + labels: + app: celery-worker + version: v1 + spec: + serviceAccountName: analysis-sa + securityContext: + runAsNonRoot: true + runAsUser: 10001 + fsGroup: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: celery-worker + image: ghcr.io/spectra-lab/analysis:latest + imagePullPolicy: Always + command: + - celery + - -A + - app.tasks + - worker + - --loglevel=info + - --concurrency=4 + - --queues=cvd_runs,diffusion_runs,oxidation_runs,monitoring,default + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: database-credentials + key: postgres-url + - name: REDIS_URL + valueFrom: + secretKeyRef: + name: redis-credentials + key: redis-url + - name: CELERY_BROKER_URL + value: "redis://redis:6379/2" + - name: CELERY_RESULT_BACKEND + value: "redis://redis:6379/2" + - name: OBJECT_STORE_ENDPOINT + value: "http://minio:9000" + - name: OBJECT_STORE_BUCKET + value: "spectra-lab" + - name: OBJECT_STORE_REGION + value: "us-east-1" + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: minio-credentials + key: root-user + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: root-password + # Same default as the API: retention sweeps report, never delete, + # until explicitly flipped. + - name: RETENTION_DRY_RUN + value: "true" + - name: ENVIRONMENT + value: "production" + resources: + requests: + memory: "512Mi" + cpu: "250m" + limits: + memory: "2Gi" + cpu: "1" + livenessProbe: + exec: + command: + - /bin/sh + - -c + - celery -A app.tasks inspect ping -d "celery@$HOSTNAME" + initialDelaySeconds: 30 + periodSeconds: 60 + timeoutSeconds: 15 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} +--- +# ============================================================================ +# Celery beat — schedule ticker (exactly one replica; two beats would +# double-fire every periodic task) +# ============================================================================ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: celery-beat + namespace: spectra-lab + labels: + app: celery-beat + app.kubernetes.io/part-of: spectra-lab +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: celery-beat + template: + metadata: + labels: + app: celery-beat + version: v1 + spec: + serviceAccountName: analysis-sa + securityContext: + runAsNonRoot: true + runAsUser: 10001 + fsGroup: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: celery-beat + image: ghcr.io/spectra-lab/analysis:latest + imagePullPolicy: Always + # --schedule under /tmp: the root filesystem is read-only and beat + # persists its schedule DB next to CWD by default. + command: + - celery + - -A + - app.tasks + - beat + - --loglevel=info + - --schedule=/tmp/celerybeat-schedule + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: database-credentials + key: postgres-url + - name: REDIS_URL + valueFrom: + secretKeyRef: + name: redis-credentials + key: redis-url + - name: CELERY_BROKER_URL + value: "redis://redis:6379/2" + - name: CELERY_RESULT_BACKEND + value: "redis://redis:6379/2" + - name: ENVIRONMENT + value: "production" + resources: + requests: + memory: "128Mi" + cpu: "50m" + limits: + memory: "256Mi" + cpu: "250m" + # No probes: beat has no ping endpoint and the scheduler loop either + # runs or the process exits (restartPolicy covers the latter). + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} +--- +# ============================================================================ +# Process-control worker — ion/rtp job execution +# ============================================================================ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pc-worker + namespace: spectra-lab + labels: + app: pc-worker + app.kubernetes.io/part-of: spectra-lab +spec: + replicas: 2 + selector: + matchLabels: + app: pc-worker + template: + metadata: + labels: + app: pc-worker + version: v1 + spec: + serviceAccountName: process-control-sa + securityContext: + runAsNonRoot: true + runAsUser: 10001 + fsGroup: 10001 + seccompProfile: + type: RuntimeDefault + containers: + - name: pc-worker + image: ghcr.io/spectra-lab/process-control:latest + imagePullPolicy: Always + command: + - celery + - -A + - app.celery_app + - worker + - --loglevel=info + - --concurrency=2 + - --queues=ion,rtp,default + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: database-credentials + key: postgres-url + # Compose parity: pc-worker's REDIS_URL IS the broker DB (2), + # unlike the API services which use DB 0. + - name: REDIS_URL + value: "redis://redis:6379/2" + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: jwt-secret + key: secret-key + - name: SIMULATION_MODE + value: "true" + - name: ENVIRONMENT + value: "production" + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "1Gi" + cpu: "500m" + livenessProbe: + exec: + command: + - /bin/sh + - -c + - celery -A app.celery_app inspect ping -d "celery@$HOSTNAME" + initialDelaySeconds: 30 + periodSeconds: 60 + timeoutSeconds: 15 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {}