feat(k8s): production parity — workers, MinIO, pull secrets, CI render gate

.github/workflows/cd.yml

@@ -240,4 +240,14 @@ jobs:
               "*=${{ env.IMAGE_PREFIX }}/$(echo "$d" | sed 's/-service//;s/frontend/web/'):${TAG}" \
               -n spectra-lab || true
           done
+          # Workers share the service images (celery-worker/beat run analysis,
+          # pc-worker runs process-control) — pin them to the same tag or they
+          # drift onto :latest.
+          for d in celery-worker celery-beat; do
+            kubectl set image "deployment/$d" \
+              "*=${{ env.IMAGE_PREFIX }}/analysis:${TAG}" -n spectra-lab || true
+          done
+          kubectl set image deployment/pc-worker \
+            "*=${{ env.IMAGE_PREFIX }}/process-control:${TAG}" -n spectra-lab || true
           kubectl rollout status deployment/analysis-service -n spectra-lab --timeout=10m
+          kubectl rollout status deployment/celery-worker -n spectra-lab --timeout=10m

.github/workflows/ci.yml

@@ -460,6 +460,23 @@ jobs:
       - name: Run backup-script smoke test
         run: make backup-test
 
+  # ============================================================================
+  # k8s manifests must render — the CD deploy job runs `kubectl apply -k`
+  # on these exact overlays, and its skip-notice claims they are "validated
+  # in CI". This job makes that claim true: a broken kustomization or YAML
+  # typo fails the PR instead of the production deploy.
+  # ============================================================================
+  k8s-manifests:
+    name: k8s manifests render
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: kustomize build both overlays
+        run: |
+          kubectl kustomize k8s/base > /dev/null
+          kubectl kustomize k8s/overlays/production > /dev/null
+          echo "both overlays render"
+
   # ============================================================================
   # Status Check
   # ============================================================================
@@ -570,7 +587,7 @@ jobs:
   status:
     name: CI Status
     runs-on: ubuntu-latest
-    needs: [lint, frontend-typecheck, test, integration-test, security, docker-build, migration-test, web-build, db-guardrail, secrets-guardrail, backup-script-sync, e2e-smoke]
+    needs: [lint, frontend-typecheck, test, integration-test, security, docker-build, migration-test, web-build, db-guardrail, secrets-guardrail, backup-script-sync, e2e-smoke, k8s-manifests]
     if: always()
     steps:
       - name: Check CI status

k8s/base/analysis-deployment.yaml

@@ -117,6 +117,24 @@ spec:
             - name: PYTHONUNBUFFERED
               value: "1"
 
+            # Object storage (compose parity — exports + telemetry blobs)
+            - name: OBJECT_STORE_ENDPOINT
+              value: "http://minio:9000"
+            - name: OBJECT_STORE_BUCKET
+              value: "spectra-lab"
+            - name: OBJECT_STORE_REGION
+              value: "us-east-1"
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: minio-credentials
+                  key: root-user
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: minio-credentials
+                  key: root-password
+
           resources:
             requests:
               memory: "512Mi"

k8s/base/kustomization.yaml

@@ -8,6 +8,8 @@ resources:
   - serviceaccounts.yaml
   - analysis-deployment.yaml
   - services.yaml  # lims, process_control, web (frontend), redis — Phase 5.2
+  - workers.yaml  # celery-worker, celery-beat, pc-worker — prod parity
+  - minio.yaml  # object storage + bucket/lifecycle init — prod parity
   - postgresql.yaml
   - ingress.yaml
   - backup  # Session 6.3 — pg_dump + mc-mirror CronJobs
@@ -37,6 +39,11 @@ secretGenerator:
     literals:
       - redis-url=redis://redis:6379/0
 
+  - name: minio-credentials
+    literals:
+      - root-user=spectra-minio-admin
+      - root-password=CHANGE_ME_IN_PRODUCTION
+
 commonLabels:
   app.kubernetes.io/managed-by: kustomize
   app.kubernetes.io/part-of: spectra-lab

k8s/base/minio.yaml

@@ -0,0 +1,186 @@
+# MinIO object storage (k8s parity for the compose `minio` + `minio-init`
+# services, Session 6.1). analysis, lims, and celery-worker write CSV
+# exports + telemetry blobs here via OBJECT_STORE_* env.
+#
+# Single-replica StatefulSet: MinIO in standalone mode. For HA object
+# storage, swap OBJECT_STORE_ENDPOINT for a managed S3 bucket instead of
+# scaling this — the app speaks plain S3 either way.
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: minio
+  namespace: spectra-lab
+  labels:
+    app: minio
+    app.kubernetes.io/part-of: spectra-lab
+spec:
+  serviceName: minio
+  replicas: 1
+  selector:
+    matchLabels:
+      app: minio
+  template:
+    metadata:
+      labels:
+        app: minio
+    spec:
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        fsGroup: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: minio
+          # Tracks the compose stack (minio/minio:latest). Pin a RELEASE.*
+          # digest here once a cluster exists and upgrades become deliberate.
+          image: minio/minio:latest
+          args: ["server", "/data"]
+          ports:
+            - name: s3
+              containerPort: 9000
+          env:
+            - name: MINIO_ROOT_USER
+              valueFrom:
+                secretKeyRef:
+                  name: minio-credentials
+                  key: root-user
+            - name: MINIO_ROOT_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: minio-credentials
+                  key: root-password
+          resources:
+            requests:
+              memory: "256Mi"
+              cpu: "100m"
+            limits:
+              memory: "1Gi"
+              cpu: "500m"
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+          livenessProbe:
+            httpGet:
+              path: /minio/health/live
+              port: s3
+            initialDelaySeconds: 15
+            periodSeconds: 15
+          readinessProbe:
+            httpGet:
+              path: /minio/health/ready
+              port: s3
+            initialDelaySeconds: 5
+            periodSeconds: 5
+          volumeMounts:
+            - name: data
+              mountPath: /data
+  volumeClaimTemplates:
+    - metadata:
+        name: data
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        resources:
+          requests:
+            storage: 20Gi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio
+  namespace: spectra-lab
+  labels:
+    app: minio
+spec:
+  type: ClusterIP
+  ports:
+    - port: 9000
+      targetPort: s3
+      name: s3
+  selector:
+    app: minio
+---
+# Bucket + lifecycle bootstrap (k8s twin of the compose `minio-init`
+# one-shot). `mc ilm import` REPLACES the bucket's lifecycle config, so
+# re-running the Job is idempotent. The JSON below mirrors the canonical
+# policy in infra/minio/lifecycle.json — keep the two in sync.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: minio-init
+  namespace: spectra-lab
+  labels:
+    app: minio-init
+    app.kubernetes.io/part-of: spectra-lab
+spec:
+  backoffLimit: 6
+  template:
+    metadata:
+      labels:
+        app: minio-init
+    spec:
+      restartPolicy: OnFailure
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 1000
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: mc
+          image: minio/mc:latest
+          env:
+            - name: MINIO_ROOT_USER
+              valueFrom:
+                secretKeyRef:
+                  name: minio-credentials
+                  key: root-user
+            - name: MINIO_ROOT_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: minio-credentials
+                  key: root-password
+          command:
+            - /bin/sh
+            - -ec
+            - |
+              mc alias set spectra http://minio:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
+              mc mb --ignore-existing spectra/spectra-lab
+              mc ilm import spectra/spectra-lab <<'POLICY'
+              {
+                "Rules": [
+                  {
+                    "ID": "expire-telemetry-blobs-after-90d",
+                    "Status": "Enabled",
+                    "Filter": { "Prefix": "telemetry/" },
+                    "Expiration": { "Days": 90 }
+                  },
+                  {
+                    "ID": "expire-export-files-after-7d",
+                    "Status": "Enabled",
+                    "Filter": { "Prefix": "exports/" },
+                    "Expiration": { "Days": 7 }
+                  },
+                  {
+                    "ID": "abort-incomplete-multipart-after-1d",
+                    "Status": "Enabled",
+                    "Filter": { "Prefix": "" },
+                    "AbortIncompleteMultipartUpload": { "DaysAfterInitiation": 1 }
+                  }
+                ]
+              }
+              POLICY
+              echo "bucket + lifecycle ready"
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: false
+            capabilities:
+              drop: ["ALL"]
+          resources:
+            requests:
+              memory: "64Mi"
+              cpu: "50m"
+            limits:
+              memory: "128Mi"
+              cpu: "200m"

k8s/base/secrets/ghcr-pull-secret-template.yaml

@@ -0,0 +1,24 @@
+# TEMPLATE — do not apply as-is. The ghcr.io/alovladi007/spectra-lab/*
+# images are private, so every pod needs registry credentials to pull.
+#
+# Create the real secret once per cluster with a GitHub PAT that has
+# read:packages (a fine-grained token scoped to nothing else):
+#
+#   kubectl create secret docker-registry ghcr-pull-secret \
+#     --namespace spectra-lab \
+#     --docker-server=ghcr.io \
+#     --docker-username=<github-username> \
+#     --docker-password=<PAT-with-read:packages> \
+#     --docker-email=unused@example.com
+#
+# The ServiceAccounts in serviceaccounts.yaml reference this name; if the
+# secret is absent, Kubernetes ignores the reference and private pulls
+# fail with ImagePullBackOff — that is the signal you skipped this step.
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ghcr-pull-secret
+  namespace: spectra-lab
+type: kubernetes.io/dockerconfigjson
+data:
+  .dockerconfigjson: REPLACE_WITH_BASE64_DOCKERCONFIGJSON

k8s/base/secrets/minio-credentials-template.yaml

@@ -0,0 +1,16 @@
+# TEMPLATE — do not apply as-is. Real values come from the kustomize
+# secretGenerator (see ../kustomization.yaml) or an external secret
+# manager. The same pair is consumed two ways:
+#   - MinIO itself: MINIO_ROOT_USER / MINIO_ROOT_PASSWORD
+#   - analysis, lims, celery-worker: AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY
+# If you point OBJECT_STORE_ENDPOINT at managed S3 instead of in-cluster
+# MinIO, put the IAM key pair here and skip deploying minio.yaml.
+apiVersion: v1
+kind: Secret
+metadata:
+  name: minio-credentials
+  namespace: spectra-lab
+type: Opaque
+stringData:
+  root-user: spectra-minio-admin
+  root-password: CHANGE_ME_IN_PRODUCTION

k8s/base/serviceaccounts.yaml

@@ -3,7 +3,9 @@
 # referenced `analysis-sa` but it was never defined — the pod would fail
 # admission. Minimal SAs (no RBAC bindings) so token automounting is
 # explicit and each workload has a distinct identity for audit/network
-# policy later.
+# policy later. All SAs carry the ghcr-pull-secret reference (the images
+# are private on ghcr) — see secrets/ghcr-pull-secret-template.yaml for
+# how to provision the actual secret per cluster.
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -12,6 +14,8 @@ metadata:
   labels:
     app.kubernetes.io/part-of: spectra-lab
 automountServiceAccountToken: false
+imagePullSecrets:
+  - name: ghcr-pull-secret
 ---
 apiVersion: v1
 kind: ServiceAccount
@@ -21,6 +25,8 @@ metadata:
   labels:
     app.kubernetes.io/part-of: spectra-lab
 automountServiceAccountToken: false
+imagePullSecrets:
+  - name: ghcr-pull-secret
 ---
 apiVersion: v1
 kind: ServiceAccount
@@ -30,6 +36,8 @@ metadata:
   labels:
     app.kubernetes.io/part-of: spectra-lab
 automountServiceAccountToken: false
+imagePullSecrets:
+  - name: ghcr-pull-secret
 ---
 apiVersion: v1
 kind: ServiceAccount
@@ -39,3 +47,5 @@ metadata:
   labels:
     app.kubernetes.io/part-of: spectra-lab
 automountServiceAccountToken: false
+imagePullSecrets:
+  - name: ghcr-pull-secret

k8s/base/services.yaml

@@ -135,6 +135,24 @@ spec:
                   key: secret-key
             - name: ENVIRONMENT
               value: "production"
+
+            # Object storage (compose parity — exports + telemetry blobs)
+            - name: OBJECT_STORE_ENDPOINT
+              value: "http://minio:9000"
+            - name: OBJECT_STORE_BUCKET
+              value: "spectra-lab"
+            - name: OBJECT_STORE_REGION
+              value: "us-east-1"
+            - name: AWS_ACCESS_KEY_ID
+              valueFrom:
+                secretKeyRef:
+                  name: minio-credentials
+                  key: root-user
+            - name: AWS_SECRET_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: minio-credentials
+                  key: root-password
           resources:
             requests:
               memory: "256Mi"