From fbcab46b9f2fcaddbc12d8a35c168a7e27488b89 Mon Sep 17 00:00:00 2001 From: yahiaongh Date: Thu, 23 Jul 2026 20:49:54 +0100 Subject: [PATCH] feat: Kubernetes deployment of core pipeline via k3d + Kustomize (log-producer, Redpanda, ingestion-go, TimescaleDB, anomaly-rust), verified end-to-end with multi-node pod scheduling (Milestone 6) --- .gitignore | 3 +- README.md | 2 +- docs/MILESTONES.md | 8 +-- infra/k8s/README.md | 57 ++++++++++++++++++++ infra/k8s/base/anomaly-rust.yaml | 50 +++++++++++++++++ infra/k8s/base/ingestion-go.yaml | 50 +++++++++++++++++ infra/k8s/base/kustomization.yaml | 10 ++++ infra/k8s/base/log-producer.yaml | 26 +++++++++ infra/k8s/base/namespace.yaml | 4 ++ infra/k8s/base/redpanda.yaml | 55 +++++++++++++++++++ infra/k8s/base/secret.example.yaml | 19 +++++++ infra/k8s/base/timescaledb.yaml | 60 +++++++++++++++++++++ infra/k8s/overlays/local/kustomization.yaml | 4 ++ 13 files changed, 343 insertions(+), 5 deletions(-) create mode 100644 infra/k8s/README.md create mode 100644 infra/k8s/base/anomaly-rust.yaml create mode 100644 infra/k8s/base/ingestion-go.yaml create mode 100644 infra/k8s/base/kustomization.yaml create mode 100644 infra/k8s/base/log-producer.yaml create mode 100644 infra/k8s/base/namespace.yaml create mode 100644 infra/k8s/base/redpanda.yaml create mode 100644 infra/k8s/base/secret.example.yaml create mode 100644 infra/k8s/base/timescaledb.yaml create mode 100644 infra/k8s/overlays/local/kustomization.yaml diff --git a/.gitignore b/.gitignore index f058fdf..8401793 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,5 @@ infra/terraform/.terraform/ infra/terraform/*.tfstate* # IDE -.vscode/ \ No newline at end of file +.vscode/ +infra/k8s/base/secret.local.yaml diff --git a/README.md b/README.md index 4429b96..11e7de0 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ flowchart LR | `dashboard-nextjs` | TypeScript | Real-time anomaly feed, service health, LLM query terminal | ✅ Running | | Prometheus + Grafana | — | Metrics collection and dashboards for all instrumented services | ✅ Running | -See [docs/MILESTONES.md](docs/MILESTONES.md) for the full roadmap and what's shipped so far. +See [docs/MILESTONES.md](docs/MILESTONES.md) for the full roadmap and what's shipped so far. A Kubernetes deployment of the core pipeline (via k3d) is also available — see [infra/k8s/README.md](infra/k8s/README.md). ## Quickstart diff --git a/docs/MILESTONES.md b/docs/MILESTONES.md index 43eba07..bcb3372 100644 --- a/docs/MILESTONES.md +++ b/docs/MILESTONES.md @@ -47,9 +47,11 @@ Status and scope for each phase of SentinelOps. Dates are approximate; this is a - Grafana with provisioned datasource and dashboard (committed JSON, not clicked together — reproducible from a fresh `docker compose up`) - 9-panel dashboard covering ingestion throughput/errors, batch performance, anomaly detection rate and severity breakdown, and LLM query latency/failure rate -## 🚧 Milestone 6 — Kubernetes deployment (next) -- Manifests for all services -- Local cluster deployment (k3d/kind) as a production-shaped deploy path +## ✅ Milestone 6 — Kubernetes deployment +- k3d local cluster (1 server + 1 agent node), kubectl's built-in Kustomize for manifest organization +- Core pipeline deployed and verified end-to-end inside the cluster: log-producer → Redpanda → ingestion-go → TimescaleDB → anomaly-rust, with real multi-node pod scheduling +- Kubernetes Secrets for credentials (never committed with real values) +- Known, documented scope boundary: llm-query-python and dashboard-nextjs are not yet deployed to K8s, since both depend on a host-native Ollama instance reachable via `host.docker.internal` in Compose — that networking path doesn't translate directly into a k3d cluster. Noted explicitly rather than silently skipped; see infra/k8s/README.md. ## Known gaps (tracked, not blocking) - No automated tests for the Next.js dashboard (API routes or components) — only manual/visual verification so far diff --git a/infra/k8s/README.md b/infra/k8s/README.md new file mode 100644 index 0000000..d9bb6ab --- /dev/null +++ b/infra/k8s/README.md @@ -0,0 +1,57 @@ +# Kubernetes Deployment + +Local Kubernetes deployment of SentinelOps's core pipeline (log producer → Redpanda → Go ingestion → TimescaleDB → Rust anomaly engine), using [k3d](https://k3d.io/) and `kubectl`'s built-in Kustomize support. + +## What's deployed here + +- `timescaledb`, `redpanda` — infrastructure +- `log-producer`, `ingestion-go`, `anomaly-rust` — the three services that make up the core detection pipeline + +## Known limitation + +`llm-query-python` and `dashboard-nextjs` are **not yet deployed to Kubernetes**. Both depend on Ollama, which runs natively on the host machine in the Docker Compose setup (via `host.docker.internal` + a systemd `OLLAMA_HOST=0.0.0.0` override — see the root README). That networking path doesn't translate directly into a k3d cluster, which is network-isolated from the host differently than a single Docker container is. Deploying these two services would need either： +- a proper Ollama Kubernetes deployment (running Ollama itself as a pod, which needs significant CPU/RAM and ideally GPU passthrough), or +- a documented cluster-to-host networking bridge specific to k3d. + +This is left as a deliberate scope boundary for this project, not an oversight — noted here rather than silently skipped. + +## Quickstart + +```bash +# 1. Create the cluster +k3d cluster create sentinelops --agents 1 --wait + +# 2. Create the namespace and secret +kubectl apply -f base/namespace.yaml +kubectl create secret generic sentinelops-secrets \ + --namespace sentinelops \ + --from-literal=POSTGRES_USER=sentinelops \ + --from-literal=POSTGRES_PASSWORD=devpassword \ + --from-literal=POSTGRES_DB=sentinelops \ + --from-literal=DATABASE_URL=postgres://sentinelops:devpassword@timescaledb:5432/sentinelops + +# 3. Deploy infrastructure and apply the schema +kubectl apply -f base/timescaledb.yaml +kubectl apply -f base/redpanda.yaml +kubectl -n sentinelops exec -i deploy/timescaledb -- psql -U sentinelops -d sentinelops < ../sql/001_init_hypertable.sql +kubectl -n sentinelops exec -i deploy/timescaledb -- psql -U sentinelops -d sentinelops < ../sql/002_anomalies_table.sql + +# 4. Build and import service images (no registry needed for local dev) +docker build -t sentinelops/log-producer:local ../../services/log-producer +docker build -t sentinelops/ingestion-go:local ../../services/ingestion-go +docker build -t sentinelops/anomaly-rust:local ../../services/anomaly-rust +k3d image import sentinelops/log-producer:local sentinelops/ingestion-go:local sentinelops/anomaly-rust:local -c sentinelops + +# 5. Deploy the services +kubectl apply -f base/log-producer.yaml -f base/ingestion-go.yaml -f base/anomaly-rust.yaml + +# 6. Verify +kubectl -n sentinelops get pods +kubectl -n sentinelops exec deploy/timescaledb -- psql -U sentinelops -d sentinelops -c "SELECT count(*) FROM log_events;" +``` + +## Teardown + +```bash +k3d cluster delete sentinelops +``` \ No newline at end of file diff --git a/infra/k8s/base/anomaly-rust.yaml b/infra/k8s/base/anomaly-rust.yaml new file mode 100644 index 0000000..70f1d7d --- /dev/null +++ b/infra/k8s/base/anomaly-rust.yaml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: anomaly-rust + namespace: sentinelops +spec: + replicas: 1 + selector: + matchLabels: { app: anomaly-rust } + template: + metadata: + labels: { app: anomaly-rust } + spec: + containers: + - name: anomaly-rust + image: sentinelops/anomaly-rust:local + imagePullPolicy: Never + ports: + - containerPort: 9200 + envFrom: + - secretRef: + name: sentinelops-secrets + env: + - name: POLL_INTERVAL_SECS + value: "15" + - name: BASELINE_WINDOW_MINUTES + value: "60" + - name: Z_SCORE_WARNING + value: "2.5" + - name: Z_SCORE_CRITICAL + value: "4.0" + - name: EWMA_ALPHA + value: "0.3" + - name: METRICS_ADDR + value: "0.0.0.0:9200" + readinessProbe: + httpGet: { path: /metrics, port: 9200 } + initialDelaySeconds: 5 + periodSeconds: 10 +--- +apiVersion: v1 +kind: Service +metadata: + name: anomaly-rust + namespace: sentinelops +spec: + selector: { app: anomaly-rust } + ports: + - port: 9200 + targetPort: 9200 \ No newline at end of file diff --git a/infra/k8s/base/ingestion-go.yaml b/infra/k8s/base/ingestion-go.yaml new file mode 100644 index 0000000..38b16b8 --- /dev/null +++ b/infra/k8s/base/ingestion-go.yaml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ingestion-go + namespace: sentinelops +spec: + replicas: 1 + selector: + matchLabels: { app: ingestion-go } + template: + metadata: + labels: { app: ingestion-go } + spec: + containers: + - name: ingestion-go + image: sentinelops/ingestion-go:local + imagePullPolicy: Never + ports: + - containerPort: 9100 + envFrom: + - secretRef: + name: sentinelops-secrets + env: + - name: KAFKA_BROKERS + value: "redpanda:9092" + - name: KAFKA_TOPIC + value: "service-logs" + - name: KAFKA_GROUP_ID + value: "sentinelops-ingestor" + - name: BATCH_SIZE + value: "100" + - name: BATCH_INTERVAL_MS + value: "500" + - name: METRICS_ADDR + value: ":9100" + readinessProbe: + httpGet: { path: /healthz, port: 9100 } + initialDelaySeconds: 5 + periodSeconds: 10 +--- +apiVersion: v1 +kind: Service +metadata: + name: ingestion-go + namespace: sentinelops +spec: + selector: { app: ingestion-go } + ports: + - port: 9100 + targetPort: 9100 \ No newline at end of file diff --git a/infra/k8s/base/kustomization.yaml b/infra/k8s/base/kustomization.yaml new file mode 100644 index 0000000..2333fdf --- /dev/null +++ b/infra/k8s/base/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: sentinelops +resources: + - namespace.yaml + - timescaledb.yaml + - redpanda.yaml + - log-producer.yaml + - ingestion-go.yaml + - anomaly-rust.yaml \ No newline at end of file diff --git a/infra/k8s/base/log-producer.yaml b/infra/k8s/base/log-producer.yaml new file mode 100644 index 0000000..7d342e5 --- /dev/null +++ b/infra/k8s/base/log-producer.yaml @@ -0,0 +1,26 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: log-producer + namespace: sentinelops +spec: + replicas: 1 + selector: + matchLabels: { app: log-producer } + template: + metadata: + labels: { app: log-producer } + spec: + containers: + - name: log-producer + image: sentinelops/log-producer:local + imagePullPolicy: Never + env: + - name: KAFKA_BROKERS + value: "redpanda:9092" + - name: LOG_TOPIC + value: "service-logs" + - name: EMIT_INTERVAL_SECONDS + value: "0.2" + - name: ANOMALY_PROBABILITY + value: "0.0015" \ No newline at end of file diff --git a/infra/k8s/base/namespace.yaml b/infra/k8s/base/namespace.yaml new file mode 100644 index 0000000..ed441a7 --- /dev/null +++ b/infra/k8s/base/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: sentinelops \ No newline at end of file diff --git a/infra/k8s/base/redpanda.yaml b/infra/k8s/base/redpanda.yaml new file mode 100644 index 0000000..567e220 --- /dev/null +++ b/infra/k8s/base/redpanda.yaml @@ -0,0 +1,55 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redpanda + namespace: sentinelops +spec: + replicas: 1 + selector: + matchLabels: { app: redpanda } + template: + metadata: + labels: { app: redpanda } + spec: + containers: + - name: redpanda + image: docker.redpanda.com/redpandadata/redpanda:v24.2.7 + args: + - redpanda + - start + - --smp=1 + - --memory=512M + - --overprovisioned + - --node-id=0 + - --kafka-addr=internal://0.0.0.0:9092 + - --advertise-kafka-addr=internal://redpanda:9092 + - --pandaproxy-addr=0.0.0.0:8082 + - --advertise-pandaproxy-addr=redpanda:8082 + ports: + - containerPort: 9092 + - containerPort: 8082 + readinessProbe: + exec: + command: ["rpk", "cluster", "health"] + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + exec: + command: ["rpk", "cluster", "health"] + initialDelaySeconds: 20 + periodSeconds: 20 +--- +apiVersion: v1 +kind: Service +metadata: + name: redpanda + namespace: sentinelops +spec: + selector: { app: redpanda } + ports: + - name: kafka + port: 9092 + targetPort: 9092 + - name: pandaproxy + port: 8082 + targetPort: 8082 \ No newline at end of file diff --git a/infra/k8s/base/secret.example.yaml b/infra/k8s/base/secret.example.yaml new file mode 100644 index 0000000..a49643c --- /dev/null +++ b/infra/k8s/base/secret.example.yaml @@ -0,0 +1,19 @@ +# Template only — copy to secret.local.yaml (gitignored) and fill in real +# values before applying, or generate directly with kubectl: +# kubectl create secret generic sentinelops-secrets \ +# --namespace sentinelops \ +# --from-literal=POSTGRES_USER=sentinelops \ +# --from-literal=POSTGRES_PASSWORD=devpassword \ +# --from-literal=POSTGRES_DB=sentinelops \ +# --from-literal=DATABASE_URL=postgres://sentinelops:devpassword@timescaledb:5432/sentinelops +apiVersion: v1 +kind: Secret +metadata: + name: sentinelops-secrets + namespace: sentinelops +type: Opaque +stringData: + POSTGRES_USER: sentinelops + POSTGRES_PASSWORD: CHANGE_ME + POSTGRES_DB: sentinelops + DATABASE_URL: postgres://sentinelops:CHANGE_ME@timescaledb:5432/sentinelops \ No newline at end of file diff --git a/infra/k8s/base/timescaledb.yaml b/infra/k8s/base/timescaledb.yaml new file mode 100644 index 0000000..eb508ce --- /dev/null +++ b/infra/k8s/base/timescaledb.yaml @@ -0,0 +1,60 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: timescaledb-data + namespace: sentinelops +spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 2Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: timescaledb + namespace: sentinelops +spec: + replicas: 1 + selector: + matchLabels: { app: timescaledb } + template: + metadata: + labels: { app: timescaledb } + spec: + containers: + - name: timescaledb + image: timescale/timescaledb:2.15.3-pg16 + ports: + - containerPort: 5432 + envFrom: + - secretRef: + name: sentinelops-secrets + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + readinessProbe: + exec: + command: ["pg_isready", "-U", "sentinelops"] + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + exec: + command: ["pg_isready", "-U", "sentinelops"] + initialDelaySeconds: 15 + periodSeconds: 20 + volumes: + - name: data + persistentVolumeClaim: + claimName: timescaledb-data +--- +apiVersion: v1 +kind: Service +metadata: + name: timescaledb + namespace: sentinelops +spec: + selector: { app: timescaledb } + ports: + - port: 5432 + targetPort: 5432 \ No newline at end of file diff --git a/infra/k8s/overlays/local/kustomization.yaml b/infra/k8s/overlays/local/kustomization.yaml new file mode 100644 index 0000000..a227ac4 --- /dev/null +++ b/infra/k8s/overlays/local/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../../base \ No newline at end of file