diff --git a/.env.example b/.env.example index dd1ec2f..28eda4c 100644 --- a/.env.example +++ b/.env.example @@ -40,6 +40,22 @@ SENTRY_ENVIRONMENT=development SENTRY_RELEASE=0.1.0 SENTRY_TRACES_SAMPLE_RATE=0.1 +# Observability — OpenTelemetry tracing + Prometheus metrics +# Master switch for distributed tracing. Set to "false" to skip starting the +# OpenTelemetry SDK entirely (metrics remain available regardless). +TRACING_ENABLED=true +# Head-based trace sampling ratio in [0,1]. 1 = sample every trace (dev), +# lower it in production to bound overhead, e.g. 0.1 = 10% of traces. +OTEL_TRACES_SAMPLER_RATIO=1.0 +# OTLP HTTP endpoint traces are exported to (Jaeger/Tempo/OTel Collector). +OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318/v1/traces +# Optional legacy Jaeger Thrift endpoint (only needed for pre-OTLP Jaeger). +# JAEGER_ENDPOINT=http://localhost:14268/api/traces +# Optional shared token protecting the Prometheus /metrics endpoint. When set, +# scrapers must send "Authorization: Bearer " or "?token=". +# Leave blank on a trusted/private network. Strongly recommended in production. +METRICS_AUTH_TOKEN= + # Email Service (Ethereal for testing, configure SMTP for production) SMTP_HOST=smtp.ethereal.email SMTP_PORT=587 diff --git a/.env.production.example b/.env.production.example index 16ff5bb..439956c 100644 --- a/.env.production.example +++ b/.env.production.example @@ -41,6 +41,17 @@ SENTRY_ENVIRONMENT=production SENTRY_RELEASE=1.0.0 SENTRY_TRACES_SAMPLE_RATE=0.05 +# Observability — OpenTelemetry tracing + Prometheus metrics +# Toggle distributed tracing on/off. Disable to remove all tracing overhead. +TRACING_ENABLED=true +# Fraction of traces to sample (0.0–1.0). Keep low in production to bound overhead. +OTEL_TRACES_SAMPLER_RATIO=0.1 +# OTLP/HTTP traces endpoint (Jaeger 1.41+, Tempo, or an OTel collector). +OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:4318/v1/traces +# REQUIRED in production: shared bearer token guarding /observability/metrics. +# Prometheus must send it via `Authorization: Bearer `. +METRICS_AUTH_TOKEN=CHANGE_ME_TO_A_LONG_RANDOM_SECRET + # Email Service - Use production SMTP (SendGrid, AWS SES, etc.) SMTP_HOST=smtp.sendgrid.net SMTP_PORT=587 diff --git a/docker-compose.yml b/docker-compose.yml index bb197c1..c70dfcc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,8 +60,11 @@ services: ports: - "9090:9090" volumes: - - ./prometheus.yml:/etc/prometheus/prometheus.yml + - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - prometheus_data:/prometheus + extra_hosts: + # Let Prometheus reach the API running on the Docker host on Windows/Mac. + - "host.docker.internal:host-gateway" command: - '--config.file=/etc/prometheus/prometheus.yml' - '--storage.tsdb.path=/prometheus' @@ -81,6 +84,8 @@ services: - GF_USERS_ALLOW_SIGN_UP=false volumes: - grafana_data:/var/lib/grafana + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning + - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards depends_on: - prometheus healthcheck: diff --git a/docs/monitoring.md b/docs/monitoring.md new file mode 100644 index 0000000..c6396d7 --- /dev/null +++ b/docs/monitoring.md @@ -0,0 +1,137 @@ +# Monitoring & Observability + +The Alian Structure API ships with a lightweight, production-ready observability +stack: **Prometheus metrics**, **OpenTelemetry tracing**, and **Grafana +dashboards**. This document covers what is exposed, how to enable it, and how to +run the local monitoring stack. + +## Architecture + +``` +┌────────────────────────┐ scrape :15s ┌────────────┐ queries ┌─────────┐ +│ Alian Structure API │ ───────────────▶│ Prometheus │ ─────────▶│ Grafana │ +│ /observability/metrics│ └────────────┘ └─────────┘ +│ │ OTLP traces ┌────────────┐ +│ OpenTelemetry SDK │ ───────────────▶│ Jaeger │ +└────────────────────────┘ └────────────┘ +``` + +- **Metrics** are collected with [`prom-client`](https://github.com/siimon/prom-client) + (`src/config/metrics.ts`), populated by `RequestTimingMiddleware`, + `DatabaseTimingInterceptor`, and service-level counters, then exposed in + Prometheus text-exposition format at `GET /api/v1/observability/metrics`. +- **Traces** are captured by the OpenTelemetry Node SDK (`src/config/tracing.ts`), + wrapped per-request by `TracingInterceptor`, and exported over OTLP/HTTP to + Jaeger (or any OTLP collector). +- **Dashboards** are provisioned from `monitoring/grafana/`. + +## Enabling monitoring + +All observability is controlled through environment variables. See +`.env.example` for the full annotated list. + +| Variable | Default | Purpose | +| --- | --- | --- | +| `TRACING_ENABLED` | `true` | Master toggle for OpenTelemetry tracing. Set `false` to remove tracing overhead entirely. | +| `OTEL_TRACES_SAMPLER_RATIO` | `1.0` (dev) / `0.1` (prod) | Fraction of traces to sample, `0.0`–`1.0`. Uses a parent-based ratio sampler so a sampled upstream trace stays sampled. | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | `http://localhost:4318/v1/traces` | OTLP/HTTP endpoint for trace export. | +| `JAEGER_ENDPOINT` / `JAEGER_AGENT_HOST` | _unset_ | Optional legacy Jaeger Thrift exporter for older Jaeger deployments. | +| `METRICS_AUTH_TOKEN` | _unset_ | When set, `/observability/metrics` requires this bearer token. Leave unset only on trusted private networks. | + +> **Production note:** always set `METRICS_AUTH_TOKEN` and keep +> `OTEL_TRACES_SAMPLER_RATIO` low (e.g. `0.1`) to bound overhead. + +## Metrics reference + +Default process/runtime metrics are exported under the `alian_structure_` +prefix (CPU, resident memory, heap usage, event-loop lag, uptime). Application +metrics include: + +| Metric | Type | Labels | Meaning | +| --- | --- | --- | --- | +| `alian_structure_http_requests_total` | Counter | `method`, `route`, `status_code` | Total HTTP requests. | +| `alian_structure_http_request_duration_seconds` | Histogram | `method`, `route`, `status_code` | Request latency distribution. | +| `alian_structure_http_requests_in_progress` | Gauge | `method`, `route` | In-flight requests. | +| `alian_structure_errors_total` | Counter | `type`, `severity` | Errors (HTTP `>=400` and internal). | +| `alian_structure_database_query_duration_seconds` | Histogram | `operation`, `table` | DB query latency. | +| `alian_structure_active_connections` | Gauge | `type` | Active connections. | +| `alian_structure_job_*` / `queue_length` | Histogram/Counter/Gauge | varies | Compute-job queue signals. | + +### Cardinality safety + +Route labels are normalised in `RequestTimingMiddleware` — UUIDs, numeric IDs, +Ethereum addresses, and tx hashes are collapsed to `:uuid`, `:id`, `:address`, +and `:hash`. Unmatched paths bucket to `unmatched`. This bounds label +cardinality so Prometheus memory stays predictable regardless of traffic shape. +Avoid adding high-cardinality labels (user IDs, raw paths) to any metric. + +## Tracing + +`TracingInterceptor` opens a span per request, propagates upstream W3C trace +context, and records status/exception on the span. Sampling is governed by +`OTEL_TRACES_SAMPLER_RATIO` via a `ParentBasedSampler(TraceIdRatioBased)`. + +**Do not put PII or secrets in span attributes.** Only method, route, status, +and request ID are attached by default; request bodies are never recorded. + +## Securing the metrics endpoint + +`/observability/metrics` is `@Public()` (a scraper has no JWT). When +`METRICS_AUTH_TOKEN` is set, the endpoint requires the token via either: + +```bash +curl -H "Authorization: Bearer $METRICS_AUTH_TOKEN" \ + http://localhost:3001/api/v1/observability/metrics +# or +curl "http://localhost:3001/api/v1/observability/metrics?token=$METRICS_AUTH_TOKEN" +``` + +The comparison is constant-time. Configure the token in Prometheus with +`authorization` / `bearer_token` in the scrape config. + +## Local monitoring stack + +`docker-compose.yml` includes Prometheus, Grafana, and Jaeger. + +```bash +docker compose up -d prometheus grafana jaeger +``` + +Then: + +- **Prometheus** — http://localhost:9090 (scrapes the API on the Docker host via + `host.docker.internal`; adjust `monitoring/prometheus/prometheus.yml` targets). +- **Grafana** — http://localhost:3001 (admin / admin). The Prometheus datasource + and the "Alian Structure - Application Monitoring" dashboard are + auto-provisioned from `monitoring/grafana/provisioning/`. +- **Jaeger** — http://localhost:16686 for trace search. + +### Files + +``` +monitoring/ +├── grafana/ +│ ├── dashboards/application-overview.json # 28-panel dashboard +│ └── provisioning/ +│ ├── datasources/prometheus.yml # auto-registers Prometheus +│ └── dashboards/dashboards.yml # loads dashboards on startup +└── prometheus/ + ├── prometheus.yml # scrape config + └── alerts.yml.example # starter SLO alerts +``` + +## Verifying + +```bash +# Metrics endpoint returns text-exposition output +curl -s http://localhost:3001/api/v1/observability/metrics | head -20 + +# Generate traffic, then confirm counters increment +curl -s http://localhost:3001/api/v1/health >/dev/null +curl -s http://localhost:3001/api/v1/observability/metrics \ + | grep alian_structure_http_requests_total +``` + +Unit tests cover the metrics endpoint and the trace sampler +(`src/observability/observability.controller.spec.ts`, +`src/config/tracing.spec.ts`). diff --git a/monitoring/grafana/provisioning/dashboards/dashboards.yml b/monitoring/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..6026e67 --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,17 @@ +# Grafana dashboard provisioning. +# +# Loads every dashboard JSON from the mounted dashboards directory (see +# docker-compose.yml) into the "Alian Structure" folder on startup. +apiVersion: 1 + +providers: + - name: alian-structure + orgId: 1 + folder: "Alian Structure" + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..4c6f1e3 --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,17 @@ +# Grafana datasource provisioning. +# +# Auto-registers the Prometheus instance defined in docker-compose.yml so the +# application-overview dashboard has a data source the moment Grafana starts. +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + uid: prometheus + access: proxy + # Service name from docker-compose.yml (containers share a network). + url: http://prometheus:9090 + isDefault: true + editable: true + jsonData: + timeInterval: 15s diff --git a/src/config/env.validation.ts b/src/config/env.validation.ts index 3e3557c..0597122 100644 --- a/src/config/env.validation.ts +++ b/src/config/env.validation.ts @@ -85,6 +85,30 @@ export class EnvironmentVariables { @IsUrl() OTEL_EXPORTER_OTLP_ENDPOINT?: string; + // Observability toggles + // Master switch for OpenTelemetry tracing. When false the SDK is never + // started, so there is zero tracing overhead. + @IsOptional() + @IsBoolean() + @Transform(({ value }) => value !== "false") + TRACING_ENABLED?: boolean = true; + + // Head-based trace sampling ratio in the range [0, 1]. 1 = sample every + // trace, 0 = sample none. Keeps tracing overhead bounded in production. + @IsOptional() + @IsNumber() + @Transform(({ value }) => (value === undefined ? 1 : parseFloat(value))) + @Min(0) + @Max(1) + OTEL_TRACES_SAMPLER_RATIO?: number = 1; + + // Optional bearer/query token that protects the Prometheus /metrics + // endpoint. When unset the endpoint is open (fine for private networks + // and local dev); set it in production so only the scraper can read it. + @IsOptional() + @IsString() + METRICS_AUTH_TOKEN?: string; + // Blockchain configuration @IsNumber() @Transform(({ value }) => parseInt(value, 10) || 1) diff --git a/src/config/tracing.spec.ts b/src/config/tracing.spec.ts new file mode 100644 index 0000000..fc4683e --- /dev/null +++ b/src/config/tracing.spec.ts @@ -0,0 +1,54 @@ +import { buildSampler, isTracingEnabled } from "./tracing"; + +/** + * The sampler is head-based and wrapped in a ParentBasedSampler so it always + * honours an upstream decision. These tests assert the ratio env var is read + * and clamped correctly, and that the tracing toggle only turns off on the + * literal string "false". + */ +describe("tracing sampler configuration", () => { + const ORIGINAL_RATIO = process.env.OTEL_TRACES_SAMPLER_RATIO; + const ORIGINAL_ENABLED = process.env.TRACING_ENABLED; + + afterEach(() => { + const restore = (key: string, value: string | undefined) => { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + }; + restore("OTEL_TRACES_SAMPLER_RATIO", ORIGINAL_RATIO); + restore("TRACING_ENABLED", ORIGINAL_ENABLED); + }); + + it("builds a sampler for the default (unset) ratio", () => { + delete process.env.OTEL_TRACES_SAMPLER_RATIO; + const sampler = buildSampler(); + expect(sampler).toBeDefined(); + // ParentBasedSampler describes itself with its delegate ratio. + expect(sampler.toString()).toContain("1"); + }); + + it("builds a sampler for a fractional ratio", () => { + process.env.OTEL_TRACES_SAMPLER_RATIO = "0.25"; + const sampler = buildSampler(); + expect(sampler.toString()).toContain("0.25"); + }); + + it("clamps out-of-range and non-numeric ratios to a valid sampler", () => { + process.env.OTEL_TRACES_SAMPLER_RATIO = "5"; + expect(() => buildSampler()).not.toThrow(); + process.env.OTEL_TRACES_SAMPLER_RATIO = "not-a-number"; + expect(() => buildSampler()).not.toThrow(); + }); + + it("is enabled by default and when TRACING_ENABLED is not 'false'", () => { + delete process.env.TRACING_ENABLED; + expect(isTracingEnabled()).toBe(true); + process.env.TRACING_ENABLED = "true"; + expect(isTracingEnabled()).toBe(true); + }); + + it("is disabled only when TRACING_ENABLED is exactly 'false'", () => { + process.env.TRACING_ENABLED = "false"; + expect(isTracingEnabled()).toBe(false); + }); +}); diff --git a/src/config/tracing.ts b/src/config/tracing.ts index 4318326..902bec7 100644 --- a/src/config/tracing.ts +++ b/src/config/tracing.ts @@ -7,6 +7,9 @@ import { BatchSpanProcessor, SimpleSpanProcessor, SpanExporter, + ParentBasedSampler, + TraceIdRatioBasedSampler, + Sampler, } from "@opentelemetry/sdk-trace-base"; import { trace, @@ -54,12 +57,32 @@ const resource = resourceFromAttributes({ let sdk: NodeSDK; +/** + * Build a head-based sampler from `OTEL_TRACES_SAMPLER_RATIO` (0..1). + * + * Wrapping the ratio sampler in a `ParentBasedSampler` means a sampling + * decision made upstream is always respected, so distributed traces are not + * broken part-way through. A ratio of 1 keeps the previous always-on + * behaviour; a lower value bounds tracing overhead in production. + */ +export function buildSampler(): Sampler { + const raw = process.env.OTEL_TRACES_SAMPLER_RATIO; + let ratio = raw === undefined ? 1 : parseFloat(raw); + if (!Number.isFinite(ratio)) ratio = 1; + ratio = Math.min(1, Math.max(0, ratio)); + + return new ParentBasedSampler({ + root: new TraceIdRatioBasedSampler(ratio), + }); +} + function buildSdk(): NodeSDK { const exporters = buildExporters(); const processors = exporters.map((exp) => new BatchSpanProcessor(exp)); return new NodeSDK({ resource, + sampler: buildSampler(), spanProcessors: processors, textMapPropagator: new W3CTraceContextPropagator(), instrumentations: [ @@ -78,11 +101,26 @@ function buildSdk(): NodeSDK { }); } +/** + * Whether tracing is enabled. Disabled only when `TRACING_ENABLED` is the + * literal string "false" so the default (unset) stays on. + */ +export const isTracingEnabled = (): boolean => + process.env.TRACING_ENABLED !== "false"; + export const startTracing = async (): Promise => { + if (!isTracingEnabled()) { + console.log("OpenTelemetry tracing disabled (TRACING_ENABLED=false)"); + return; + } try { sdk = buildSdk(); sdk.start(); - console.log("OpenTelemetry tracing initialized"); + console.log( + `OpenTelemetry tracing initialized (sampler ratio=${ + process.env.OTEL_TRACES_SAMPLER_RATIO ?? "1" + })`, + ); } catch (err) { console.error("Failed to start OpenTelemetry SDK:", err); } diff --git a/src/observability/observability.controller.spec.ts b/src/observability/observability.controller.spec.ts index 157be27..f50d467 100644 --- a/src/observability/observability.controller.spec.ts +++ b/src/observability/observability.controller.spec.ts @@ -19,6 +19,17 @@ function makeRes(): { setHeader: jest.Mock; headers: Record } { } as any; } +/** + * Minimal Express-request stub carrying the auth header / query token the + * metrics endpoint inspects when `METRICS_AUTH_TOKEN` is set. + */ +function makeReq(opts: { authorization?: string; token?: string } = {}): any { + return { + headers: opts.authorization ? { authorization: opts.authorization } : {}, + query: opts.token ? { token: opts.token } : {}, + }; +} + describe("ObservabilityController - Prometheus /metrics endpoint (issue #25)", () => { let controller: ObservabilityController; @@ -56,7 +67,7 @@ describe("ObservabilityController - Prometheus /metrics endpoint (issue #25)", ( httpRequestTotal.reset(); const res = makeRes(); - const body = await controller.getMetrics(res as any); + const body = await controller.getMetrics(makeReq(), res as any); expect(typeof body).toBe("string"); expect(body.length).toBeGreaterThan(0); @@ -76,7 +87,7 @@ describe("ObservabilityController - Prometheus /metrics endpoint (issue #25)", ( }); const res = makeRes(); - const body = await controller.getMetrics(res as any); + const body = await controller.getMetrics(makeReq(), res as any); expect(body).toContain( "# TYPE alian_structure_http_requests_total counter", @@ -85,6 +96,59 @@ describe("ObservabilityController - Prometheus /metrics endpoint (issue #25)", ( /alian_structure_http_requests_total\{[^}]*method="GET"[^}]*\}/, ); }); + + describe("token protection via METRICS_AUTH_TOKEN", () => { + const ORIGINAL = process.env.METRICS_AUTH_TOKEN; + + afterEach(() => { + if (ORIGINAL === undefined) { + delete process.env.METRICS_AUTH_TOKEN; + } else { + process.env.METRICS_AUTH_TOKEN = ORIGINAL; + } + }); + + it("stays open when no token is configured", async () => { + delete process.env.METRICS_AUTH_TOKEN; + const body = await controller.getMetrics(makeReq(), makeRes() as any); + expect(typeof body).toBe("string"); + }); + + it("rejects requests with no token when one is configured", async () => { + process.env.METRICS_AUTH_TOKEN = "s3cret"; + await expect( + controller.getMetrics(makeReq(), makeRes() as any), + ).rejects.toThrow(/token/i); + }); + + it("rejects requests bearing the wrong token", async () => { + process.env.METRICS_AUTH_TOKEN = "s3cret"; + await expect( + controller.getMetrics( + makeReq({ authorization: "Bearer nope" }), + makeRes() as any, + ), + ).rejects.toThrow(/token/i); + }); + + it("accepts the correct token via Authorization header", async () => { + process.env.METRICS_AUTH_TOKEN = "s3cret"; + const body = await controller.getMetrics( + makeReq({ authorization: "Bearer s3cret" }), + makeRes() as any, + ); + expect(typeof body).toBe("string"); + }); + + it("accepts the correct token via query param", async () => { + process.env.METRICS_AUTH_TOKEN = "s3cret"; + const body = await controller.getMetrics( + makeReq({ token: "s3cret" }), + makeRes() as any, + ); + expect(typeof body).toBe("string"); + }); + }); }); diff --git a/src/observability/observability.controller.ts b/src/observability/observability.controller.ts index 8151bcc..62094fc 100644 --- a/src/observability/observability.controller.ts +++ b/src/observability/observability.controller.ts @@ -1,6 +1,14 @@ -import { Controller, Get, Post, Res } from "@nestjs/common"; +import { + Controller, + Get, + Post, + Req, + Res, + UnauthorizedException, +} from "@nestjs/common"; import { ApiOperation, ApiResponse, ApiTags } from "@nestjs/swagger"; -import { Response } from "express"; +import { Request, Response } from "express"; +import { timingSafeEqual } from "crypto"; import { Public } from "../common/decorators/public.decorator"; import { SkipKyc } from "../common/decorators/skip-kyc.decorator"; import { @@ -110,7 +118,9 @@ export class ObservabilityController { * Exposes every metric registered in `src/config/metrics.ts` in Prometheus * text-exposition format so that a Prometheus server can scrape it. * Marked `@Public()` / `@SkipKyc()` because the scraper does not have a JWT - * and KYC is unrelated to operational metrics. + * and KYC is unrelated to operational metrics. In production, set + * `METRICS_AUTH_TOKEN` so only an authorised scraper can read it (see + * `assertMetricsAuthorized`). */ @Get("metrics") @ApiOperation({ @@ -123,13 +133,58 @@ export class ObservabilityController { status: 200, description: "Prometheus metrics in text exposition format", }) - async getMetrics(@Res({ passthrough: true }) res: Response): Promise { + async getMetrics( + @Req() req: Request, + @Res({ passthrough: true }) res: Response, + ): Promise { + this.assertMetricsAuthorized(req); + // Make sure the response carries the Prometheus text-exposition content // type. Without this, NestJS's default handler would JSON-encode the // response body and scrapers would reject the payload as invalid. res.setHeader("Content-Type", register.contentType); return register.metrics(); } + + /** + * Optionally protect the metrics endpoint with a shared token. + * + * When `METRICS_AUTH_TOKEN` is unset the endpoint stays open — fine for a + * private network or local dev. When it is set, the scraper must present + * the token via `Authorization: Bearer ` or a `?token=` query + * param. The comparison is constant-time to avoid leaking the token + * through timing side-channels. + */ + private assertMetricsAuthorized(req: Request): void { + const expected = process.env.METRICS_AUTH_TOKEN; + if (!expected) return; + + const header = req.headers["authorization"]; + const headerToken = + typeof header === "string" && header.startsWith("Bearer ") + ? header.slice("Bearer ".length) + : undefined; + const queryToken = + typeof req.query?.token === "string" ? req.query.token : undefined; + const provided = headerToken ?? queryToken; + + if (!provided || !this.constantTimeEquals(provided, expected)) { + throw new UnauthorizedException("Invalid or missing metrics token"); + } + } + + private constantTimeEquals(a: string, b: string): boolean { + const bufA = Buffer.from(a); + const bufB = Buffer.from(b); + // timingSafeEqual throws on length mismatch; compare a self-equal buffer + // of matching length first so the early return itself stays constant-time + // for equal-length inputs. + if (bufA.length !== bufB.length) { + timingSafeEqual(bufA, bufA); + return false; + } + return timingSafeEqual(bufA, bufB); + } }