feat(observability): add telemetry toggles, trace sampling, metrics auth

.env.example

@@ -40,6 +40,22 @@ SENTRY_ENVIRONMENT=development
 SENTRY_RELEASE=0.1.0
 SENTRY_TRACES_SAMPLE_RATE=0.1
 
+# Observability — OpenTelemetry tracing + Prometheus metrics
+# Master switch for distributed tracing. Set to "false" to skip starting the
+# OpenTelemetry SDK entirely (metrics remain available regardless).
+TRACING_ENABLED=true
+# Head-based trace sampling ratio in [0,1]. 1 = sample every trace (dev),
+# lower it in production to bound overhead, e.g. 0.1 = 10% of traces.
+OTEL_TRACES_SAMPLER_RATIO=1.0
+# OTLP HTTP endpoint traces are exported to (Jaeger/Tempo/OTel Collector).
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318/v1/traces
+# Optional legacy Jaeger Thrift endpoint (only needed for pre-OTLP Jaeger).
+# JAEGER_ENDPOINT=http://localhost:14268/api/traces
+# Optional shared token protecting the Prometheus /metrics endpoint. When set,
+# scrapers must send "Authorization: Bearer <token>" or "?token=<token>".
+# Leave blank on a trusted/private network. Strongly recommended in production.
+METRICS_AUTH_TOKEN=
+
 # Email Service (Ethereal for testing, configure SMTP for production)
 SMTP_HOST=smtp.ethereal.email
 SMTP_PORT=587

.env.production.example

@@ -41,6 +41,17 @@ SENTRY_ENVIRONMENT=production
 SENTRY_RELEASE=1.0.0
 SENTRY_TRACES_SAMPLE_RATE=0.05
 
+# Observability — OpenTelemetry tracing + Prometheus metrics
+# Toggle distributed tracing on/off. Disable to remove all tracing overhead.
+TRACING_ENABLED=true
+# Fraction of traces to sample (0.0–1.0). Keep low in production to bound overhead.
+OTEL_TRACES_SAMPLER_RATIO=0.1
+# OTLP/HTTP traces endpoint (Jaeger 1.41+, Tempo, or an OTel collector).
+OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:4318/v1/traces
+# REQUIRED in production: shared bearer token guarding /observability/metrics.
+# Prometheus must send it via `Authorization: Bearer <token>`.
+METRICS_AUTH_TOKEN=CHANGE_ME_TO_A_LONG_RANDOM_SECRET
+
 # Email Service - Use production SMTP (SendGrid, AWS SES, etc.)
 SMTP_HOST=smtp.sendgrid.net
 SMTP_PORT=587

docker-compose.yml

@@ -60,8 +60,11 @@ services:
     ports:
       - "9090:9090"
     volumes:
-      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+      - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
       - prometheus_data:/prometheus
+    extra_hosts:
+      # Let Prometheus reach the API running on the Docker host on Windows/Mac.
+      - "host.docker.internal:host-gateway"
     command:
       - '--config.file=/etc/prometheus/prometheus.yml'
       - '--storage.tsdb.path=/prometheus'
@@ -81,6 +84,8 @@ services:
       - GF_USERS_ALLOW_SIGN_UP=false
     volumes:
       - grafana_data:/var/lib/grafana
+      - ./monitoring/grafana/provisioning:/etc/grafana/provisioning
+      - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards
     depends_on:
       - prometheus
     healthcheck:

docs/monitoring.md

@@ -0,0 +1,137 @@
+# Monitoring & Observability
+
+The Alian Structure API ships with a lightweight, production-ready observability
+stack: **Prometheus metrics**, **OpenTelemetry tracing**, and **Grafana
+dashboards**. This document covers what is exposed, how to enable it, and how to
+run the local monitoring stack.
+
+## Architecture
+
+```
+┌────────────────────────┐   scrape :15s   ┌────────────┐  queries  ┌─────────┐
+│  Alian Structure API   │ ───────────────▶│ Prometheus │ ─────────▶│ Grafana │
+│  /observability/metrics│                 └────────────┘           └─────────┘
+│                        │   OTLP traces    ┌────────────┐
+│  OpenTelemetry SDK     │ ───────────────▶│   Jaeger    │
+└────────────────────────┘                 └────────────┘
+```
+
+- **Metrics** are collected with [`prom-client`](https://github.com/siimon/prom-client)
+  (`src/config/metrics.ts`), populated by `RequestTimingMiddleware`,
+  `DatabaseTimingInterceptor`, and service-level counters, then exposed in
+  Prometheus text-exposition format at `GET /api/v1/observability/metrics`.
+- **Traces** are captured by the OpenTelemetry Node SDK (`src/config/tracing.ts`),
+  wrapped per-request by `TracingInterceptor`, and exported over OTLP/HTTP to
+  Jaeger (or any OTLP collector).
+- **Dashboards** are provisioned from `monitoring/grafana/`.
+
+## Enabling monitoring
+
+All observability is controlled through environment variables. See
+`.env.example` for the full annotated list.
+
+| Variable | Default | Purpose |
+| --- | --- | --- |
+| `TRACING_ENABLED` | `true` | Master toggle for OpenTelemetry tracing. Set `false` to remove tracing overhead entirely. |
+| `OTEL_TRACES_SAMPLER_RATIO` | `1.0` (dev) / `0.1` (prod) | Fraction of traces to sample, `0.0`–`1.0`. Uses a parent-based ratio sampler so a sampled upstream trace stays sampled. |
+| `OTEL_EXPORTER_OTLP_ENDPOINT` | `http://localhost:4318/v1/traces` | OTLP/HTTP endpoint for trace export. |
+| `JAEGER_ENDPOINT` / `JAEGER_AGENT_HOST` | _unset_ | Optional legacy Jaeger Thrift exporter for older Jaeger deployments. |
+| `METRICS_AUTH_TOKEN` | _unset_ | When set, `/observability/metrics` requires this bearer token. Leave unset only on trusted private networks. |
+
+> **Production note:** always set `METRICS_AUTH_TOKEN` and keep
+> `OTEL_TRACES_SAMPLER_RATIO` low (e.g. `0.1`) to bound overhead.
+
+## Metrics reference
+
+Default process/runtime metrics are exported under the `alian_structure_`
+prefix (CPU, resident memory, heap usage, event-loop lag, uptime). Application
+metrics include:
+
+| Metric | Type | Labels | Meaning |
+| --- | --- | --- | --- |
+| `alian_structure_http_requests_total` | Counter | `method`, `route`, `status_code` | Total HTTP requests. |
+| `alian_structure_http_request_duration_seconds` | Histogram | `method`, `route`, `status_code` | Request latency distribution. |
+| `alian_structure_http_requests_in_progress` | Gauge | `method`, `route` | In-flight requests. |
+| `alian_structure_errors_total` | Counter | `type`, `severity` | Errors (HTTP `>=400` and internal). |
+| `alian_structure_database_query_duration_seconds` | Histogram | `operation`, `table` | DB query latency. |
+| `alian_structure_active_connections` | Gauge | `type` | Active connections. |
+| `alian_structure_job_*` / `queue_length` | Histogram/Counter/Gauge | varies | Compute-job queue signals. |
+
+### Cardinality safety
+
+Route labels are normalised in `RequestTimingMiddleware` — UUIDs, numeric IDs,
+Ethereum addresses, and tx hashes are collapsed to `:uuid`, `:id`, `:address`,
+and `:hash`. Unmatched paths bucket to `unmatched`. This bounds label
+cardinality so Prometheus memory stays predictable regardless of traffic shape.
+Avoid adding high-cardinality labels (user IDs, raw paths) to any metric.
+
+## Tracing
+
+`TracingInterceptor` opens a span per request, propagates upstream W3C trace
+context, and records status/exception on the span. Sampling is governed by
+`OTEL_TRACES_SAMPLER_RATIO` via a `ParentBasedSampler(TraceIdRatioBased)`.
+
+**Do not put PII or secrets in span attributes.** Only method, route, status,
+and request ID are attached by default; request bodies are never recorded.
+
+## Securing the metrics endpoint
+
+`/observability/metrics` is `@Public()` (a scraper has no JWT). When
+`METRICS_AUTH_TOKEN` is set, the endpoint requires the token via either:
+
+```bash
+curl -H "Authorization: Bearer $METRICS_AUTH_TOKEN" \
+  http://localhost:3001/api/v1/observability/metrics
+# or
+curl "http://localhost:3001/api/v1/observability/metrics?token=$METRICS_AUTH_TOKEN"
+```
+
+The comparison is constant-time. Configure the token in Prometheus with
+`authorization` / `bearer_token` in the scrape config.
+
+## Local monitoring stack
+
+`docker-compose.yml` includes Prometheus, Grafana, and Jaeger.
+
+```bash
+docker compose up -d prometheus grafana jaeger
+```
+
+Then:
+
+- **Prometheus** — http://localhost:9090 (scrapes the API on the Docker host via
+  `host.docker.internal`; adjust `monitoring/prometheus/prometheus.yml` targets).
+- **Grafana** — http://localhost:3001 (admin / admin). The Prometheus datasource
+  and the "Alian Structure - Application Monitoring" dashboard are
+  auto-provisioned from `monitoring/grafana/provisioning/`.
+- **Jaeger** — http://localhost:16686 for trace search.
+
+### Files
+
+```
+monitoring/
+├── grafana/
+│   ├── dashboards/application-overview.json      # 28-panel dashboard
+│   └── provisioning/
+│       ├── datasources/prometheus.yml            # auto-registers Prometheus
+│       └── dashboards/dashboards.yml             # loads dashboards on startup
+└── prometheus/
+    ├── prometheus.yml                            # scrape config
+    └── alerts.yml.example                        # starter SLO alerts
+```
+
+## Verifying
+
+```bash
+# Metrics endpoint returns text-exposition output
+curl -s http://localhost:3001/api/v1/observability/metrics | head -20
+
+# Generate traffic, then confirm counters increment
+curl -s http://localhost:3001/api/v1/health >/dev/null
+curl -s http://localhost:3001/api/v1/observability/metrics \
+  | grep alian_structure_http_requests_total
+```
+
+Unit tests cover the metrics endpoint and the trace sampler
+(`src/observability/observability.controller.spec.ts`,
+`src/config/tracing.spec.ts`).

monitoring/grafana/provisioning/dashboards/dashboards.yml

@@ -0,0 +1,17 @@
+# Grafana dashboard provisioning.
+#
+# Loads every dashboard JSON from the mounted dashboards directory (see
+# docker-compose.yml) into the "Alian Structure" folder on startup.
+apiVersion: 1
+
+providers:
+  - name: alian-structure
+    orgId: 1
+    folder: "Alian Structure"
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 30
+    allowUiUpdates: true
+    options:
+      path: /var/lib/grafana/dashboards
+      foldersFromFilesStructure: false

monitoring/grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,17 @@
+# Grafana datasource provisioning.
+#
+# Auto-registers the Prometheus instance defined in docker-compose.yml so the
+# application-overview dashboard has a data source the moment Grafana starts.
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    uid: prometheus
+    access: proxy
+    # Service name from docker-compose.yml (containers share a network).
+    url: http://prometheus:9090
+    isDefault: true
+    editable: true
+    jsonData:
+      timeInterval: 15s

src/config/env.validation.ts

@@ -85,6 +85,30 @@ export class EnvironmentVariables {
   @IsUrl()
   OTEL_EXPORTER_OTLP_ENDPOINT?: string;
 
+  // Observability toggles
+  // Master switch for OpenTelemetry tracing. When false the SDK is never
+  // started, so there is zero tracing overhead.
+  @IsOptional()
+  @IsBoolean()
+  @Transform(({ value }) => value !== "false")
+  TRACING_ENABLED?: boolean = true;
+
+  // Head-based trace sampling ratio in the range [0, 1]. 1 = sample every
+  // trace, 0 = sample none. Keeps tracing overhead bounded in production.
+  @IsOptional()
+  @IsNumber()
+  @Transform(({ value }) => (value === undefined ? 1 : parseFloat(value)))
+  @Min(0)
+  @Max(1)
+  OTEL_TRACES_SAMPLER_RATIO?: number = 1;
+
+  // Optional bearer/query token that protects the Prometheus /metrics
+  // endpoint. When unset the endpoint is open (fine for private networks
+  // and local dev); set it in production so only the scraper can read it.
+  @IsOptional()
+  @IsString()
+  METRICS_AUTH_TOKEN?: string;
+
   // Blockchain configuration
   @IsNumber()
   @Transform(({ value }) => parseInt(value, 10) || 1)

src/config/tracing.spec.ts

@@ -0,0 +1,54 @@
+import { buildSampler, isTracingEnabled } from "./tracing";
+
+/**
+ * The sampler is head-based and wrapped in a ParentBasedSampler so it always
+ * honours an upstream decision. These tests assert the ratio env var is read
+ * and clamped correctly, and that the tracing toggle only turns off on the
+ * literal string "false".
+ */
+describe("tracing sampler configuration", () => {
+  const ORIGINAL_RATIO = process.env.OTEL_TRACES_SAMPLER_RATIO;
+  const ORIGINAL_ENABLED = process.env.TRACING_ENABLED;
+
+  afterEach(() => {
+    const restore = (key: string, value: string | undefined) => {
+      if (value === undefined) delete process.env[key];
+      else process.env[key] = value;
+    };
+    restore("OTEL_TRACES_SAMPLER_RATIO", ORIGINAL_RATIO);
+    restore("TRACING_ENABLED", ORIGINAL_ENABLED);
+  });
+
+  it("builds a sampler for the default (unset) ratio", () => {
+    delete process.env.OTEL_TRACES_SAMPLER_RATIO;
+    const sampler = buildSampler();
+    expect(sampler).toBeDefined();
+    // ParentBasedSampler describes itself with its delegate ratio.
+    expect(sampler.toString()).toContain("1");
+  });
+
+  it("builds a sampler for a fractional ratio", () => {
+    process.env.OTEL_TRACES_SAMPLER_RATIO = "0.25";
+    const sampler = buildSampler();
+    expect(sampler.toString()).toContain("0.25");
+  });
+
+  it("clamps out-of-range and non-numeric ratios to a valid sampler", () => {
+    process.env.OTEL_TRACES_SAMPLER_RATIO = "5";
+    expect(() => buildSampler()).not.toThrow();
+    process.env.OTEL_TRACES_SAMPLER_RATIO = "not-a-number";
+    expect(() => buildSampler()).not.toThrow();
+  });
+
+  it("is enabled by default and when TRACING_ENABLED is not 'false'", () => {
+    delete process.env.TRACING_ENABLED;
+    expect(isTracingEnabled()).toBe(true);
+    process.env.TRACING_ENABLED = "true";
+    expect(isTracingEnabled()).toBe(true);
+  });
+
+  it("is disabled only when TRACING_ENABLED is exactly 'false'", () => {
+    process.env.TRACING_ENABLED = "false";
+    expect(isTracingEnabled()).toBe(false);
+  });
+});

src/config/tracing.ts

@@ -7,6 +7,9 @@ import {
   BatchSpanProcessor,
   SimpleSpanProcessor,
   SpanExporter,
+  ParentBasedSampler,
+  TraceIdRatioBasedSampler,
+  Sampler,
 } from "@opentelemetry/sdk-trace-base";
 import {
   trace,
@@ -54,12 +57,32 @@ const resource = resourceFromAttributes({
 
 let sdk: NodeSDK;
 
+/**
+ * Build a head-based sampler from `OTEL_TRACES_SAMPLER_RATIO` (0..1).
+ *
+ * Wrapping the ratio sampler in a `ParentBasedSampler` means a sampling
+ * decision made upstream is always respected, so distributed traces are not
+ * broken part-way through. A ratio of 1 keeps the previous always-on
+ * behaviour; a lower value bounds tracing overhead in production.
+ */
+export function buildSampler(): Sampler {
+  const raw = process.env.OTEL_TRACES_SAMPLER_RATIO;
+  let ratio = raw === undefined ? 1 : parseFloat(raw);
+  if (!Number.isFinite(ratio)) ratio = 1;
+  ratio = Math.min(1, Math.max(0, ratio));
+
+  return new ParentBasedSampler({
+    root: new TraceIdRatioBasedSampler(ratio),
+  });
+}
+
 function buildSdk(): NodeSDK {
   const exporters = buildExporters();
   const processors = exporters.map((exp) => new BatchSpanProcessor(exp));
 
   return new NodeSDK({
     resource,
+    sampler: buildSampler(),
     spanProcessors: processors,
     textMapPropagator: new W3CTraceContextPropagator(),
     instrumentations: [
@@ -78,11 +101,26 @@ function buildSdk(): NodeSDK {
   });
 }
 
+/**
+ * Whether tracing is enabled. Disabled only when `TRACING_ENABLED` is the
+ * literal string "false" so the default (unset) stays on.
+ */
+export const isTracingEnabled = (): boolean =>
+  process.env.TRACING_ENABLED !== "false";
+
 export const startTracing = async (): Promise<void> => {
+  if (!isTracingEnabled()) {
+    console.log("OpenTelemetry tracing disabled (TRACING_ENABLED=false)");
+    return;
+  }
   try {
     sdk = buildSdk();
     sdk.start();
-    console.log("OpenTelemetry tracing initialized");
+    console.log(
+      `OpenTelemetry tracing initialized (sampler ratio=${
+        process.env.OTEL_TRACES_SAMPLER_RATIO ?? "1"
+      })`,
+    );
   } catch (err) {
     console.error("Failed to start OpenTelemetry SDK:", err);
   }