feat(observability): distributed tracing via OpenTelemetry + Tempo

apps/api/package.json

@@ -53,6 +53,12 @@
     "@elysiajs/cors": "1.4.2",
     "@elysiajs/jwt": "1.4.2",
     "@elysiajs/swagger": "1.3.1",
+    "@opentelemetry/api": "1.9.1",
+    "@opentelemetry/auto-instrumentations-node": "0.76.0",
+    "@opentelemetry/exporter-trace-otlp-http": "0.218.0",
+    "@opentelemetry/resources": "2.7.1",
+    "@opentelemetry/sdk-node": "0.218.0",
+    "@opentelemetry/semantic-conventions": "1.41.1",
     "@sendgrid/mail": "8.1.6",
     "@sentry/bun": "10.53.1",
     "@sinclair/typebox": "0.34.49",

apps/api/src/config/env/schema.ts

@@ -79,6 +79,16 @@ export const envSchema = t.Object({
     default: 0.1,
   }),
 
+  /*
+   * OpenTelemetry tracing. When OTEL_EXPORTER_OTLP_ENDPOINT is set, the
+   * API ships spans via OTLP/HTTP to that endpoint (Tempo, the trace
+   * backend bundled in compose). Empty = OTel SDK is not initialized.
+   * OTEL_SERVICE_NAME shows up as the `service.name` attribute Grafana
+   * uses to group spans in Tempo's Explore tab.
+   */
+  OTEL_EXPORTER_OTLP_ENDPOINT: t.String({ default: "" }),
+  OTEL_SERVICE_NAME: t.String({ default: "boringstack-api" }),
+
   EMAIL_PROVIDER: t.Union(
     [
       t.Literal("cloudflare"),

apps/api/src/config/env/validate.ts

@@ -124,6 +124,11 @@ const readSentry = (source: EnvSource) => ({
   SENTRY_TRACES_SAMPLE_RATE: toFloat(source.SENTRY_TRACES_SAMPLE_RATE, 0.1),
 });
 
+const readOpenTelemetry = (source: EnvSource) => ({
+  OTEL_EXPORTER_OTLP_ENDPOINT: source.OTEL_EXPORTER_OTLP_ENDPOINT ?? "",
+  OTEL_SERVICE_NAME: source.OTEL_SERVICE_NAME ?? "boringstack-api",
+});
+
 const readEmail = (source: EnvSource) => ({
   EMAIL_PROVIDER: source.EMAIL_PROVIDER ?? "cloudflare",
   EMAIL_FROM: source.EMAIL_FROM ?? "noreply@example.com",
@@ -203,6 +208,7 @@ const readRaw = (source: EnvSource): Record<string, unknown> => ({
   ...readUrls(source),
   ...readRateLimit(source),
   ...readSentry(source),
+  ...readOpenTelemetry(source),
   ...readEmail(source),
   ...readOAuth(source),
   ...readAI(source),

apps/api/src/config/logger/logger.ts

@@ -1,3 +1,4 @@
+import { trace } from "@opentelemetry/api";
 import * as Sentry from "@sentry/bun";
 import pino from "pino";
 import { env } from "../env";
@@ -6,28 +7,37 @@ import type { LOG_EVENTS } from "./logger.events";
 type ILogEventName = (typeof LOG_EVENTS)[number];
 
 /*
- * Inject Sentry-scoped correlation fields on every log record:
- *   - trace_id + span_id from the active Sentry/OTel span
- *   - userId from the current scope (set by auth.plugin.ts after the
- *     user is resolved on an authenticated request)
+ * Inject correlation fields on every log record:
+ *   - trace_id + span_id from the active OpenTelemetry span (set by
+ *     the OTel SDK's HTTP / undici / ioredis auto-instrumentations, or
+ *     by manual `withQueueSpan` / `withDbSpan` wrappers).
+ *   - userId from the current Sentry scope (set by auth.plugin.ts
+ *     after the user is resolved on an authenticated request).
  *
  * Promtail extracts these as Loki structured metadata so a log line
- * surfaced in Grafana can be pivoted to its trace or its user in
- * Sentry/GlitchTip by the same id. Each field is a no-op when its
- * source isn't set — unauthenticated requests get trace ids but no
- * userId; everything is `{}` before Sentry.init when no DSN is
- * configured.
+ * surfaced in Grafana can be pivoted to the matching Tempo trace
+ * (trace_id), the matching GlitchTip event (trace_id / user.id), or
+ * filtered to a single user's activity. Each field is a no-op when
+ * its source isn't set — pre-init, unauthenticated requests, or
+ * code that runs outside a span context.
+ *
+ * @opentelemetry/api is used rather than Sentry's getActiveSpan so a
+ * single API works both when the OTel SDK is the source of truth
+ * (Tempo enabled) and when only Sentry's internal OTel context is
+ * running (DSN set, OTel endpoint empty).
  */
 const traceMixin = (): Record<string, string> => {
   const fields: Record<string, string> = {};
 
-  const span = Sentry.getActiveSpan();
+  const span = trace.getActiveSpan();
 
   if (span !== undefined) {
     const ctx = span.spanContext();
 
-    fields.trace_id = ctx.traceId;
-    fields.span_id = ctx.spanId;
+    if (ctx.traceId !== "00000000000000000000000000000000") {
+      fields.trace_id = ctx.traceId;
+      fields.span_id = ctx.spanId;
+    }
   }
 
   const userId = Sentry.getCurrentScope().getUser()?.id;

apps/api/src/config/otel/index.ts

@@ -0,0 +1 @@
+export { initializeOpenTelemetry, shutdownOpenTelemetry } from "./otel";

apps/api/src/config/otel/otel.ts

@@ -0,0 +1,77 @@
+import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
+import { resourceFromAttributes } from "@opentelemetry/resources";
+import { NodeSDK } from "@opentelemetry/sdk-node";
+import {
+  ATTR_SERVICE_NAME,
+  ATTR_SERVICE_VERSION,
+} from "@opentelemetry/semantic-conventions";
+
+import { env } from "../env";
+
+/*
+ * Distributed tracing via OpenTelemetry.
+ *
+ * Spans flow: this SDK collects them in-process and ships via OTLP/HTTP to
+ * Tempo (the trace backend bundled in the observability compose stack).
+ * Grafana queries Tempo by trace_id; the same trace_id is stamped on every
+ * Pino log record (see config/logger/logger.ts) and on every Sentry/GlitchTip
+ * error event, so a slow request found in metrics can be pivoted to its
+ * trace and the logs around it without leaving Grafana.
+ *
+ * Initialised once at boot, before any instrumented code runs. The auto-
+ * instrumentations patch HTTP / undici (outgoing fetch) / ioredis (Valkey +
+ * BullMQ) / fs / dns / and a handful more — see
+ * @opentelemetry/auto-instrumentations-node for the full list.
+ *
+ * Not auto-instrumented: postgres-js (Drizzle's underlying driver — no
+ * upstream OTel instrumentation exists for it). Wrap DB calls manually
+ * with `withDbSpan` from lib/tracing when you want them visible.
+ *
+ * No-op when OTEL_EXPORTER_OTLP_ENDPOINT is empty (the default outside of
+ * compose), so unit tests + standalone runs don't try to export to a host
+ * that isn't there.
+ */
+let sdk: NodeSDK | null = null;
+
+export const initializeOpenTelemetry = (): void => {
+  if (env.OTEL_EXPORTER_OTLP_ENDPOINT === "") {
+    return;
+  }
+
+  sdk = new NodeSDK({
+    resource: resourceFromAttributes({
+      [ATTR_SERVICE_NAME]: env.OTEL_SERVICE_NAME,
+      [ATTR_SERVICE_VERSION]: env.APP_NAME,
+      "deployment.environment": env.NODE_ENV,
+    }),
+    traceExporter: new OTLPTraceExporter({
+      url: `${env.OTEL_EXPORTER_OTLP_ENDPOINT}/v1/traces`,
+    }),
+    instrumentations: [
+      getNodeAutoInstrumentations({
+        /*
+         * Disable file-system instrumentation: it generates an enormous
+         * volume of spans for normal Node operations and drowns out the
+         * application signal. Disable dns for the same reason.
+         */
+        "@opentelemetry/instrumentation-fs": { enabled: false },
+        "@opentelemetry/instrumentation-dns": { enabled: false },
+      }),
+    ],
+  });
+
+  sdk.start();
+};
+
+/*
+ * Best-effort shutdown — called from the process exit handlers so
+ * in-flight spans get a chance to flush before the runtime exits.
+ */
+export const shutdownOpenTelemetry = async (): Promise<void> => {
+  if (sdk === null) {
+    return;
+  }
+
+  await sdk.shutdown();
+};

apps/api/src/index.ts

@@ -1,3 +1,10 @@
+/*
+ * OpenTelemetry init must run before anything that touches HTTP / ioredis /
+ * undici, so the auto-instrumentations can patch them at import time. See
+ * src/instrument.ts.
+ */
+import "./instrument";
+
 import { createApp } from "./config/app";
 import { env } from "./config/env";
 import {
@@ -8,7 +15,7 @@ import { logStartup } from "./config/logger";
 import { initializeSentry } from "./config/sentry";
 import { setupNotifications, setupQueues } from "./config/setup";
 
-// Initialize Sentry FIRST so any bootstrap error is captured.
+// Initialize Sentry after OTel so error events pick up the OTel trace context.
 initializeSentry();
 
 const app = createApp().listen(env.PORT);

apps/api/src/instrument.ts

@@ -0,0 +1,13 @@
+/*
+ * Bootstrap entry-point for the OpenTelemetry SDK.
+ *
+ * Auto-instrumentations patch the modules they cover (http, undici,
+ * ioredis, ...) at *import time*. That patching must happen before any
+ * other module imports those targets, or it won't take effect. The
+ * cleanest enforcement of "run this first" in JavaScript is a side-
+ * effect import placed at the top of src/index.ts — this file is that
+ * side effect.
+ */
+import { initializeOpenTelemetry } from "./config/otel";
+
+initializeOpenTelemetry();

apps/api/src/lib/tracing/index.ts

@@ -0,0 +1,2 @@
+export { withDbSpan } from "./withDbSpan";
+export { withQueueSpan } from "./withQueueSpan";

apps/api/src/lib/tracing/withDbSpan.ts

@@ -0,0 +1,66 @@
+import { SpanStatusCode, trace } from "@opentelemetry/api";
+
+import { getErrorMessage } from "../errors";
+
+const tracer = trace.getTracer("boringstack-api/db");
+
+/*
+ * Wrap a Drizzle query (or any async DB call) in an OpenTelemetry span
+ * so the duration becomes a child of the current request span — visible
+ * as a row in the Tempo trace waterfall under the parent HTTP span.
+ *
+ * postgres-js has no upstream OTel auto-instrumentation, so DB query
+ * spans are opt-in at the call site. Use this for hot paths and code
+ * you're benchmarking; the trace will pinpoint slow queries that look
+ * like "anonymous internal time" inside the parent request span without
+ * it.
+ *
+ * Usage:
+ *
+ *   const user = await withDbSpan(
+ *     "users.findById",
+ *     { "db.statement": "select id, email from users where id = $1" },
+ *     () => db.query.users.findFirst({ where: eq(users.id, userId) })
+ *   );
+ *
+ * The `attributes` arg follows OTel's db.* semantic conventions
+ * (https://opentelemetry.io/docs/specs/semconv/database/) — keep
+ * statements parameterised; never put PII or unbounded values in span
+ * attributes.
+ */
+export const withDbSpan = async <T>(
+  spanName: string,
+  attributes: Record<string, string | number | boolean>,
+  handler: () => Promise<T>
+): Promise<T> =>
+  tracer.startActiveSpan(
+    `db.${spanName}`,
+    {
+      attributes: {
+        "db.system": "postgresql",
+        ...attributes,
+      },
+    },
+    async (span) => {
+      try {
+        const result = await handler();
+
+        span.setStatus({ code: SpanStatusCode.OK });
+
+        return result;
+      } catch (error) {
+        span.setStatus({
+          code: SpanStatusCode.ERROR,
+          message: getErrorMessage(error),
+        });
+
+        if (error instanceof Error) {
+          span.recordException(error);
+        }
+
+        throw error;
+      } finally {
+        span.end();
+      }
+    }
+  );

apps/api/src/lib/tracing/withQueueSpan.ts

@@ -0,0 +1,64 @@
+import { SpanStatusCode, trace } from "@opentelemetry/api";
+import type { Job } from "bullmq";
+
+import { getErrorMessage } from "../errors";
+
+const tracer = trace.getTracer("boringstack-api/queue");
+
+/*
+ * Wrap a BullMQ job processor in an OpenTelemetry span so queue work
+ * shows up in Tempo (and in any other trace backend wired up via the
+ * OTLP exporter). Each invocation gets a span named
+ * `queue.<name>.process` with messaging.* attributes Grafana's Tempo
+ * Explore can filter on (system, destination, message id, attempt).
+ *
+ * Use inside a worker's processJob method:
+ *
+ *   private async processJob(job: Job<X>): Promise<void> {
+ *     return withQueueSpan("email-delivery", job, async () => {
+ *       // existing body
+ *     });
+ *   }
+ *
+ * Exceptions are recorded on the span before being re-thrown so BullMQ
+ * sees the failure exactly as it did before.
+ */
+export const withQueueSpan = async <T>(
+  queueName: string,
+  job: Job,
+  handler: () => Promise<T>
+): Promise<T> =>
+  tracer.startActiveSpan(
+    `queue.${queueName}.process`,
+    {
+      attributes: {
+        "messaging.system": "bullmq",
+        "messaging.destination.name": queueName,
+        "messaging.message.id": job.id ?? "",
+        "messaging.bullmq.job.name": job.name,
+        "messaging.bullmq.job.attempt": job.attemptsMade + 1,
+      },
+    },
+    async (span) => {
+      try {
+        const result = await handler();
+
+        span.setStatus({ code: SpanStatusCode.OK });
+
+        return result;
+      } catch (error) {
+        span.setStatus({
+          code: SpanStatusCode.ERROR,
+          message: getErrorMessage(error),
+        });
+
+        if (error instanceof Error) {
+          span.recordException(error);
+        }
+
+        throw error;
+      } finally {
+        span.end();
+      }
+    }
+  );

apps/api/src/queues/account-maintenance/account-maintenance.worker.ts

@@ -1,6 +1,7 @@
 import { Worker, type Job, type WorkerOptions } from "bullmq";
 import { BULL_PREFIX, getValkeyConnectionOptions } from "../../clients/valkey";
 import { logger } from "../../config/logger";
+import { withQueueSpan } from "../../lib/tracing";
 import {
   ACCOUNT_MAINTENANCE_DEFAULTS,
   ACCOUNT_MAINTENANCE_JOB_NAME,
@@ -51,7 +52,10 @@ export class AccountMaintenanceWorker {
 
     this.worker = new Worker<IAccountMaintenanceJobData>(
       ACCOUNT_MAINTENANCE_QUEUE_NAME,
-      this.processJob.bind(this),
+      (job) =>
+        withQueueSpan(ACCOUNT_MAINTENANCE_QUEUE_NAME, job, () =>
+          this.processJob(job)
+        ),
       options
     );
 

apps/api/src/queues/email-delivery/email-delivery.worker.ts

@@ -6,6 +6,7 @@ import { notificationDelivery } from "../../clients/postgres/schema";
 import { BULL_PREFIX, getValkeyConnectionOptions } from "../../clients/valkey";
 import { logger } from "../../config/logger";
 import { getErrorMessage } from "../../lib/errors";
+import { withQueueSpan } from "../../lib/tracing";
 import { maskEmailForLogging, sendTemplateNow } from "../../lib/email";
 import { DELIVERY_STATUS } from "../../lib/notifications/notifications.constants";
 import {
@@ -26,7 +27,10 @@ export class EmailDeliveryWorker {
 
     this.worker = new Worker<IEmailDeliveryJobData>(
       EMAIL_DELIVERY_QUEUE_NAME,
-      this.processJob.bind(this),
+      (job) =>
+        withQueueSpan(EMAIL_DELIVERY_QUEUE_NAME, job, () =>
+          this.processJob(job)
+        ),
       options
     );