Add isolated Vercel Speed Insights drain receiver

apps/observability-drain/README.md

@@ -0,0 +1,66 @@
+# MapAble Observability Drain
+
+A small, isolated Vercel project for receiving **Vercel Speed Insights** Drain events without putting drain ingestion on the MapAble user-facing request path.
+
+## Endpoints
+
+- `GET /health` — readiness check; does not expose secrets.
+- `POST /v1/vercel/speed-insights` — accepts Vercel Speed Insights Drain JSON or NDJSON batches.
+
+## Security and privacy defaults
+
+- Verifies `x-vercel-signature` against `VERCEL_DRAIN_SECRET` using HMAC-SHA1 over the raw request body.
+- Accepts only `vercel.speed_insights.v1` events and known Core Web Vital metric types.
+- Never logs the incoming payload.
+- Always removes `deviceId`, `city`, and `attribution`.
+- Removes raw `path` by default. Route patterns such as `/participant/[id]` are retained.
+- Strips path/query data from `origin`, retaining only the origin.
+- Rejects oversized bodies and excessive event counts.
+- Provides no CORS headers; this endpoint is intended for server-to-server Vercel delivery only.
+
+## Environment variables
+
+| Variable | Required | Purpose |
+| --- | --- | --- |
+| `VERCEL_DRAIN_SECRET` | Yes | Vercel Drain signing secret. Store as a Vercel secret; never commit it. |
+| `DRAIN_FORWARD_URL` | No | Independent HTTPS destination for sanitized batches. Without it, batches are validated and summarized to structured runtime logs only. |
+| `DRAIN_FORWARD_AUTHORIZATION` | No | Full `Authorization` header value for the independent sink. Store as a Vercel secret. |
+| `MAPABLE_DRAIN_INCLUDE_PATH` | No | Set exactly to `true` to retain query-free raw paths. Default is false. |
+| `DRAIN_MAX_BODY_BYTES` | No | Maximum request size. Default 2,000,000 bytes. |
+| `DRAIN_FORWARD_TIMEOUT_MS` | No | Forwarding timeout. Default 5,000 ms. |
+
+## Local verification
+
+```bash
+npm test
+npm start
+curl http://localhost:3000/health
+```
+
+## Vercel project setup
+
+Create a separate Vercel project from the canonical repository with:
+
+- Repository: `ausdisau/MapAble`
+- Root Directory: `apps/observability-drain`
+- Framework preset: Other / Node.js
+- Production environment: configure `VERCEL_DRAIN_SECRET`
+
+Then create a Vercel Drain using the deployed endpoint:
+
+```text
+https://<observability-project>.vercel.app/v1/vercel/speed-insights
+```
+
+Recommended initial Drain configuration:
+
+- Schema/data type: Speed Insights only
+- Environment: production
+- Encoding: NDJSON or JSON
+- Sampling: 100% initially if traffic/cost permits
+
+Do not add Logs, Web Analytics, AI Gateway, or Trace schemas to this receiver until explicit schema handlers and privacy review are added.
+
+## Forwarding model
+
+For durable storage and alerting, set `DRAIN_FORWARD_URL` to an **independent** observability endpoint. The receiver forwards only the sanitized batch. Keeping the durable sink independent prevents the MapAble application and its observability history from failing together.

apps/observability-drain/lib/drain.mjs

@@ -0,0 +1,88 @@
+import { createHmac, timingSafeEqual } from 'node:crypto';
+
+export const SPEED_INSIGHTS_SCHEMA = 'vercel.speed_insights.v1';
+export const ALLOWED_METRIC_TYPES = new Set(['CLS', 'LCP', 'FID', 'FCP', 'TTFB', 'INP']);
+
+export function verifyDrainSignature(rawBody, signature, secret) {
+  if (!rawBody || !signature || !secret) return false;
+  const expected = createHmac('sha1', secret).update(rawBody).digest('hex');
+  if (expected.length !== signature.length) return false;
+  return timingSafeEqual(Buffer.from(expected), Buffer.from(signature));
+}
+
+export function parseDrainPayload(rawBody, contentType = '') {
+  const trimmed = rawBody.trim();
+  if (!trimmed) return [];
+
+  const looksNdjson = contentType.includes('ndjson') || (!trimmed.startsWith('[') && trimmed.includes('\n'));
+  if (looksNdjson) {
+    return trimmed
+      .split(/\r?\n/)
+      .filter(Boolean)
+      .map((line) => JSON.parse(line));
+  }
+
+  const parsed = JSON.parse(trimmed);
+  return Array.isArray(parsed) ? parsed : [parsed];
+}
+
+export function sanitizeSpeedInsight(event, { includePath = false } = {}) {
+  if (!event || typeof event !== 'object') throw new Error('invalid_event');
+  if (event.schema !== SPEED_INSIGHTS_SCHEMA) throw new Error('unsupported_schema');
+  if (!ALLOWED_METRIC_TYPES.has(event.metricType)) throw new Error('unsupported_metric_type');
+  if (typeof event.value !== 'number' || !Number.isFinite(event.value)) throw new Error('invalid_metric_value');
+  if (typeof event.timestamp !== 'string' || Number.isNaN(Date.parse(event.timestamp))) throw new Error('invalid_timestamp');
+  if (typeof event.projectId !== 'string' || event.projectId.length < 1) throw new Error('invalid_project_id');
+
+  const sanitized = {
+    schema: SPEED_INSIGHTS_SCHEMA,
+    timestamp: event.timestamp,
+    projectId: event.projectId,
+    metricType: event.metricType,
+    value: event.value,
+    origin: safeOrigin(event.origin),
+    route: safeString(event.route, 500),
+    country: safeString(event.country, 8),
+    region: safeString(event.region, 32),
+    osName: safeString(event.osName, 80),
+    osVersion: safeString(event.osVersion, 80),
+    clientName: safeString(event.clientName, 80),
+    clientType: safeString(event.clientType, 40),
+    clientVersion: safeString(event.clientVersion, 80),
+    deviceType: safeString(event.deviceType, 40),
+    deviceBrand: safeString(event.deviceBrand, 80),
+    connectionSpeed: safeString(event.connectionSpeed, 40),
+    browserEngine: safeString(event.browserEngine, 80),
+    browserEngineVersion: safeString(event.browserEngineVersion, 80),
+    scriptVersion: safeString(event.scriptVersion, 80),
+    sdkVersion: safeString(event.sdkVersion, 80),
+    sdkName: safeString(event.sdkName, 120),
+    vercelEnvironment: safeString(event.vercelEnvironment, 40),
+    deploymentId: safeString(event.deploymentId, 160),
+  };
+
+  if (includePath) sanitized.path = safePath(event.path);
+
+  return Object.fromEntries(Object.entries(sanitized).filter(([, value]) => value !== undefined));
+}
+
+function safeString(value, maxLength) {
+  if (typeof value !== 'string' || value.length === 0) return undefined;
+  return value.slice(0, maxLength);
+}
+
+function safeOrigin(value) {
+  if (typeof value !== 'string' || value.length === 0) return undefined;
+  try {
+    const url = new URL(value);
+    return url.origin;
+  } catch {
+    return undefined;
+  }
+}
+
+function safePath(value) {
+  if (typeof value !== 'string' || !value.startsWith('/')) return undefined;
+  const pathOnly = value.split(/[?#]/, 1)[0];
+  return pathOnly.slice(0, 500);
+}

apps/observability-drain/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "@mapable/observability-drain",
+  "version": "0.1.0",
+  "private": true,
+  "type": "module",
+  "description": "Isolated receiver for sanitized Vercel Speed Insights drain events.",
+  "engines": {
+    "node": "24.x"
+  },
+  "scripts": {
+    "dev": "node --watch server.mjs",
+    "start": "node server.mjs",
+    "test": "node --test"
+  }
+}

apps/observability-drain/server.mjs

@@ -0,0 +1,148 @@
+import { createServer } from 'node:http';
+import {
+  parseDrainPayload,
+  sanitizeSpeedInsight,
+  verifyDrainSignature,
+} from './lib/drain.mjs';
+
+const PORT = Number(process.env.PORT ?? 3000);
+const MAX_BODY_BYTES = Number(process.env.DRAIN_MAX_BODY_BYTES ?? 2_000_000);
+const FORWARD_TIMEOUT_MS = Number(process.env.DRAIN_FORWARD_TIMEOUT_MS ?? 5_000);
+
+const server = createServer(async (request, response) => {
+  const url = new URL(request.url ?? '/', `http://${request.headers.host ?? 'localhost'}`);
+
+  if (request.method === 'GET' && url.pathname === '/health') {
+    return json(response, 200, {
+      status: 'ok',
+      service: 'mapable-observability-drain',
+      forwardingConfigured: Boolean(process.env.DRAIN_FORWARD_URL),
+    });
+  }
+
+  if (request.method !== 'POST' || url.pathname !== '/v1/vercel/speed-insights') {
+    return json(response, 404, { error: 'not_found' });
+  }
+
+  const secret = process.env.VERCEL_DRAIN_SECRET;
+  if (!secret) {
+    console.error(JSON.stringify({ level: 'error', msg: 'drain_secret_missing' }));
+    return json(response, 503, { error: 'receiver_not_configured' });
+  }
+
+  let rawBody;
+  try {
+    rawBody = await readRawBody(request, MAX_BODY_BYTES);
+  } catch (error) {
+    const code = error instanceof Error ? error.message : 'invalid_body';
+    return json(response, code === 'body_too_large' ? 413 : 400, { error: code });
+  }
+
+  const signature = headerValue(request.headers['x-vercel-signature']);
+  if (!verifyDrainSignature(rawBody, signature, secret)) {
+    console.warn(JSON.stringify({ level: 'warning', msg: 'drain_signature_rejected' }));
+    return json(response, 401, { error: 'invalid_signature' });
+  }
+
+  let events;
+  try {
+    const parsed = parseDrainPayload(rawBody, headerValue(request.headers['content-type']));
+    if (parsed.length === 0) return json(response, 400, { error: 'empty_batch' });
+    if (parsed.length > 10_000) return json(response, 413, { error: 'too_many_events' });
+    const includePath = process.env.MAPABLE_DRAIN_INCLUDE_PATH === 'true';
+    events = parsed.map((event) => sanitizeSpeedInsight(event, { includePath }));
+  } catch (error) {
+    const reason = error instanceof Error ? error.message : 'invalid_payload';
+    console.warn(JSON.stringify({ level: 'warning', msg: 'drain_payload_rejected', reason }));
+    return json(response, 400, { error: reason });
+  }
+
+  const summary = summarize(events);
+  console.log(JSON.stringify({
+    level: 'info',
+    msg: 'speed_insights_batch_accepted',
+    eventCount: events.length,
+    metricTypes: summary.metricTypes,
+    projects: summary.projects,
+    environments: summary.environments,
+  }));
+
+  if (process.env.DRAIN_FORWARD_URL) {
+    try {
+      const forwarded = await forwardBatch(events);
+      if (!forwarded.ok) {
+        console.error(JSON.stringify({
+          level: 'error',
+          msg: 'drain_forward_failed',
+          status: forwarded.status,
+        }));
+        return json(response, 502, { error: 'forward_failed' });
+      }
+    } catch (error) {
+      console.error(JSON.stringify({
+        level: 'error',
+        msg: 'drain_forward_exception',
+        error: error instanceof Error ? error.name : 'unknown_error',
+      }));
+      return json(response, 502, { error: 'forward_failed' });
+    }
+  }
+
+  return json(response, 202, {
+    accepted: events.length,
+    forwarded: Boolean(process.env.DRAIN_FORWARD_URL),
+  });
+});
+
+server.listen(PORT, () => {
+  console.log(JSON.stringify({ level: 'info', msg: 'drain_receiver_started', port: PORT }));
+});
+
+async function readRawBody(request, maxBytes) {
+  const declared = Number(headerValue(request.headers['content-length']) || 0);
+  if (declared > maxBytes) throw new Error('body_too_large');
+
+  const chunks = [];
+  let total = 0;
+  for await (const chunk of request) {
+    total += chunk.length;
+    if (total > maxBytes) throw new Error('body_too_large');
+    chunks.push(chunk);
+  }
+  return Buffer.concat(chunks).toString('utf8');
+}
+
+async function forwardBatch(events) {
+  const headers = { 'content-type': 'application/json' };
+  if (process.env.DRAIN_FORWARD_AUTHORIZATION) {
+    headers.authorization = process.env.DRAIN_FORWARD_AUTHORIZATION;
+  }
+
+  return fetch(process.env.DRAIN_FORWARD_URL, {
+    method: 'POST',
+    headers,
+    body: JSON.stringify(events),
+    signal: AbortSignal.timeout(FORWARD_TIMEOUT_MS),
+  });
+}
+
+function summarize(events) {
+  return {
+    metricTypes: [...new Set(events.map((event) => event.metricType))].sort(),
+    projects: [...new Set(events.map((event) => event.projectId))].slice(0, 20),
+    environments: [...new Set(events.map((event) => event.vercelEnvironment).filter(Boolean))],
+  };
+}
+
+function headerValue(value) {
+  return Array.isArray(value) ? value[0] ?? '' : value ?? '';
+}
+
+function json(response, status, body) {
+  response.writeHead(status, {
+    'content-type': 'application/json; charset=utf-8',
+    'cache-control': 'no-store',
+    'x-content-type-options': 'nosniff',
+  });
+  response.end(JSON.stringify(body));
+}

apps/observability-drain/test/drain.test.mjs

@@ -0,0 +1,57 @@
+import assert from 'node:assert/strict';
+import { createHmac } from 'node:crypto';
+import test from 'node:test';
+import {
+  parseDrainPayload,
+  sanitizeSpeedInsight,
+  verifyDrainSignature,
+} from '../lib/drain.mjs';
+
+const baseEvent = {
+  schema: 'vercel.speed_insights.v1',
+  timestamp: '2026-08-18T00:00:00.000Z',
+  projectId: 'prj_test',
+  deviceId: 12345,
+  metricType: 'LCP',
+  value: 2.1,
+  origin: 'https://mapable.com.au/path?secret=1',
+  path: '/participant/abc?token=nope',
+  route: '/participant/[id]',
+  city: 'Sydney',
+  country: 'AU',
+  attribution: 'sensitive-ish-data',
+};
+
+test('verifies the Vercel drain HMAC over the raw body', () => {
+  const raw = JSON.stringify([baseEvent]);
+  const secret = 'test-secret';
+  const signature = createHmac('sha1', secret).update(raw).digest('hex');
+  assert.equal(verifyDrainSignature(raw, signature, secret), true);
+  assert.equal(verifyDrainSignature(raw, 'bad', secret), false);
+});
+
+test('parses JSON arrays and NDJSON', () => {
+  assert.equal(parseDrainPayload(JSON.stringify([baseEvent]), 'application/json').length, 1);
+  const ndjson = `${JSON.stringify(baseEvent)}\n${JSON.stringify(baseEvent)}`;
+  assert.equal(parseDrainPayload(ndjson, 'application/x-ndjson').length, 2);
+});
+
+test('drops device id, city, attribution and path by default', () => {
+  const sanitized = sanitizeSpeedInsight(baseEvent);
+  assert.equal('deviceId' in sanitized, false);
+  assert.equal('city' in sanitized, false);
+  assert.equal('attribution' in sanitized, false);
+  assert.equal('path' in sanitized, false);
+  assert.equal(sanitized.route, '/participant/[id]');
+  assert.equal(sanitized.origin, 'https://mapable.com.au');
+});
+
+test('can include a query-free path only when explicitly enabled', () => {
+  const sanitized = sanitizeSpeedInsight(baseEvent, { includePath: true });
+  assert.equal(sanitized.path, '/participant/abc');
+});
+
+test('rejects unsupported schemas and invalid metric values', () => {
+  assert.throws(() => sanitizeSpeedInsight({ ...baseEvent, schema: 'other' }), /unsupported_schema/);
+  assert.throws(() => sanitizeSpeedInsight({ ...baseEvent, value: Number.NaN }), /invalid_metric_value/);
+});