feat: classify and exclude bot traffic at click ingestion

package.json

@@ -37,6 +37,7 @@
   "scripts": {
     "build": "tsc",
     "dev": "tsx watch src/index.ts",
+    "backfill:bot-flags": "tsx src/scripts/backfill-bot-flags.ts",
     "migrate": "tsx src/scripts/migrate.ts",
     "test": "vitest run",
     "test:watch": "vitest",
@@ -54,6 +55,7 @@
     "dotenv": "^16.3.0",
     "fastify": "^4.24.0",
     "geoip-lite": "^1.4.7",
+    "isbot": "^5.1.44",
     "nanoid": "^3.3.11",
     "pg": "^8.11.0",
     "qrcode": "^1.5.4",
@@ -66,9 +68,9 @@
     "@semantic-release/github": "^10.0.0",
     "@semantic-release/npm": "^13.0.0",
     "@types/geoip-lite": "^1.4.4",
-    "@types/qrcode": "^1.5.6",
     "@types/node": "^20.8.0",
     "@types/pg": "^8.10.0",
+    "@types/qrcode": "^1.5.6",
     "@types/ua-parser-js": "^0.7.39",
     "@vitest/coverage-v8": "^1.0.0",
     "conventional-changelog-conventionalcommits": "^7.0.2",

src/lib/bot-detection.test.ts

@@ -0,0 +1,53 @@
+import { describe, it, expect, afterEach } from 'vitest';
+import { classifyBot, edgeBotSignal } from './bot-detection.js';
+
+const CHROME = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36';
+
+describe('classifyBot', () => {
+  it('flags known crawlers/scrapers by user-agent', () => {
+    expect(classifyBot('Googlebot/2.1 (+http://www.google.com/bot.html)', 'GET')).toEqual({ isBot: true, reason: 'ua' });
+    expect(classifyBot('facebookexternalhit/1.1', 'GET').isBot).toBe(true);
+    expect(classifyBot('curl/8.4.0', 'GET').isBot).toBe(true);
+  });
+
+  it('does not flag a normal browser', () => {
+    expect(classifyBot(CHROME, 'GET')).toEqual({ isBot: false, reason: null });
+  });
+
+  it('flags HEAD/OPTIONS as non-human, case-insensitively', () => {
+    expect(classifyBot(CHROME, 'HEAD')).toEqual({ isBot: true, reason: 'method' });
+    expect(classifyBot(CHROME, 'options')).toEqual({ isBot: true, reason: 'method' });
+  });
+
+  it('honors the edge signal with highest authority', () => {
+    // Edge wins even for a normal browser UA on a GET.
+    expect(classifyBot(CHROME, 'GET', true)).toEqual({ isBot: true, reason: 'edge' });
+    // An explicit false edge signal does not override real UA/method detection.
+    expect(classifyBot('Googlebot/2.1', 'GET', false)).toEqual({ isBot: true, reason: 'ua' });
+  });
+
+  it('handles a missing/empty user-agent', () => {
+    expect(classifyBot(undefined, 'GET')).toEqual({ isBot: false, reason: null });
+    expect(classifyBot('', 'GET').isBot).toBe(false);
+  });
+});
+
+describe('edgeBotSignal', () => {
+  afterEach(() => {
+    delete process.env.TRUST_EDGE_BOT_HEADER;
+  });
+
+  it('is ignored unless TRUST_EDGE_BOT_HEADER=true', () => {
+    expect(edgeBotSignal('1')).toBeUndefined();
+  });
+
+  it('parses the header only when trusted', () => {
+    process.env.TRUST_EDGE_BOT_HEADER = 'true';
+    expect(edgeBotSignal('1')).toBe(true);
+    expect(edgeBotSignal('true')).toBe(true);
+    expect(edgeBotSignal('TRUE')).toBe(true);
+    expect(edgeBotSignal('0')).toBe(false);
+    expect(edgeBotSignal(['1'])).toBe(true);
+    expect(edgeBotSignal(undefined)).toBeUndefined();
+  });
+});

src/lib/bot-detection.ts

@@ -0,0 +1,51 @@
+import { isbot } from 'isbot';
+
+/**
+ * Bot classification for click ingestion.
+ *
+ * A click's "is this a real human?" quality is determined by the request that
+ * created it, so it's classified here at write time and persisted on the
+ * click_events row. Downstream analytics simply read the stored flag — they
+ * never re-detect (the raw request signals aren't available at read time).
+ *
+ * Signals, in order of authority:
+ *   1. edge  — a trusted upstream (reverse proxy / CDN bot management) already
+ *              classified this request as a bot. Only honored when the caller
+ *              opts in (the header must come from a trusted proxy, not the
+ *              client). Highest confidence.
+ *   2. method — HEAD / OPTIONS requests are link probes/prefetch, never a human
+ *              opening a link.
+ *   3. ua    — the `isbot` user-agent database (crawlers, scrapers, monitors).
+ */
+export type BotReason = 'edge' | 'method' | 'ua';
+
+export interface BotClassification {
+  isBot: boolean;
+  reason: BotReason | null;
+}
+
+const NON_HUMAN_METHODS = new Set(['HEAD', 'OPTIONS']);
+
+export function classifyBot(
+  userAgent: string | undefined,
+  method: string | undefined,
+  edgeIsBot?: boolean
+): BotClassification {
+  if (edgeIsBot === true) return { isBot: true, reason: 'edge' };
+  if (method && NON_HUMAN_METHODS.has(method.toUpperCase())) return { isBot: true, reason: 'method' };
+  if (isbot(userAgent ?? '')) return { isBot: true, reason: 'ua' };
+  return { isBot: false, reason: null };
+}
+
+/**
+ * Resolve the optional edge bot signal from request headers. Only trusted when
+ * `TRUST_EDGE_BOT_HEADER=true` — otherwise a client could set the header to
+ * mark its own clicks as bots. Deployments behind a proxy that authoritatively
+ * sets (and strips client-supplied copies of) `x-lf-bot` can enable it.
+ */
+export function edgeBotSignal(headerValue: string | string[] | undefined): boolean | undefined {
+  if (process.env.TRUST_EDGE_BOT_HEADER !== 'true') return undefined;
+  const v = Array.isArray(headerValue) ? headerValue[0] : headerValue;
+  if (v === undefined) return undefined;
+  return v === '1' || v.toLowerCase() === 'true';
+}

src/lib/database.ts

@@ -108,7 +108,9 @@ export async function initializeDatabase(options: DatabaseOptions = {}) {
         utm_source VARCHAR(255),
         utm_medium VARCHAR(255),
         utm_campaign VARCHAR(255),
-        referrer TEXT
+        referrer TEXT,
+        is_bot BOOLEAN NOT NULL DEFAULT false,
+        bot_reason VARCHAR(16)
       )
     `);
 
@@ -389,6 +391,22 @@ export async function initializeDatabase(options: DatabaseOptions = {}) {
       END $$;
     `);
 
+    // Bot classification columns on click_events (SIT-298). Classified at
+    // ingestion (see lib/bot-detection.ts) and persisted so every consumer reads
+    // one consistent flag; analytics excludes is_bot rows. Backward compatible:
+    // legacy rows default to is_bot=false until the one-time backfill runs.
+    await client.query(`
+      DO $$
+      BEGIN
+        IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='click_events' AND column_name='is_bot') THEN
+          ALTER TABLE click_events ADD COLUMN is_bot BOOLEAN NOT NULL DEFAULT false;
+        END IF;
+        IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='click_events' AND column_name='bot_reason') THEN
+          ALTER TABLE click_events ADD COLUMN bot_reason VARCHAR(16);
+        END IF;
+      END $$;
+    `);
+
     // Last-click attribution columns on in_app_events (SIT-237).
     // Events (screen views + custom events) are attributed to the deep link that
     // drove them, not just the original install link. The SDK stamps each event
@@ -447,6 +465,8 @@ export async function initializeDatabase(options: DatabaseOptions = {}) {
     await client.query('CREATE INDEX IF NOT EXISTS idx_clicks_link_id ON click_events(link_id)');
     await client.query('CREATE INDEX IF NOT EXISTS idx_clicks_timestamp ON click_events(clicked_at DESC)');
     await client.query('CREATE INDEX IF NOT EXISTS idx_clicks_link_date ON click_events(link_id, clicked_at DESC)');
+    // Partial index for the common analytics filter (human clicks only, is_bot = false).
+    await client.query('CREATE INDEX IF NOT EXISTS idx_clicks_human_link_date ON click_events(link_id, clicked_at DESC) WHERE is_bot = false');
     // Indexes for deferred deep linking
     await client.query('CREATE INDEX IF NOT EXISTS idx_fingerprints_hash ON device_fingerprints(fingerprint_hash)');
     await client.query('CREATE INDEX IF NOT EXISTS idx_fingerprints_click_id ON device_fingerprints(click_id)');

src/routes/analytics.ts

@@ -19,7 +19,7 @@ export async function analyticsRoutes(fastify: FastifyInstance) {
          COUNT(DISTINCT ip_address) as unique_clicks
        FROM click_events ce
               JOIN links l ON ce.link_id = l.id
-       WHERE ce.clicked_at >= NOW() - INTERVAL '${days} days' ${userFilter}`,
+       WHERE ce.clicked_at >= NOW() - INTERVAL '${days} days' ${userFilter} AND ce.is_bot = false`,
       params
     );
 
@@ -30,7 +30,7 @@ export async function analyticsRoutes(fastify: FastifyInstance) {
          COUNT(*) as clicks
        FROM click_events ce
          JOIN links l ON ce.link_id = l.id
-       WHERE ce.clicked_at >= NOW() - INTERVAL '${days} days' ${userFilter}
+       WHERE ce.clicked_at >= NOW() - INTERVAL '${days} days' ${userFilter} AND ce.is_bot = false
        GROUP BY DATE(ce.clicked_at)
        ORDER BY date`,
       params
@@ -44,7 +44,7 @@ export async function analyticsRoutes(fastify: FastifyInstance) {
          COUNT(*) as clicks
        FROM click_events ce
               JOIN links l ON ce.link_id = l.id
-       WHERE ce.clicked_at >= NOW() - INTERVAL '${days} days' ${userFilter}
+       WHERE ce.clicked_at >= NOW() - INTERVAL '${days} days' ${userFilter} AND ce.is_bot = false
        GROUP BY ce.country_code, ce.country_name
        ORDER BY clicks DESC`,
       params
@@ -57,7 +57,7 @@ export async function analyticsRoutes(fastify: FastifyInstance) {
          COUNT(*) as clicks
        FROM click_events ce
               JOIN links l ON ce.link_id = l.id
-       WHERE ce.clicked_at >= NOW() - INTERVAL '${days} days' ${userFilter}
+       WHERE ce.clicked_at >= NOW() - INTERVAL '${days} days' ${userFilter} AND ce.is_bot = false
        GROUP BY ce.device_type
        ORDER BY clicks DESC`,
       params
@@ -70,7 +70,7 @@ export async function analyticsRoutes(fastify: FastifyInstance) {
          COUNT(*) as clicks
        FROM click_events ce
               JOIN links l ON ce.link_id = l.id
-       WHERE ce.clicked_at >= NOW() - INTERVAL '${days} days' ${userFilter}
+       WHERE ce.clicked_at >= NOW() - INTERVAL '${days} days' ${userFilter} AND ce.is_bot = false
        GROUP BY ce.platform
        ORDER BY clicks DESC`,
       params
@@ -88,6 +88,7 @@ export async function analyticsRoutes(fastify: FastifyInstance) {
        FROM links l
               LEFT JOIN click_events ce ON l.id = ce.link_id
          AND ce.clicked_at >= NOW() - INTERVAL '${days} days'
+         AND ce.is_bot = false
        ${userFilterWhere}
        GROUP BY l.id
        ORDER BY total_clicks DESC
@@ -158,7 +159,7 @@ export async function analyticsRoutes(fastify: FastifyInstance) {
          COUNT(*) as total_clicks,
          COUNT(DISTINCT ip_address) as unique_clicks
        FROM click_events
-       WHERE link_id = $1 AND clicked_at >= NOW() - INTERVAL '${days} days'`,
+       WHERE link_id = $1 AND clicked_at >= NOW() - INTERVAL '${days} days' AND is_bot = false`,
       [linkId]
     );
 
@@ -167,7 +168,7 @@ export async function analyticsRoutes(fastify: FastifyInstance) {
          DATE(clicked_at) as date,
          COUNT(*) as clicks
        FROM click_events
-       WHERE link_id = $1 AND clicked_at >= NOW() - INTERVAL '${days} days'
+       WHERE link_id = $1 AND clicked_at >= NOW() - INTERVAL '${days} days' AND is_bot = false
        GROUP BY DATE(clicked_at)
        ORDER BY date`,
       [linkId]
@@ -179,7 +180,7 @@ export async function analyticsRoutes(fastify: FastifyInstance) {
          COALESCE(country_name, country_code, 'Unknown') as country,
          COUNT(*) as clicks
        FROM click_events
-       WHERE link_id = $1 AND clicked_at >= NOW() - INTERVAL '${days} days'
+       WHERE link_id = $1 AND clicked_at >= NOW() - INTERVAL '${days} days' AND is_bot = false
        GROUP BY country_code, country_name
        ORDER BY clicks DESC`,
       [linkId]
@@ -190,7 +191,7 @@ export async function analyticsRoutes(fastify: FastifyInstance) {
          COALESCE(device_type, 'Unknown') as device,
          COUNT(*) as clicks
        FROM click_events
-       WHERE link_id = $1 AND clicked_at >= NOW() - INTERVAL '${days} days'
+       WHERE link_id = $1 AND clicked_at >= NOW() - INTERVAL '${days} days' AND is_bot = false
        GROUP BY device_type
        ORDER BY clicks DESC`,
       [linkId]
@@ -201,7 +202,7 @@ export async function analyticsRoutes(fastify: FastifyInstance) {
          COALESCE(platform, 'Unknown') as platform,
          COUNT(*) as clicks
        FROM click_events
-       WHERE link_id = $1 AND clicked_at >= NOW() - INTERVAL '${days} days'
+       WHERE link_id = $1 AND clicked_at >= NOW() - INTERVAL '${days} days' AND is_bot = false
        GROUP BY platform
        ORDER BY clicks DESC`,
       [linkId]

src/routes/redirect.ts

@@ -5,6 +5,7 @@ import { getClientIp } from '../lib/client-ip.js';
 import { parseUserAgent, getLocationFromIP, buildRedirectUrl, detectDevice } from '../lib/utils.js';
 import { storeFingerprintForClick, type FingerprintData } from '../lib/fingerprint.js';
 import { emitClickEvent } from '../lib/event-emitter.js';
+import { classifyBot, edgeBotSignal } from '../lib/bot-detection.js';
 
 /**
  * Detect iOS in-app browsers where Universal Links don't fire.
@@ -266,6 +267,15 @@ export async function redirectRoutes(fastify: FastifyInstance) {
         const { platform, platformVersion } = parseUserAgent(userAgent);
         const { countryCode, countryName, region, city, latitude, longitude, timezone } = getLocationFromIP(ip);
 
+        // Classify bots at ingestion (SIT-298) — persisted on the row so
+        // analytics reads a consistent flag instead of re-detecting from the
+        // stored user-agent.
+        const { isBot, reason: botReason } = classifyBot(
+          userAgent,
+          request.method,
+          edgeBotSignal(request.headers['x-lf-bot'])
+        );
+
         // Extract UTM parameters from query string
         const query = request.query as Record<string, string | undefined>;
         const utmSource = query?.utm_source;
@@ -284,8 +294,8 @@ export async function redirectRoutes(fastify: FastifyInstance) {
           `INSERT INTO click_events (
             id, link_id, ip_address, user_agent, device_type, platform,
             country_code, country_name, region, city, latitude, longitude, timezone,
-            utm_source, utm_medium, utm_campaign, referrer
-          ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)`,
+            utm_source, utm_medium, utm_campaign, referrer, is_bot, bot_reason
+          ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19)`,
           [
             clickId,
             link.id,
@@ -304,6 +314,8 @@ export async function redirectRoutes(fastify: FastifyInstance) {
             utmMedium,
             utmCampaign,
             referrer,
+            isBot,
+            botReason,
           ]
         );