From 43ddb9997755d11dea07f2573ca3ef842e2a98b8 Mon Sep 17 00:00:00 2001 From: Brandon Estrella Date: Tue, 30 Jun 2026 21:29:55 -0700 Subject: [PATCH] feat: classify and exclude bot traffic at click ingestion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bot detection belongs at ingestion, where the full request context exists and the classification can be persisted once for every consumer to read — rather than re-deriving it from a stored user-agent at read time. - click_events: add is_bot (NOT NULL DEFAULT false) + bot_reason columns, with a partial index for the common human-only analytics filter. Idempotent ALTER for existing databases. - lib/bot-detection.ts: classify each click via the isbot user-agent database, the HTTP method (HEAD/OPTIONS are probes, never human), and an optional trusted upstream bot signal (honored only when TRUST_EDGE_BOT_HEADER=true). Keep-and- flag, not drop — the row is recorded and flagged, preserving an audit trail. - Wire classification into both click-recording paths (redirect + SDK resolve). - Analytics: exclude is_bot rows from all click counts. - scripts/backfill-bot-flags.ts: one-time, keyset-paginated backfill that flags historical rows from their stored user-agent (SIT-298). Verified: 139 unit tests pass (incl. classifier cases) and an integration check confirms the schema applies, flags persist, and analytics excludes bots. --- package-lock.json | 19 ++++++++++- package.json | 4 ++- src/lib/bot-detection.test.ts | 53 +++++++++++++++++++++++++++++ src/lib/bot-detection.ts | 51 ++++++++++++++++++++++++++++ src/lib/database.ts | 22 ++++++++++++- src/routes/analytics.ts | 21 ++++++------ src/routes/redirect.ts | 16 +++++++-- src/routes/sdk.ts | 14 ++++++-- src/scripts/backfill-bot-flags.ts | 55 +++++++++++++++++++++++++++++++ 9 files changed, 238 insertions(+), 17 deletions(-) create mode 100644 src/lib/bot-detection.test.ts create mode 100644 src/lib/bot-detection.ts create mode 100644 src/scripts/backfill-bot-flags.ts diff --git a/package-lock.json b/package-lock.json index 2c889d0..a8eb482 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,10 +15,10 @@ "@onamfc/api-mocker": "^1.0.4", "@onamfc/developer-log": "^1.2.3", "@onamfc/pkg-inspect": "^1.0.2", - "@types/qrcode": "^1.5.6", "dotenv": "^16.3.0", "fastify": "^4.24.0", "geoip-lite": "^1.4.7", + "isbot": "^5.1.44", "nanoid": "^3.3.11", "pg": "^8.11.0", "qrcode": "^1.5.4", @@ -33,6 +33,7 @@ "@types/geoip-lite": "^1.4.4", "@types/node": "^20.8.0", "@types/pg": "^8.10.0", + "@types/qrcode": "^1.5.6", "@types/ua-parser-js": "^0.7.39", "@vitest/coverage-v8": "^1.0.0", "conventional-changelog-conventionalcommits": "^7.0.2", @@ -40,6 +41,10 @@ "tsx": "^4.0.0", "typescript": "^5.2.0", "vitest": "^1.0.0" + }, + "engines": { + "node": ">=18", + "npm": ">=9" } }, "node_modules/@actions/core": { @@ -2094,6 +2099,7 @@ "version": "20.19.24", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.24.tgz", "integrity": "sha512-FE5u0ezmi6y9OZEzlJfg37mqqf6ZDSF2V/NLjUyGrR9uTZ7Sb9F7bLNZ03S4XVUNRWGA7Ck4c1kK+YnuWjl+DA==", + "dev": true, "license": "MIT", "dependencies": { "undici-types": "~6.21.0" @@ -2122,6 +2128,7 @@ "version": "1.5.6", "resolved": "https://registry.npmjs.org/@types/qrcode/-/qrcode-1.5.6.tgz", "integrity": "sha512-te7NQcV2BOvdj2b1hCAHzAoMNuj65kNBMz0KBaxM6c3VGBOhU0dURQKOtH8CFNI/dsKkwlv32p26qYQTWoB5bw==", + "dev": true, "license": "MIT", "dependencies": { "@types/node": "*" @@ -4236,6 +4243,15 @@ "dev": true, "license": "MIT" }, + "node_modules/isbot": { + "version": "5.1.44", + "resolved": "https://registry.npmjs.org/isbot/-/isbot-5.1.44.tgz", + "integrity": "sha512-PGEHtwMnKbZpeSEXW2Utx+/JWed7dp6DiH0WWg33vGSDA7RUvpUeJSVlLrVkQ1RCpvDOUc/eH9ql7VsdbBZ8pA==", + "license": "Unlicense", + "engines": { + "node": ">=18" + } + }, "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -9793,6 +9809,7 @@ "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, "license": "MIT" }, "node_modules/unicode-emoji-modifier-base": { diff --git a/package.json b/package.json index b4edfed..9e3742d 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "scripts": { "build": "tsc", "dev": "tsx watch src/index.ts", + "backfill:bot-flags": "tsx src/scripts/backfill-bot-flags.ts", "migrate": "tsx src/scripts/migrate.ts", "test": "vitest run", "test:watch": "vitest", @@ -54,6 +55,7 @@ "dotenv": "^16.3.0", "fastify": "^4.24.0", "geoip-lite": "^1.4.7", + "isbot": "^5.1.44", "nanoid": "^3.3.11", "pg": "^8.11.0", "qrcode": "^1.5.4", @@ -66,9 +68,9 @@ "@semantic-release/github": "^10.0.0", "@semantic-release/npm": "^13.0.0", "@types/geoip-lite": "^1.4.4", - "@types/qrcode": "^1.5.6", "@types/node": "^20.8.0", "@types/pg": "^8.10.0", + "@types/qrcode": "^1.5.6", "@types/ua-parser-js": "^0.7.39", "@vitest/coverage-v8": "^1.0.0", "conventional-changelog-conventionalcommits": "^7.0.2", diff --git a/src/lib/bot-detection.test.ts b/src/lib/bot-detection.test.ts new file mode 100644 index 0000000..1db283f --- /dev/null +++ b/src/lib/bot-detection.test.ts @@ -0,0 +1,53 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import { classifyBot, edgeBotSignal } from './bot-detection.js'; + +const CHROME = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36'; + +describe('classifyBot', () => { + it('flags known crawlers/scrapers by user-agent', () => { + expect(classifyBot('Googlebot/2.1 (+http://www.google.com/bot.html)', 'GET')).toEqual({ isBot: true, reason: 'ua' }); + expect(classifyBot('facebookexternalhit/1.1', 'GET').isBot).toBe(true); + expect(classifyBot('curl/8.4.0', 'GET').isBot).toBe(true); + }); + + it('does not flag a normal browser', () => { + expect(classifyBot(CHROME, 'GET')).toEqual({ isBot: false, reason: null }); + }); + + it('flags HEAD/OPTIONS as non-human, case-insensitively', () => { + expect(classifyBot(CHROME, 'HEAD')).toEqual({ isBot: true, reason: 'method' }); + expect(classifyBot(CHROME, 'options')).toEqual({ isBot: true, reason: 'method' }); + }); + + it('honors the edge signal with highest authority', () => { + // Edge wins even for a normal browser UA on a GET. + expect(classifyBot(CHROME, 'GET', true)).toEqual({ isBot: true, reason: 'edge' }); + // An explicit false edge signal does not override real UA/method detection. + expect(classifyBot('Googlebot/2.1', 'GET', false)).toEqual({ isBot: true, reason: 'ua' }); + }); + + it('handles a missing/empty user-agent', () => { + expect(classifyBot(undefined, 'GET')).toEqual({ isBot: false, reason: null }); + expect(classifyBot('', 'GET').isBot).toBe(false); + }); +}); + +describe('edgeBotSignal', () => { + afterEach(() => { + delete process.env.TRUST_EDGE_BOT_HEADER; + }); + + it('is ignored unless TRUST_EDGE_BOT_HEADER=true', () => { + expect(edgeBotSignal('1')).toBeUndefined(); + }); + + it('parses the header only when trusted', () => { + process.env.TRUST_EDGE_BOT_HEADER = 'true'; + expect(edgeBotSignal('1')).toBe(true); + expect(edgeBotSignal('true')).toBe(true); + expect(edgeBotSignal('TRUE')).toBe(true); + expect(edgeBotSignal('0')).toBe(false); + expect(edgeBotSignal(['1'])).toBe(true); + expect(edgeBotSignal(undefined)).toBeUndefined(); + }); +}); diff --git a/src/lib/bot-detection.ts b/src/lib/bot-detection.ts new file mode 100644 index 0000000..21371dc --- /dev/null +++ b/src/lib/bot-detection.ts @@ -0,0 +1,51 @@ +import { isbot } from 'isbot'; + +/** + * Bot classification for click ingestion. + * + * A click's "is this a real human?" quality is determined by the request that + * created it, so it's classified here at write time and persisted on the + * click_events row. Downstream analytics simply read the stored flag — they + * never re-detect (the raw request signals aren't available at read time). + * + * Signals, in order of authority: + * 1. edge — a trusted upstream (reverse proxy / CDN bot management) already + * classified this request as a bot. Only honored when the caller + * opts in (the header must come from a trusted proxy, not the + * client). Highest confidence. + * 2. method — HEAD / OPTIONS requests are link probes/prefetch, never a human + * opening a link. + * 3. ua — the `isbot` user-agent database (crawlers, scrapers, monitors). + */ +export type BotReason = 'edge' | 'method' | 'ua'; + +export interface BotClassification { + isBot: boolean; + reason: BotReason | null; +} + +const NON_HUMAN_METHODS = new Set(['HEAD', 'OPTIONS']); + +export function classifyBot( + userAgent: string | undefined, + method: string | undefined, + edgeIsBot?: boolean +): BotClassification { + if (edgeIsBot === true) return { isBot: true, reason: 'edge' }; + if (method && NON_HUMAN_METHODS.has(method.toUpperCase())) return { isBot: true, reason: 'method' }; + if (isbot(userAgent ?? '')) return { isBot: true, reason: 'ua' }; + return { isBot: false, reason: null }; +} + +/** + * Resolve the optional edge bot signal from request headers. Only trusted when + * `TRUST_EDGE_BOT_HEADER=true` — otherwise a client could set the header to + * mark its own clicks as bots. Deployments behind a proxy that authoritatively + * sets (and strips client-supplied copies of) `x-lf-bot` can enable it. + */ +export function edgeBotSignal(headerValue: string | string[] | undefined): boolean | undefined { + if (process.env.TRUST_EDGE_BOT_HEADER !== 'true') return undefined; + const v = Array.isArray(headerValue) ? headerValue[0] : headerValue; + if (v === undefined) return undefined; + return v === '1' || v.toLowerCase() === 'true'; +} diff --git a/src/lib/database.ts b/src/lib/database.ts index f20cc0a..f1531c0 100644 --- a/src/lib/database.ts +++ b/src/lib/database.ts @@ -108,7 +108,9 @@ export async function initializeDatabase(options: DatabaseOptions = {}) { utm_source VARCHAR(255), utm_medium VARCHAR(255), utm_campaign VARCHAR(255), - referrer TEXT + referrer TEXT, + is_bot BOOLEAN NOT NULL DEFAULT false, + bot_reason VARCHAR(16) ) `); @@ -389,6 +391,22 @@ export async function initializeDatabase(options: DatabaseOptions = {}) { END $$; `); + // Bot classification columns on click_events (SIT-298). Classified at + // ingestion (see lib/bot-detection.ts) and persisted so every consumer reads + // one consistent flag; analytics excludes is_bot rows. Backward compatible: + // legacy rows default to is_bot=false until the one-time backfill runs. + await client.query(` + DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='click_events' AND column_name='is_bot') THEN + ALTER TABLE click_events ADD COLUMN is_bot BOOLEAN NOT NULL DEFAULT false; + END IF; + IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='click_events' AND column_name='bot_reason') THEN + ALTER TABLE click_events ADD COLUMN bot_reason VARCHAR(16); + END IF; + END $$; + `); + // Last-click attribution columns on in_app_events (SIT-237). // Events (screen views + custom events) are attributed to the deep link that // drove them, not just the original install link. The SDK stamps each event @@ -447,6 +465,8 @@ export async function initializeDatabase(options: DatabaseOptions = {}) { await client.query('CREATE INDEX IF NOT EXISTS idx_clicks_link_id ON click_events(link_id)'); await client.query('CREATE INDEX IF NOT EXISTS idx_clicks_timestamp ON click_events(clicked_at DESC)'); await client.query('CREATE INDEX IF NOT EXISTS idx_clicks_link_date ON click_events(link_id, clicked_at DESC)'); + // Partial index for the common analytics filter (human clicks only, is_bot = false). + await client.query('CREATE INDEX IF NOT EXISTS idx_clicks_human_link_date ON click_events(link_id, clicked_at DESC) WHERE is_bot = false'); // Indexes for deferred deep linking await client.query('CREATE INDEX IF NOT EXISTS idx_fingerprints_hash ON device_fingerprints(fingerprint_hash)'); await client.query('CREATE INDEX IF NOT EXISTS idx_fingerprints_click_id ON device_fingerprints(click_id)'); diff --git a/src/routes/analytics.ts b/src/routes/analytics.ts index e70ba30..49ac486 100644 --- a/src/routes/analytics.ts +++ b/src/routes/analytics.ts @@ -19,7 +19,7 @@ export async function analyticsRoutes(fastify: FastifyInstance) { COUNT(DISTINCT ip_address) as unique_clicks FROM click_events ce JOIN links l ON ce.link_id = l.id - WHERE ce.clicked_at >= NOW() - INTERVAL '${days} days' ${userFilter}`, + WHERE ce.clicked_at >= NOW() - INTERVAL '${days} days' ${userFilter} AND ce.is_bot = false`, params ); @@ -30,7 +30,7 @@ export async function analyticsRoutes(fastify: FastifyInstance) { COUNT(*) as clicks FROM click_events ce JOIN links l ON ce.link_id = l.id - WHERE ce.clicked_at >= NOW() - INTERVAL '${days} days' ${userFilter} + WHERE ce.clicked_at >= NOW() - INTERVAL '${days} days' ${userFilter} AND ce.is_bot = false GROUP BY DATE(ce.clicked_at) ORDER BY date`, params @@ -44,7 +44,7 @@ export async function analyticsRoutes(fastify: FastifyInstance) { COUNT(*) as clicks FROM click_events ce JOIN links l ON ce.link_id = l.id - WHERE ce.clicked_at >= NOW() - INTERVAL '${days} days' ${userFilter} + WHERE ce.clicked_at >= NOW() - INTERVAL '${days} days' ${userFilter} AND ce.is_bot = false GROUP BY ce.country_code, ce.country_name ORDER BY clicks DESC`, params @@ -57,7 +57,7 @@ export async function analyticsRoutes(fastify: FastifyInstance) { COUNT(*) as clicks FROM click_events ce JOIN links l ON ce.link_id = l.id - WHERE ce.clicked_at >= NOW() - INTERVAL '${days} days' ${userFilter} + WHERE ce.clicked_at >= NOW() - INTERVAL '${days} days' ${userFilter} AND ce.is_bot = false GROUP BY ce.device_type ORDER BY clicks DESC`, params @@ -70,7 +70,7 @@ export async function analyticsRoutes(fastify: FastifyInstance) { COUNT(*) as clicks FROM click_events ce JOIN links l ON ce.link_id = l.id - WHERE ce.clicked_at >= NOW() - INTERVAL '${days} days' ${userFilter} + WHERE ce.clicked_at >= NOW() - INTERVAL '${days} days' ${userFilter} AND ce.is_bot = false GROUP BY ce.platform ORDER BY clicks DESC`, params @@ -88,6 +88,7 @@ export async function analyticsRoutes(fastify: FastifyInstance) { FROM links l LEFT JOIN click_events ce ON l.id = ce.link_id AND ce.clicked_at >= NOW() - INTERVAL '${days} days' + AND ce.is_bot = false ${userFilterWhere} GROUP BY l.id ORDER BY total_clicks DESC @@ -158,7 +159,7 @@ export async function analyticsRoutes(fastify: FastifyInstance) { COUNT(*) as total_clicks, COUNT(DISTINCT ip_address) as unique_clicks FROM click_events - WHERE link_id = $1 AND clicked_at >= NOW() - INTERVAL '${days} days'`, + WHERE link_id = $1 AND clicked_at >= NOW() - INTERVAL '${days} days' AND is_bot = false`, [linkId] ); @@ -167,7 +168,7 @@ export async function analyticsRoutes(fastify: FastifyInstance) { DATE(clicked_at) as date, COUNT(*) as clicks FROM click_events - WHERE link_id = $1 AND clicked_at >= NOW() - INTERVAL '${days} days' + WHERE link_id = $1 AND clicked_at >= NOW() - INTERVAL '${days} days' AND is_bot = false GROUP BY DATE(clicked_at) ORDER BY date`, [linkId] @@ -179,7 +180,7 @@ export async function analyticsRoutes(fastify: FastifyInstance) { COALESCE(country_name, country_code, 'Unknown') as country, COUNT(*) as clicks FROM click_events - WHERE link_id = $1 AND clicked_at >= NOW() - INTERVAL '${days} days' + WHERE link_id = $1 AND clicked_at >= NOW() - INTERVAL '${days} days' AND is_bot = false GROUP BY country_code, country_name ORDER BY clicks DESC`, [linkId] @@ -190,7 +191,7 @@ export async function analyticsRoutes(fastify: FastifyInstance) { COALESCE(device_type, 'Unknown') as device, COUNT(*) as clicks FROM click_events - WHERE link_id = $1 AND clicked_at >= NOW() - INTERVAL '${days} days' + WHERE link_id = $1 AND clicked_at >= NOW() - INTERVAL '${days} days' AND is_bot = false GROUP BY device_type ORDER BY clicks DESC`, [linkId] @@ -201,7 +202,7 @@ export async function analyticsRoutes(fastify: FastifyInstance) { COALESCE(platform, 'Unknown') as platform, COUNT(*) as clicks FROM click_events - WHERE link_id = $1 AND clicked_at >= NOW() - INTERVAL '${days} days' + WHERE link_id = $1 AND clicked_at >= NOW() - INTERVAL '${days} days' AND is_bot = false GROUP BY platform ORDER BY clicks DESC`, [linkId] diff --git a/src/routes/redirect.ts b/src/routes/redirect.ts index 358faf6..5b05632 100644 --- a/src/routes/redirect.ts +++ b/src/routes/redirect.ts @@ -5,6 +5,7 @@ import { getClientIp } from '../lib/client-ip.js'; import { parseUserAgent, getLocationFromIP, buildRedirectUrl, detectDevice } from '../lib/utils.js'; import { storeFingerprintForClick, type FingerprintData } from '../lib/fingerprint.js'; import { emitClickEvent } from '../lib/event-emitter.js'; +import { classifyBot, edgeBotSignal } from '../lib/bot-detection.js'; /** * Detect iOS in-app browsers where Universal Links don't fire. @@ -266,6 +267,15 @@ export async function redirectRoutes(fastify: FastifyInstance) { const { platform, platformVersion } = parseUserAgent(userAgent); const { countryCode, countryName, region, city, latitude, longitude, timezone } = getLocationFromIP(ip); + // Classify bots at ingestion (SIT-298) — persisted on the row so + // analytics reads a consistent flag instead of re-detecting from the + // stored user-agent. + const { isBot, reason: botReason } = classifyBot( + userAgent, + request.method, + edgeBotSignal(request.headers['x-lf-bot']) + ); + // Extract UTM parameters from query string const query = request.query as Record; const utmSource = query?.utm_source; @@ -284,8 +294,8 @@ export async function redirectRoutes(fastify: FastifyInstance) { `INSERT INTO click_events ( id, link_id, ip_address, user_agent, device_type, platform, country_code, country_name, region, city, latitude, longitude, timezone, - utm_source, utm_medium, utm_campaign, referrer - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)`, + utm_source, utm_medium, utm_campaign, referrer, is_bot, bot_reason + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19)`, [ clickId, link.id, @@ -304,6 +314,8 @@ export async function redirectRoutes(fastify: FastifyInstance) { utmMedium, utmCampaign, referrer, + isBot, + botReason, ] ); diff --git a/src/routes/sdk.ts b/src/routes/sdk.ts index f3a3610..37f40db 100644 --- a/src/routes/sdk.ts +++ b/src/routes/sdk.ts @@ -11,6 +11,7 @@ import { import { triggerWebhooks } from '../lib/webhook.js'; import { parseUserAgent, getLocationFromIP, detectDevice } from '../lib/utils.js'; import { emitClickEvent } from '../lib/event-emitter.js'; +import { classifyBot, edgeBotSignal } from '../lib/bot-detection.js'; /** * SDK Routes - Mobile SDK endpoints for deferred deep linking @@ -497,6 +498,13 @@ export async function sdkRoutes(fastify: FastifyInstance) { const { platform, platformVersion } = parseUserAgent(userAgent); const { countryCode, countryName, region, city, latitude, longitude, timezone } = getLocationFromIP(ip); + // Classify bots at ingestion (SIT-298); persisted for consistent reads. + const { isBot, reason: botReason } = classifyBot( + userAgent, + request.method, + edgeBotSignal(request.headers['x-lf-bot']) + ); + // Extract fingerprint data from query params (sent by SDK) const query = request.query as Record; const fpTimezone = query?.fp_tz || timezone || undefined; @@ -509,8 +517,8 @@ export async function sdkRoutes(fastify: FastifyInstance) { `INSERT INTO click_events ( link_id, ip_address, user_agent, device_type, platform, country_code, country_name, region, city, latitude, longitude, timezone, - utm_source, utm_medium, utm_campaign, referrer - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) + utm_source, utm_medium, utm_campaign, referrer, is_bot, bot_reason + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18) RETURNING id`, [ link.id, @@ -529,6 +537,8 @@ export async function sdkRoutes(fastify: FastifyInstance) { query?.utm_medium || null, query?.utm_campaign || null, referrer, + isBot, + botReason, ] ); diff --git a/src/scripts/backfill-bot-flags.ts b/src/scripts/backfill-bot-flags.ts new file mode 100644 index 0000000..2dba2a8 --- /dev/null +++ b/src/scripts/backfill-bot-flags.ts @@ -0,0 +1,55 @@ +/** + * One-time backfill of click_events.is_bot for rows recorded before bot + * classification existed (SIT-298). Historical rows only have the stored + * user-agent, so this classifies UA-only (method/edge signals aren't available + * retroactively) and flags matches as is_bot=true, bot_reason='ua'. + * + * Keyset-paginated over the primary key so it scales to large tables and visits + * every row once. Idempotent: re-running re-checks and leaves already-correct + * rows unchanged. + * + * Run: `npm run backfill:bot-flags` (needs DATABASE_URL). + */ +import { db, initializeDatabase } from '../lib/database.js'; +import { classifyBot } from '../lib/bot-detection.js'; + +const BATCH = 5000; + +async function backfill() { + await initializeDatabase(); + let cursor = '00000000-0000-0000-0000-000000000000'; + let scanned = 0; + let flagged = 0; + + for (;;) { + const { rows } = await db.query( + `SELECT id, user_agent FROM click_events WHERE id > $1 ORDER BY id LIMIT $2`, + [cursor, BATCH] + ); + if (rows.length === 0) break; + + const botIds: string[] = []; + for (const r of rows) { + if (classifyBot(r.user_agent ?? undefined, undefined, undefined).isBot) botIds.push(r.id); + } + if (botIds.length > 0) { + await db.query( + `UPDATE click_events SET is_bot = true, bot_reason = 'ua' WHERE id = ANY($1::uuid[])`, + [botIds] + ); + flagged += botIds.length; + } + + scanned += rows.length; + cursor = rows[rows.length - 1].id; + console.log(`[backfill-bot-flags] scanned ${scanned}, flagged ${flagged}`); + } + + console.log(`[backfill-bot-flags] done: scanned ${scanned}, flagged ${flagged} as bots`); + process.exit(0); +} + +backfill().catch((error) => { + console.error('[backfill-bot-flags] failed:', error); + process.exit(1); +});