From 4121daa9b93ee4c271655c43226ebe41295fe2cc Mon Sep 17 00:00:00 2001 From: Brandon Estrella Date: Wed, 1 Jul 2026 09:22:42 -0700 Subject: [PATCH] chore: remove unused backfill scripts (bot-flags, attribution) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These one-time backfill scripts were dead weight: undocumented, so no operator would run them, and unnecessary — retention already deletes click_events/install_events (and the rollup) per tier, so un-flagged legacy rows age out on their own. The bot backfill was also incomplete (it left already-built rollups stale). Steady state is correct without them: new clicks are classified at ingestion. Removes src/scripts/backfill-bot-flags.ts, src/scripts/backfill-attribution.ts, and their npm script entries. isbot stays (used by the live classifier). --- package.json | 2 -- src/lib/database.ts | 2 +- src/scripts/backfill-attribution.ts | 50 -------------------------- src/scripts/backfill-bot-flags.ts | 55 ----------------------------- 4 files changed, 1 insertion(+), 108 deletions(-) delete mode 100644 src/scripts/backfill-attribution.ts delete mode 100644 src/scripts/backfill-bot-flags.ts diff --git a/package.json b/package.json index 5ccc855..78efc1f 100644 --- a/package.json +++ b/package.json @@ -37,8 +37,6 @@ "scripts": { "build": "tsc", "dev": "tsx watch src/index.ts", - "backfill:bot-flags": "tsx src/scripts/backfill-bot-flags.ts", - "backfill:attribution": "tsx src/scripts/backfill-attribution.ts", "migrate": "tsx src/scripts/migrate.ts", "test": "vitest run", "test:watch": "vitest", diff --git a/src/lib/database.ts b/src/lib/database.ts index 3366f71..d4c0848 100644 --- a/src/lib/database.ts +++ b/src/lib/database.ts @@ -396,7 +396,7 @@ export async function initializeDatabase(options: DatabaseOptions = {}) { // Bot classification columns on click_events (SIT-298). Classified at // ingestion (see lib/bot-detection.ts) and persisted so every consumer reads // one consistent flag; analytics excludes is_bot rows. Backward compatible: - // legacy rows default to is_bot=false until the one-time backfill runs. + // legacy rows default to is_bot=false and age out of the retention window. await client.query(` DO $$ BEGIN diff --git a/src/scripts/backfill-attribution.ts b/src/scripts/backfill-attribution.ts deleted file mode 100644 index deb3b08..0000000 --- a/src/scripts/backfill-attribution.ts +++ /dev/null @@ -1,50 +0,0 @@ -/** - * One-time backfill of install_events.attribution_method for rows recorded - * before attribution metadata existed (SIT-296). Historical rows only tell us - * whether they were attributed (link_id present), so this sets: - * - 'fingerprint' when link_id IS NOT NULL (current attribution is fingerprint-based) - * - 'none' otherwise (organic) - * `matched_factors` is left NULL for history — the matched signals weren't - * stored and can't be reconstructed; new installs record it going forward. - * - * Keyset-paginated over the primary key; idempotent (only touches NULL rows). - * Run: `npm run backfill:attribution` (needs DATABASE_URL). - */ -import { db, initializeDatabase } from '../lib/database.js'; - -const BATCH = 5000; - -async function backfill() { - await initializeDatabase(); - let cursor = '00000000-0000-0000-0000-000000000000'; - let scanned = 0; - let updated = 0; - - for (;;) { - const { rows } = await db.query( - `SELECT id FROM install_events WHERE id > $1 ORDER BY id LIMIT $2`, - [cursor, BATCH] - ); - if (rows.length === 0) break; - - const ids = rows.map((r: { id: string }) => r.id); - const res = await db.query( - `UPDATE install_events - SET attribution_method = CASE WHEN link_id IS NOT NULL THEN 'fingerprint' ELSE 'none' END - WHERE id = ANY($1::uuid[]) AND attribution_method IS NULL`, - [ids] - ); - updated += res.rowCount ?? 0; - scanned += rows.length; - cursor = rows[rows.length - 1].id; - console.log(`[backfill-attribution] scanned ${scanned}, set ${updated}`); - } - - console.log(`[backfill-attribution] done: scanned ${scanned}, set attribution_method on ${updated}`); - process.exit(0); -} - -backfill().catch((error) => { - console.error('[backfill-attribution] failed:', error); - process.exit(1); -}); diff --git a/src/scripts/backfill-bot-flags.ts b/src/scripts/backfill-bot-flags.ts deleted file mode 100644 index 2dba2a8..0000000 --- a/src/scripts/backfill-bot-flags.ts +++ /dev/null @@ -1,55 +0,0 @@ -/** - * One-time backfill of click_events.is_bot for rows recorded before bot - * classification existed (SIT-298). Historical rows only have the stored - * user-agent, so this classifies UA-only (method/edge signals aren't available - * retroactively) and flags matches as is_bot=true, bot_reason='ua'. - * - * Keyset-paginated over the primary key so it scales to large tables and visits - * every row once. Idempotent: re-running re-checks and leaves already-correct - * rows unchanged. - * - * Run: `npm run backfill:bot-flags` (needs DATABASE_URL). - */ -import { db, initializeDatabase } from '../lib/database.js'; -import { classifyBot } from '../lib/bot-detection.js'; - -const BATCH = 5000; - -async function backfill() { - await initializeDatabase(); - let cursor = '00000000-0000-0000-0000-000000000000'; - let scanned = 0; - let flagged = 0; - - for (;;) { - const { rows } = await db.query( - `SELECT id, user_agent FROM click_events WHERE id > $1 ORDER BY id LIMIT $2`, - [cursor, BATCH] - ); - if (rows.length === 0) break; - - const botIds: string[] = []; - for (const r of rows) { - if (classifyBot(r.user_agent ?? undefined, undefined, undefined).isBot) botIds.push(r.id); - } - if (botIds.length > 0) { - await db.query( - `UPDATE click_events SET is_bot = true, bot_reason = 'ua' WHERE id = ANY($1::uuid[])`, - [botIds] - ); - flagged += botIds.length; - } - - scanned += rows.length; - cursor = rows[rows.length - 1].id; - console.log(`[backfill-bot-flags] scanned ${scanned}, flagged ${flagged}`); - } - - console.log(`[backfill-bot-flags] done: scanned ${scanned}, flagged ${flagged} as bots`); - process.exit(0); -} - -backfill().catch((error) => { - console.error('[backfill-bot-flags] failed:', error); - process.exit(1); -});