feat: agentPolicy() — decide what to do with an agent, by intent

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apideck/agent-analytics",
-  "version": "0.12.0",
+  "version": "0.13.0",
   "description": "Track AI agent and bot traffic to your Next.js / Vercel app — PostHog, webhooks, or any custom analytics backend. Detects Claude, ChatGPT, Perplexity, Google-Extended, and more.",
   "keywords": [
     "ai",

src/index.ts

@@ -13,6 +13,13 @@ export {
 export type { AgentClassification, AgentKind, HeadlessDetection } from './bots.js'
 export { hashId, randomSecret, HashSecretError } from './hash.js'
 export { CaptureTransportError } from './errors.js'
+export { agentIntent, agentPolicy } from './policy.js'
+export type {
+  AgentAction,
+  AgentDecision,
+  AgentIntent,
+  AgentPolicyOptions
+} from './policy.js'
 export { posthogAnalytics } from './adapters/posthog.js'
 export { webhookAnalytics } from './adapters/webhook.js'
 export { customAnalytics } from './adapters/custom.js'

src/policy.ts

@@ -0,0 +1,165 @@
+import { classifyRequest } from './bots.js'
+import type { BotVerificationLike } from './types.js'
+
+/**
+ * What to do with an agent request.
+ *
+ * - `'allow'`  — serve it, free. Humans, search crawlers, and the retrieval
+ *   agents you *want* reading your site.
+ * - `'meter'`  — serve it, but count it as billable. Bulk corpus collection.
+ * - `'charge'` — don't serve it until it pays (HTTP 402).
+ * - `'block'`  — refuse. Failed identity verification, mostly.
+ */
+export type AgentAction = 'allow' | 'meter' | 'charge' | 'block'
+
+/**
+ * Why an agent fetched the page. This is the distinction the whole module
+ * exists for, and no other signal on the request carries it.
+ *
+ * - `'retrieval'` — a person asked a question and the assistant went to read
+ *   the page for them. This is *demand*: the agent is a distribution channel,
+ *   and charging for it is charging for your own marketing.
+ * - `'training'`  — bulk corpus collection for model training. You get nothing
+ *   back per fetch, which is where a price makes sense.
+ * - `'search'`    — classic index crawlers. Blocking these costs you SEO.
+ * - `'tooling'`   — coding agents and HTTP clients. Usually developers using
+ *   your docs; treat like retrieval unless you see abuse.
+ * - `'unknown'`   — everything else, including real browsers.
+ */
+export type AgentIntent = 'retrieval' | 'training' | 'search' | 'tooling' | 'unknown'
+
+/**
+ * User agents where a human is waiting on the answer. Deliberately explicit
+ * rather than pattern-guessed: `-User` is not a reliable marker (OpenAI's
+ * ChatGPT-User fetches server-side; Claude Code's Claude-User runs on a
+ * laptop), and getting this wrong means charging your own demand channel.
+ */
+const RETRIEVAL = /ChatGPT-User|OAI-SearchBot|Claude-User|Claude-SearchBot|Perplexity-User|claude-code|DuckAssistBot|MistralAI-User|Gemini-Deep-Research|Manus-User|YouBot/i
+
+/** Bulk crawlers that collect corpora. No human is waiting on these. */
+const TRAINING = /GPTBot|ClaudeBot|Claude-Web|CCBot|Bytespider|Amazonbot|Amzn-SearchBot|Meta-ExternalAgent|meta-externalfetcher|meta-webindexer|FacebookBot|Google-Extended|Applebot-Extended|AI2Bot|Diffbot|omgili|Webzio-Extended|Timpibot|PanguBot|cohere|DeepSeek|Grok|quillbot|MyCentralAIScraperBot|NovaAct|AzureAI-SearchBot|Google-CloudVertexBot/i
+
+/** Classic search indexers — blocking these costs you organic traffic. */
+const SEARCH = /bingbot|Googlebot|DuckDuckBot|YandexBot|Baiduspider|PetalBot|Sogou|Applebot(?!-Extended)/i
+
+export interface AgentDecision {
+  action: AgentAction
+  intent: AgentIntent
+  /** Vendor label, same string `parseBotName` returns. */
+  label: string
+  /** Identity verdict, when a verifier was supplied. */
+  verification?: string
+  /** Short human-readable justification — log it, don't parse it. */
+  reason: string
+}
+
+export interface AgentPolicyOptions {
+  /**
+   * Identity verifier. Import `verifyRequest` from
+   * `@apideck/agent-analytics/verify` and pass it here to have a `spoofed`
+   * verdict produce `'block'`.
+   *
+   * Injected rather than imported so the published IP range tables only reach
+   * bundles that use them. Only meaningful when your edge controls
+   * `x-forwarded-for`: behind a proxy that forwards a client-supplied header,
+   * an attacker picks their own verdict.
+   */
+  verify?: (req: Request) => BotVerificationLike
+  /** What to do with bulk training crawlers. Defaults to `'meter'`. */
+  onTraining?: AgentAction
+  /** What to do with retrieval agents. Defaults to `'allow'` — see AgentIntent. */
+  onRetrieval?: AgentAction
+  /** What to do with search indexers. Defaults to `'allow'`. */
+  onSearch?: AgentAction
+  /** What to do with coding agents and HTTP clients. Defaults to `'allow'`. */
+  onTooling?: AgentAction
+  /** Vendor labels or UA substrings always allowed, whatever the intent. */
+  allowList?: readonly string[]
+}
+
+/** Classify why an agent is here, from its user agent alone. */
+export function agentIntent(userAgent: string | null | undefined): AgentIntent {
+  const ua = userAgent ?? ''
+  if (!ua) return 'unknown'
+  // Retrieval is checked first: several vendors ship both a bulk crawler and a
+  // user-facing fetcher whose tokens overlap (ClaudeBot vs Claude-User).
+  if (RETRIEVAL.test(ua)) return 'retrieval'
+  if (TRAINING.test(ua)) return 'training'
+  if (SEARCH.test(ua)) return 'search'
+  return 'unknown'
+}
+
+/**
+ * Decide what to do with a request. Pure classification plus policy — no
+ * payment rails, no network calls, nothing to configure beyond the four
+ * intent knobs.
+ *
+ * @example
+ * ```ts
+ * const decision = agentPolicy(req, { verify: true, onTraining: 'charge' })
+ * if (decision.action === 'block') return new Response(null, { status: 403 })
+ * if (decision.action === 'charge') return paymentRequired(decision)
+ * ```
+ */
+export function agentPolicy(req: Request, opts: AgentPolicyOptions = {}): AgentDecision {
+  const ua = req.headers.get('user-agent') || ''
+  const classification = classifyRequest(req)
+  const label = classification.label
+
+  let intent = agentIntent(ua)
+  // An HTTP-library UA that matched no vendor is a coding agent or a script.
+  if (intent === 'unknown' && classification.codingAgentHint) intent = 'tooling'
+
+  const allowed = opts.allowList?.some(
+    (entry) => entry === label || ua.toLowerCase().includes(entry.toLowerCase())
+  )
+  if (allowed) {
+    return { action: 'allow', intent, label, reason: 'on allowList' }
+  }
+
+  let verification: string | undefined
+  if (opts.verify) {
+    verification = opts.verify(req).verdict
+    // Only 'spoofed' is actionable. 'unverifiable' means we couldn't check —
+    // blocking on it would refuse every vendor without a published feed and
+    // every coding agent running on someone's own machine.
+    if (verification === 'spoofed') {
+      return {
+        action: 'block',
+        intent,
+        label,
+        verification,
+        reason: `${label} claimed but client IP is outside its published ranges`
+      }
+    }
+  }
+
+  const action: AgentAction =
+    intent === 'training'
+      ? (opts.onTraining ?? 'meter')
+      : intent === 'retrieval'
+        ? (opts.onRetrieval ?? 'allow')
+        : intent === 'search'
+          ? (opts.onSearch ?? 'allow')
+          : intent === 'tooling'
+            ? (opts.onTooling ?? 'allow')
+            : 'allow'
+
+  const REASONS: Record<AgentIntent, string> = {
+    retrieval: 'a person is waiting on this answer',
+    training: 'bulk corpus collection',
+    search: 'search index crawler',
+    tooling: 'coding agent or HTTP client',
+    unknown: 'not a recognised agent'
+  }
+
+  return {
+    action,
+    intent,
+    label,
+    // Spread rather than assign: `exactOptionalPropertyTypes` distinguishes an
+    // absent key from one explicitly set to undefined.
+    ...(verification ? { verification } : {}),
+    reason: REASONS[intent]
+  }
+}

test/policy.test.ts

@@ -0,0 +1,120 @@
+import { describe, expect, it } from 'vitest'
+import { agentIntent, agentPolicy } from '../src/policy.js'
+import { verifyRequest } from '../src/verify.js'
+
+function req(ua: string, headers: Record<string, string> = {}) {
+  return new Request('https://example.com/docs', {
+    headers: { 'user-agent': ua, ...headers }
+  })
+}
+
+const GPTBOT = 'Mozilla/5.0 (compatible; GPTBot/1.1; +https://openai.com/gptbot)'
+const CHATGPT_USER = 'Mozilla/5.0 (compatible; ChatGPT-User/1.0; +https://openai.com/bot)'
+const CLAUDEBOT = 'Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)'
+const CLAUDE_CODE = 'Claude-User (claude-code/2.1.218; +https://support.anthropic.com/)'
+const GOOGLEBOT = 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
+const BROWSER =
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+
+describe('agentIntent', () => {
+  it('separates a vendor\'s bulk crawler from its user-facing fetcher', () => {
+    // The whole point of the module: same vendor, opposite economics.
+    expect(agentIntent(GPTBOT)).toBe('training')
+    expect(agentIntent(CHATGPT_USER)).toBe('retrieval')
+    expect(agentIntent(CLAUDEBOT)).toBe('training')
+    expect(agentIntent(CLAUDE_CODE)).toBe('retrieval')
+  })
+
+  it('treats classic indexers as search, not training', () => {
+    expect(agentIntent(GOOGLEBOT)).toBe('search')
+    expect(agentIntent('Mozilla/5.0 (compatible; bingbot/2.0)')).toBe('search')
+    // Applebot indexes; Applebot-Extended is the training opt-in. Different intent.
+    expect(agentIntent('Mozilla/5.0 (compatible; Applebot/0.1)')).toBe('search')
+    expect(agentIntent('Mozilla/5.0 (compatible; Applebot-Extended/0.1)')).toBe('training')
+  })
+
+  it('returns unknown for browsers and empty input', () => {
+    expect(agentIntent(BROWSER)).toBe('unknown')
+    expect(agentIntent('')).toBe('unknown')
+    expect(agentIntent(null)).toBe('unknown')
+  })
+})
+
+describe('agentPolicy defaults', () => {
+  it('meters training crawlers but never charges retrieval', () => {
+    // Charging retrieval means charging your own distribution channel.
+    expect(agentPolicy(req(GPTBOT)).action).toBe('meter')
+    expect(agentPolicy(req(CLAUDEBOT)).action).toBe('meter')
+    expect(agentPolicy(req(CHATGPT_USER)).action).toBe('allow')
+    expect(agentPolicy(req(CLAUDE_CODE)).action).toBe('allow')
+  })
+
+  it('lets search crawlers and browsers straight through', () => {
+    expect(agentPolicy(req(GOOGLEBOT)).action).toBe('allow')
+    expect(agentPolicy(req(BROWSER)).action).toBe('allow')
+  })
+
+  it('classifies bare HTTP clients as tooling', () => {
+    const d = agentPolicy(req('curl/8.4.0'))
+    expect(d.intent).toBe('tooling')
+    expect(d.action).toBe('allow')
+  })
+
+  it('carries a reason for logging', () => {
+    expect(agentPolicy(req(GPTBOT)).reason).toBe('bulk corpus collection')
+    expect(agentPolicy(req(CHATGPT_USER)).reason).toBe('a person is waiting on this answer')
+  })
+})
+
+describe('agentPolicy overrides', () => {
+  it('can charge training crawlers instead of metering them', () => {
+    expect(agentPolicy(req(GPTBOT), { onTraining: 'charge' }).action).toBe('charge')
+    // Retrieval is unaffected by the training knob.
+    expect(agentPolicy(req(CHATGPT_USER), { onTraining: 'charge' }).action).toBe('allow')
+  })
+
+  it('honours the allowList over any intent rule', () => {
+    const d = agentPolicy(req(GPTBOT), { onTraining: 'block', allowList: ['ChatGPT'] })
+    expect(d.action).toBe('allow')
+    expect(d.reason).toBe('on allowList')
+  })
+})
+
+describe('agentPolicy with verification', () => {
+  const REAL_OPENAI_IP = '104.208.184.193'
+
+  it('blocks a spoofed crawler', () => {
+    const d = agentPolicy(req(CHATGPT_USER, { 'x-forwarded-for': '1.2.3.4' }), { verify: verifyRequest })
+    expect(d.action).toBe('block')
+    expect(d.verification).toBe('spoofed')
+  })
+
+  it('allows the same UA from a published range', () => {
+    const d = agentPolicy(req(CHATGPT_USER, { 'x-forwarded-for': REAL_OPENAI_IP }), { verify: verifyRequest })
+    expect(d.action).toBe('allow')
+    expect(d.verification).toBe('verified')
+  })
+
+  it('does not block on unverifiable', () => {
+    // Bytespider publishes no ranges, and Claude Code runs on a laptop.
+    // Blocking either would refuse legitimate traffic we simply can't check.
+    const bytespider = agentPolicy(
+      req('Mozilla/5.0 (compatible; Bytespider/1.0)', { 'x-forwarded-for': '1.2.3.4' }),
+      { verify: verifyRequest }
+    )
+    expect(bytespider.verification).toBe('unverifiable')
+    expect(bytespider.action).not.toBe('block')
+
+    const local = agentPolicy(req(CLAUDE_CODE, { 'x-forwarded-for': '109.135.42.185' }), {
+      verify: verifyRequest
+    })
+    expect(local.verification).toBe('unverifiable')
+    expect(local.action).toBe('allow')
+  })
+
+  it('skips verification entirely when not asked', () => {
+    const d = agentPolicy(req(CHATGPT_USER, { 'x-forwarded-for': '1.2.3.4' }))
+    expect(d.verification).toBeUndefined()
+    expect(d.action).toBe('allow')
+  })
+})