From 0eb9721f5aef49950a44aa7d19610f1ad65c3fff Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Fri, 31 Jul 2026 05:52:14 -0700 Subject: [PATCH] feat(observability): opt-in capture of AI prompts and completions PostHog's LLM Analytics conversation view, sentiment, evaluations and LLM-judge surfaces all read $ai_input/$ai_output_choices, which were deliberately never populated -- so every trace in the project showed a dash for input, output and sentiment, and the whole content-dependent half of the product was unavailable. Populate them behind LOOPOVER_POSTHOG_AI_CONTENT, default OFF. Upgrading an ORB must never start shipping private PR diffs and model completions to a vendor as a side effect of a version bump, so an operator who has not opted in is byte-identical to before. The gate lives in the capture function rather than at each call site: ai.ts always passes the content and posthog.ts alone decides, so a future call site cannot forget it. Captured content passes through the same before_send scrub as every other field, not a second weaker path. Each message is capped at 10,000 chars (LOOPOVER_POSTHOG_AI_CONTENT_MAX_CHARS) and marked when truncated -- a review prompt can exceed 300k chars and would otherwise blow past PostHog's payload limits and be dropped wholesale; a non-numeric, zero or negative override falls back to the default rather than disabling the cap. An image block is dropped rather than base64-encoded into the event, reusing the same projection the subscription CLIs already apply when building their stdin prompt. Embedding requests report their raw texts under a synthetic user role so one property shape covers both request kinds. The failure path captures the prompt and no completion -- a claude_stalled_no_output or a context-length rejection is only diagnosable against the input that produced it. The self-hosting doc promised metadata-only unconditionally and would have become false; it now documents the opt-in, the truncation, and what enabling it actually sends. The absence-assertion test now covers BOTH states rather than being deleted. Closes #10218 --- .../content/docs/self-hosting-operations.mdx | 24 ++++++- .../src/lib/selfhost-env-reference.ts | 10 +++ src/selfhost/ai.ts | 21 ++++++ src/selfhost/posthog.ts | 65 +++++++++++++++++ ...ocs-selfhost-posthog-observability.test.ts | 9 +++ test/unit/selfhost-ai.test.ts | 69 +++++++++++++++++++ test/unit/selfhost-posthog.test.ts | 69 ++++++++++++++++++- 7 files changed, 264 insertions(+), 3 deletions(-) diff --git a/apps/loopover-ui/content/docs/self-hosting-operations.mdx b/apps/loopover-ui/content/docs/self-hosting-operations.mdx index 17de50324..1ef889dfb 100644 --- a/apps/loopover-ui/content/docs/self-hosting-operations.mdx +++ b/apps/loopover-ui/content/docs/self-hosting-operations.mdx @@ -676,7 +676,7 @@ three standalone bindings (`AI_EMBED`, `AI_VISION`, `AI_ADVISORY`). { title: "Content policy", description: - "Metadata only — never the prompt or completion text. $ai_generation's own optional content fields ($ai_input/$ai_output_choices) are never populated, the same redaction posture as every other capture path on this page.", + "Metadata only by default — no prompt or completion text leaves the box unless you explicitly opt in. $ai_generation's optional content fields ($ai_input/$ai_output_choices) stay unpopulated unless LOOPOVER_POSTHOG_AI_CONTENT is set, so upgrading never starts shipping content on its own.", }, { title: "Model labelling", @@ -691,6 +691,28 @@ three standalone bindings (`AI_EMBED`, `AI_VISION`, `AI_ADVISORY`). ]} /> +### Capturing prompts and completions (`LOOPOVER_POSTHOG_AI_CONTENT`) + +**Off by default, and deliberately so.** With it unset, no prompt, diff or model completion ever leaves the +box — only the metadata listed above. Set `LOOPOVER_POSTHOG_AI_CONTENT=1` to populate `$ai_input` and +`$ai_output_choices`, which is what unlocks PostHog's conversation view, sentiment, evaluations and +LLM-judge surfaces. Read this before turning it on: + +- **Your PR diffs and review text go to PostHog.** The prompt for a review contains the diff and the + assembled repo context. On a private repository that is private source code leaving your infrastructure. + Only enable it on a project you are willing to have that content in. +- **The same redaction still applies.** Captured content passes through the identical `before_send` scrub + (`src/selfhost/redaction-scrub.ts`) as every other field — credential shapes, JWTs and query-string + secrets are redacted from it. That is a backstop, not a licence: it removes credential shapes, not the + source code itself. +- **Content is truncated.** Each message is capped at 10,000 characters + (`LOOPOVER_POSTHOG_AI_CONTENT_MAX_CHARS`) and marked `…[truncated]`. A review prompt can exceed 300,000 + characters, so an uncapped capture would blow past PostHog's payload limits and be dropped entirely. A + non-numeric, zero or negative override falls back to the default rather than disabling the cap. +- **Images are never captured.** An image content block is dropped, not base64-encoded into the event. +- **On a failure, the prompt is captured but there is no completion** — a `claude_stalled_no_output` or a + context-length rejection is only diagnosable against the input that produced it. + A CLI-subscription provider declining an embedding request (`claude_code_no_embed` / `codex_no_embed`) is routing working as designed — it is how an embed reaches an embed-capable provider — so it is exempt from both captures. A chain with **no** embed-capable member at all is a real misconfiguration, but it diff --git a/apps/loopover-ui/src/lib/selfhost-env-reference.ts b/apps/loopover-ui/src/lib/selfhost-env-reference.ts index 0199348e8..5dc45cf7d 100644 --- a/apps/loopover-ui/src/lib/selfhost-env-reference.ts +++ b/apps/loopover-ui/src/lib/selfhost-env-reference.ts @@ -321,6 +321,14 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ name: "LOOPOVER_METRICS_REPO_LABELS", firstReference: "src/server.ts", }, + { + name: "LOOPOVER_POSTHOG_AI_CONTENT", + firstReference: "src/selfhost/posthog.ts", + }, + { + name: "LOOPOVER_POSTHOG_AI_CONTENT_MAX_CHARS", + firstReference: "src/selfhost/posthog.ts", + }, { name: "LOOPOVER_PUBLIC_SCORE_TERMS_ALLOWED_REPOS", firstReference: "src/selfhost/inert-config.ts", @@ -801,6 +809,8 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `LOOPOVER_LEDGER_UNCHAINED_WAIVER` | `src/selfhost/preflight.ts` |", "| `LOOPOVER_MCP_TOKEN` | `src/selfhost/preflight.ts` |", "| `LOOPOVER_METRICS_REPO_LABELS` | `src/server.ts` |", + "| `LOOPOVER_POSTHOG_AI_CONTENT` | `src/selfhost/posthog.ts` |", + "| `LOOPOVER_POSTHOG_AI_CONTENT_MAX_CHARS` | `src/selfhost/posthog.ts` |", "| `LOOPOVER_PUBLIC_SCORE_TERMS_ALLOWED_REPOS` | `src/selfhost/inert-config.ts` |", "| `LOOPOVER_PUBLIC_STATS_REPOS` | `src/selfhost/inert-config.ts` |", "| `LOOPOVER_REPO_CONFIG_DIR` | `src/server.ts` |", diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts index a779d20ef..7a27f8064 100644 --- a/src/selfhost/ai.ts +++ b/src/selfhost/ai.ts @@ -1559,6 +1559,19 @@ function isExpectedEmbeddingRoutingError(options: AiRunOptions, error: unknown): return requestKind(options) === "embedding" && EXPECTED_EMBEDDING_ROUTING_ERRORS.has(errorMessage(error)); } +/** The prompt, flattened to the `{role, content}` list PostHog's `$ai_input` expects (#10218). + * + * Reuses `contentText`, so an image block is dropped rather than base64-encoded into a telemetry event -- + * the same projection the subscription CLIs already apply when building their stdin prompt, so what is + * captured is what the provider was actually given. An embedding request has no messages at all; its + * inputs are the raw texts, reported under the same synthetic `user` role so one property shape covers + * both request kinds. Capture is gated in posthog.ts, so this always builds and is simply ignored when + * the operator has not opted in. */ +function aiContentInput(options: AiRunOptions): Array<{ role: string; content: string }> { + if (Array.isArray(options.text)) return options.text.map((text) => ({ role: "user", content: text })); + return toMessages(options).map((message) => ({ role: message.role, content: contentText(message.content) })); +} + async function runProviderWithOtel( provider: { name: string; ai: SelfHostAi }, model: string, @@ -1619,6 +1632,8 @@ async function runProviderWithOtel( totalCostUsd: usage?.costUsd, effort: usage?.effort, context: { repo: options.repoFullName, pullNumber: options.pullNumber }, + input: aiContentInput(options), + outputText: result.response, }); return usage ? { ...result, usage } : result; } catch (error) { @@ -1635,6 +1650,9 @@ async function runProviderWithOtel( isError: true, error, context: { repo: options.repoFullName, pullNumber: options.pullNumber }, + // The prompt is what a failure needs most: a `claude_stalled_no_output` or a context-length rejection + // is only diagnosable against the input that produced it. There is no completion to report. + input: aiContentInput(options), }); incr("loopover_ai_provider_failures_total", { provider: provider.name }); incr("loopover_ai_provider_request_errors_total", { provider: provider.name, request_kind: requestKindLabel }); @@ -1809,6 +1827,8 @@ export function withAiGenerationCapture(providerName: string, ai: SelfHostAi): S outputTokens: usage?.outputTokens, totalCostUsd: usage?.costUsd, effort: usage?.effort, + input: aiContentInput(options), + outputText: result.response, }); return usage ? { ...result, usage } : result; } catch (error) { @@ -1819,6 +1839,7 @@ export function withAiGenerationCapture(providerName: string, ai: SelfHostAi): S latencyMs: Date.now() - startedAtMs, isError: true, error, + input: aiContentInput(options), }); throw error; } diff --git a/src/selfhost/posthog.ts b/src/selfhost/posthog.ts index 8abc9ceb7..a867ab64d 100644 --- a/src/selfhost/posthog.ts +++ b/src/selfhost/posthog.ts @@ -74,6 +74,37 @@ function processEnvString(env: NodeJS.ProcessEnv, name: string): string | undefi return nonBlank(env[name]); } +/** Truthy-string env flag, matching the repo-wide convention for a flag-gated capability that ships OFF. */ +const TRUE_ENV_VALUES = new Set(["1", "true", "yes", "on"]); + +/** Default ceiling on a single captured message's characters (#10218). A review prompt carries up to 120k + * chars of diff plus a 240k-char aggregate context budget, so an uncapped capture would push events past + * PostHog's own payload limits and be dropped wholesale -- a cap is what makes this survivable at all. */ +const DEFAULT_AI_CONTENT_MAX_CHARS = 10_000; + +/** #10218: whether `$ai_input`/`$ai_output_choices` are populated at all. OFF unless + * `LOOPOVER_POSTHOG_AI_CONTENT` is explicitly truthy. Self-host-only, read off real process.env, matching + * this file's precedent for every other self-host-exclusive knob. */ +let aiContentCapture = false; +let aiContentMaxChars = DEFAULT_AI_CONTENT_MAX_CHARS; + +function resolveAiContentCapture(env: NodeJS.ProcessEnv): boolean { + return TRUE_ENV_VALUES.has((env.LOOPOVER_POSTHOG_AI_CONTENT ?? "").trim().toLowerCase()); +} + +/** Per-message character ceiling. A non-numeric, zero, negative or fractional override falls back to the + * default rather than disabling the cap -- an operator typo must never become "send the whole diff". */ +function resolveAiContentMaxChars(env: NodeJS.ProcessEnv): number { + const raw = Number(env.LOOPOVER_POSTHOG_AI_CONTENT_MAX_CHARS); + return Number.isInteger(raw) && raw > 0 ? raw : DEFAULT_AI_CONTENT_MAX_CHARS; +} + +/** Truncate to the configured ceiling, marking that it happened so a reader never mistakes a clipped prompt + * for the whole one. */ +function boundedContent(value: string): string { + return value.length <= aiContentMaxChars ? value : `${value.slice(0, aiContentMaxChars)}…[truncated]`; +} + /** Resolve the PostHog release id: explicit override first, then the image-baked self-host version, matching * {@link resolveSentryRelease}'s identical precedence in sentry.ts. */ export function resolvePostHogRelease(env: NodeJS.ProcessEnv): string | undefined { @@ -181,6 +212,11 @@ export async function initPostHog(env: NodeJS.ProcessEnv): Promise { usingCentralPostHogKey = !explicitKey; await loadNodeHasher(); const { PostHog } = await import("posthog-node"); + // #10218: content capture is OFF unless the operator turns it on. Upgrading an ORB must never start + // shipping private PR diffs and model completions to a vendor as a side effect of a version bump, so the + // default stays metadata-only exactly as before -- see resolveAiContentCapture. + aiContentCapture = resolveAiContentCapture(env); + aiContentMaxChars = resolveAiContentMaxChars(env); posthogEnvironment = processEnvString(env, "POSTHOG_ENVIRONMENT") ?? "production"; activeRelease = resolvePostHogRelease(env); const host = processEnvString(env, "POSTHOG_HOST") ?? DEFAULT_POSTHOG_HOST; @@ -406,8 +442,34 @@ export type PostHogAiGenerationEvent = { /** Raw caught value on the error path (mirrors capturePostHogError's own `error` param) -- never a * caller-preformatted string. Ignored when `isError` is false. */ error?: unknown; + /** #10218: the prompt messages, already flattened to plain text by the caller (an image block has no + * place in a telemetry event and is dropped upstream). IGNORED unless content capture is enabled -- + * passing it is always safe, the gate lives here rather than at every call site. */ + input?: ReadonlyArray<{ role: string; content: string }> | undefined; + /** #10218: the model's completion text. Same gating as {@link PostHogAiGenerationEvent.input}. */ + outputText?: string | undefined; }; +/** Build the `$ai_input`/`$ai_output_choices` properties PostHog's LLM-analytics views read (#10218). + * Returns an EMPTY bag when content capture is off, which is the default and the pre-#10218 behavior + * byte-for-byte. Every string is bounded here; the vendor-bound redaction pass still runs afterwards over + * the whole properties bag via `before_send` (scrubPostHogEvent -> scrubRecord), so captured content goes + * through exactly the same credential/vocabulary scrubbing as every other field in this file -- it is not + * a second, weaker path. */ +function aiContentProperties(event: PostHogAiGenerationEvent): Record { + if (!aiContentCapture) return {}; + const properties: Record = {}; + if (event.input && event.input.length > 0) { + properties.$ai_input = event.input.map((message) => ({ role: message.role, content: boundedContent(message.content) })); + } + // PostHog's shape is a list of choices; the self-host providers are all single-completion, so this is + // always a one-element list rather than a fabricated multi-choice response. + if (event.outputText !== undefined && event.outputText !== "") { + properties.$ai_output_choices = [{ role: "assistant", content: boundedContent(event.outputText) }]; + } + return properties; +} + /** Capture one AI provider attempt as PostHog's `$ai_generation` (`$ai_embedding` for an embedding * request) event (#8296). No-op when PostHog is off -- same contract as every other capture function in * this file. Unlike {@link capturePostHogError}, this never gates on POSTHOG_MIN_SEVERITY: a successful @@ -451,6 +513,7 @@ export function capturePostHogAiGeneration(event: PostHogAiGenerationEvent): voi const properties: Record = { ...operational, ...aiTraceProperties(operational), + ...aiContentProperties(event), // Names the node in the trace tree. Provider-and-kind rather than the tool/feature name, because // that is what this function actually knows -- the feature lives in ai_usage_events, not here. $ai_span_name: `ai.${event.requestKind}/${nonBlank(event.provider) ?? "unknown"}`, @@ -566,5 +629,7 @@ export function resetPostHogForTest(): void { activeRelease = undefined; usingCentralPostHogKey = false; centralKeyAnonSecret = undefined; + aiContentCapture = false; + aiContentMaxChars = DEFAULT_AI_CONTENT_MAX_CHARS; resetRedactionScrubForTest(); } diff --git a/test/unit/docs-selfhost-posthog-observability.test.ts b/test/unit/docs-selfhost-posthog-observability.test.ts index 39d0ef3f3..7bd4d31aa 100644 --- a/test/unit/docs-selfhost-posthog-observability.test.ts +++ b/test/unit/docs-selfhost-posthog-observability.test.ts @@ -40,6 +40,15 @@ describe("self-host PostHog observability docs (#8287)", () => { expect(operations).toContain("-default"); }); + it("documents content capture as opt-in, with the real env var names and the privacy warning (#10218)", () => { + // The page previously promised metadata-only unconditionally. That is now conditional, so the page must + // say WHICH var changes it and what enabling it actually sends -- a self-hoster's private diffs. + expect(operations).toContain("LOOPOVER_POSTHOG_AI_CONTENT"); + expect(operations).toContain("LOOPOVER_POSTHOG_AI_CONTENT_MAX_CHARS"); + expect(operations).toContain("Off by default"); + expect(operations).toContain("private source code leaving your infrastructure"); + }); + it("documents the cron-monitor heartbeat replacement with the real exported event name", () => { expect(operations).toContain("Cron Monitors"); expect(operations).toContain(POSTHOG_MONITOR_HEARTBEAT_EVENT); diff --git a/test/unit/selfhost-ai.test.ts b/test/unit/selfhost-ai.test.ts index 3f231bf75..734dd0641 100644 --- a/test/unit/selfhost-ai.test.ts +++ b/test/unit/selfhost-ai.test.ts @@ -740,6 +740,75 @@ describe("$ai_generation PostHog capture at the chain chokepoint (#8296)", () => expect(posthogMocks.capture).not.toHaveBeenCalled(); }); + // #10218: content capture. ai.ts always passes the prompt/completion; posthog.ts decides whether it is + // emitted, so these drive the real chain with the flag on. + it("captures the flattened prompt and the completion when content capture is enabled (#10218)", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key", LOOPOVER_POSTHOG_AI_CONTENT: "1" } as unknown as NodeJS.ProcessEnv); + const provider = { name: "ollama", ai: { run: async () => ({ response: "no blockers found" }) } }; + await createChainAi([provider]).run("m", { + messages: [ + { role: "system", content: "you are a reviewer" }, + { role: "user", content: "review this diff" }, + ], + }); + const { properties } = posthogMocks.capture.mock.calls[0]?.[0]; + expect(properties.$ai_input).toEqual([ + { role: "system", content: "you are a reviewer" }, + { role: "user", content: "review this diff" }, + ]); + expect(properties.$ai_output_choices).toEqual([{ role: "assistant", content: "no blockers found" }]); + }); + + it("drops an image block rather than base64-encoding it into the event (#10218)", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key", LOOPOVER_POSTHOG_AI_CONTENT: "1" } as unknown as NodeJS.ProcessEnv); + const provider = { name: "ollama", ai: { run: async () => ({ response: "ok" }) } }; + await createChainAi([provider]).run("m", { + messages: [ + { + role: "user", + content: [ + { type: "text", text: "describe this" }, + { type: "image", mimeType: "image/png", data: "AAAABBBBCCCC" }, + ], + }, + ], + }); + const captured = JSON.stringify(posthogMocks.capture.mock.calls[0]?.[0].properties.$ai_input); + expect(captured).toContain("describe this"); + expect(captured).not.toContain("AAAABBBBCCCC"); + }); + + it("reports an embedding request's raw texts as the input, under a synthetic user role (#10218)", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key", LOOPOVER_POSTHOG_AI_CONTENT: "1" } as unknown as NodeJS.ProcessEnv); + const provider = { name: "ollama", ai: { run: async () => ({ data: [[0.1]] }) } }; + await createChainAi([provider]).run("m", { text: ["chunk one", "chunk two"] }); + const { properties } = posthogMocks.capture.mock.calls[0]?.[0]; + expect(properties.$ai_input).toEqual([ + { role: "user", content: "chunk one" }, + { role: "user", content: "chunk two" }, + ]); + // An embedding has no completion to report. + expect("$ai_output_choices" in properties).toBe(false); + }); + + it("captures the prompt on the FAILURE path, where it is what makes the failure diagnosable (#10218)", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key", LOOPOVER_POSTHOG_AI_CONTENT: "1" } as unknown as NodeJS.ProcessEnv); + const provider = { name: "claude-code", ai: { run: async () => { throw new Error("claude_stalled_no_output"); } } }; + await expect(createChainAi([provider]).run("m", { prompt: "review this" })).rejects.toThrow(/stalled/); + const generation = posthogMocks.capture.mock.calls.find((call) => call[0].event === "$ai_generation")?.[0]; + expect(generation.properties.$ai_input).toEqual([{ role: "user", content: "review this" }]); + expect("$ai_output_choices" in generation.properties).toBe(false); + }); + + it("emits NO content when the operator has not opted in, even though ai.ts always passes it (#10218)", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + const provider = { name: "ollama", ai: { run: async () => ({ response: "secret completion" }) } }; + await createChainAi([provider]).run("m", { prompt: "private diff" }); + const keys = Object.keys(posthogMocks.capture.mock.calls[0]?.[0].properties); + expect(keys).not.toContain("$ai_input"); + expect(keys).not.toContain("$ai_output_choices"); + }); + it("stays a no-op end-to-end when PostHog is unconfigured (no posthog-node client constructed)", async () => { const provider = { name: "posthog-unconfigured-provider", ai: { run: async () => ({ response: "ok" }) } }; await createChainAi([provider]).run("m", { prompt: "x" }); diff --git a/test/unit/selfhost-posthog.test.ts b/test/unit/selfhost-posthog.test.ts index a9df1e2f6..d96ce5de9 100644 --- a/test/unit/selfhost-posthog.test.ts +++ b/test/unit/selfhost-posthog.test.ts @@ -723,14 +723,79 @@ describe("capturePostHogAiGeneration (#8296)", () => { expect(mocks.capture.mock.calls[0]?.[0].properties.$ai_error).toBe("just a string"); }); - it("never carries prompt/response content -- no field beyond model/provider ids", async () => { + it("carries NO prompt/response content by default, even when the caller supplies it (#10218)", async () => { + // The gate lives in posthog.ts, not at the call sites: ai.ts always passes the content, so this is the + // single place that decides. An operator who has not opted in is byte-identical to before #10218. await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); - capturePostHogAiGeneration(BASE); + capturePostHogAiGeneration({ ...BASE, input: [{ role: "user", content: "review this diff" }], outputText: "looks good" }); const keys = Object.keys(mocks.capture.mock.calls[0]?.[0].properties); expect(keys).not.toContain("$ai_input"); expect(keys).not.toContain("$ai_output_choices"); }); + it("populates $ai_input/$ai_output_choices once LOOPOVER_POSTHOG_AI_CONTENT is set (#10218)", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key", LOOPOVER_POSTHOG_AI_CONTENT: "1" } as unknown as NodeJS.ProcessEnv); + capturePostHogAiGeneration({ + ...BASE, + input: [{ role: "system", content: "you are a reviewer" }, { role: "user", content: "review this diff" }], + outputText: "looks good", + }); + const { properties } = mocks.capture.mock.calls[0]?.[0]; + expect(properties.$ai_input).toEqual([ + { role: "system", content: "you are a reviewer" }, + { role: "user", content: "review this diff" }, + ]); + // PostHog's shape is a list of choices; the self-host providers are single-completion, so exactly one. + expect(properties.$ai_output_choices).toEqual([{ role: "assistant", content: "looks good" }]); + }); + + it("omits each content field independently when it has nothing to say", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key", LOOPOVER_POSTHOG_AI_CONTENT: "true" } as unknown as NodeJS.ProcessEnv); + // A failure has a prompt but no completion; an empty prompt list must not become an empty array. + capturePostHogAiGeneration({ ...BASE, input: [{ role: "user", content: "x" }] }); + capturePostHogAiGeneration({ ...BASE, input: [], outputText: "" }); + const withPrompt = mocks.capture.mock.calls[0]?.[0].properties; + expect(withPrompt.$ai_input).toHaveLength(1); + expect("$ai_output_choices" in withPrompt).toBe(false); + const withNeither = mocks.capture.mock.calls[1]?.[0].properties; + expect("$ai_input" in withNeither).toBe(false); + expect("$ai_output_choices" in withNeither).toBe(false); + }); + + it("truncates content at the cap and marks that it happened (#10218)", async () => { + await initPostHog({ + POSTHOG_API_KEY: "phc_test_key", + LOOPOVER_POSTHOG_AI_CONTENT: "on", + LOOPOVER_POSTHOG_AI_CONTENT_MAX_CHARS: "10", + } as unknown as NodeJS.ProcessEnv); + capturePostHogAiGeneration({ ...BASE, input: [{ role: "user", content: "a".repeat(50) }], outputText: "b".repeat(50) }); + const { properties } = mocks.capture.mock.calls[0]?.[0]; + // A reader must never mistake a clipped prompt for the whole one. + expect((properties.$ai_input as Array<{ content: string }>)[0]?.content).toBe(`${"a".repeat(10)}…[truncated]`); + expect((properties.$ai_output_choices as Array<{ content: string }>)[0]?.content).toBe(`${"b".repeat(10)}…[truncated]`); + }); + + it("leaves content at or under the cap untouched", async () => { + await initPostHog({ + POSTHOG_API_KEY: "phc_test_key", + LOOPOVER_POSTHOG_AI_CONTENT: "yes", + LOOPOVER_POSTHOG_AI_CONTENT_MAX_CHARS: "10", + } as unknown as NodeJS.ProcessEnv); + capturePostHogAiGeneration({ ...BASE, input: [{ role: "user", content: "a".repeat(10) }] }); + expect((mocks.capture.mock.calls[0]?.[0].properties.$ai_input as Array<{ content: string }>)[0]?.content).toBe("a".repeat(10)); + }); + + it("captured content still goes through the same before_send redaction as every other field", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key", LOOPOVER_POSTHOG_AI_CONTENT: "1" } as unknown as NodeJS.ProcessEnv); + const leaked = ["ghp", "abcdefghijklmnopqrst123456"].join("_"); + capturePostHogAiGeneration({ ...BASE, input: [{ role: "user", content: `token is ${leaked}` }] }); + // scrubPostHogEvent is posthog-node's before_send, so assert on it directly the way the client would. + const event = scrubPostHogEvent(mocks.capture.mock.calls[0]?.[0] as never) as unknown as { properties: Record }; + const captured = JSON.stringify(event.properties.$ai_input); + expect(captured).not.toContain(leaked); + expect(captured).toContain("[redacted]"); + }); + it("mints a fresh, real UUID trace id on every call", async () => { await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); capturePostHogAiGeneration(BASE);