JSONbored · loopover-orb · Jul 31, 2026 · Jul 31, 2026
@@ -676,7 +676,7 @@ three standalone bindings (`AI_EMBED`, `AI_VISION`, `AI_ADVISORY`).
     {
       title: "Content policy",
       description:
-        "Metadata only — never the prompt or completion text. $ai_generation's own optional content fields ($ai_input/$ai_output_choices) are never populated, the same redaction posture as every other capture path on this page.",
+        "Metadata only by default — no prompt or completion text leaves the box unless you explicitly opt in. $ai_generation's optional content fields ($ai_input/$ai_output_choices) stay unpopulated unless LOOPOVER_POSTHOG_AI_CONTENT is set, so upgrading never starts shipping content on its own.",
     },
     {
       title: "Model labelling",
@@ -691,6 +691,28 @@ three standalone bindings (`AI_EMBED`, `AI_VISION`, `AI_ADVISORY`).
   ]}
 />
 
+### Capturing prompts and completions (`LOOPOVER_POSTHOG_AI_CONTENT`)
+
+**Off by default, and deliberately so.** With it unset, no prompt, diff or model completion ever leaves the
+box — only the metadata listed above. Set `LOOPOVER_POSTHOG_AI_CONTENT=1` to populate `$ai_input` and
+`$ai_output_choices`, which is what unlocks PostHog's conversation view, sentiment, evaluations and
+LLM-judge surfaces. Read this before turning it on:
+
+- **Your PR diffs and review text go to PostHog.** The prompt for a review contains the diff and the
+  assembled repo context. On a private repository that is private source code leaving your infrastructure.
+  Only enable it on a project you are willing to have that content in.
+- **The same redaction still applies.** Captured content passes through the identical `before_send` scrub
+  (`src/selfhost/redaction-scrub.ts`) as every other field — credential shapes, JWTs and query-string
+  secrets are redacted from it. That is a backstop, not a licence: it removes credential shapes, not the
+  source code itself.
+- **Content is truncated.** Each message is capped at 10,000 characters
+  (`LOOPOVER_POSTHOG_AI_CONTENT_MAX_CHARS`) and marked `…[truncated]`. A review prompt can exceed 300,000
+  characters, so an uncapped capture would blow past PostHog's payload limits and be dropped entirely. A
+  non-numeric, zero or negative override falls back to the default rather than disabling the cap.
+- **Images are never captured.** An image content block is dropped, not base64-encoded into the event.
+- **On a failure, the prompt is captured but there is no completion** — a `claude_stalled_no_output` or a
+  context-length rejection is only diagnosable against the input that produced it.
+
 A CLI-subscription provider declining an embedding request (`claude_code_no_embed` / `codex_no_embed`)
 is routing working as designed — it is how an embed reaches an embed-capable provider — so it is exempt
 from both captures. A chain with **no** embed-capable member at all is a real misconfiguration, but it

@@ -321,6 +321,14 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [
     name: "LOOPOVER_METRICS_REPO_LABELS",
     firstReference: "src/server.ts",
   },
+  {
+    name: "LOOPOVER_POSTHOG_AI_CONTENT",
+    firstReference: "src/selfhost/posthog.ts",
+  },
+  {
+    name: "LOOPOVER_POSTHOG_AI_CONTENT_MAX_CHARS",
+    firstReference: "src/selfhost/posthog.ts",
+  },
   {
     name: "LOOPOVER_PUBLIC_SCORE_TERMS_ALLOWED_REPOS",
     firstReference: "src/selfhost/inert-config.ts",
@@ -801,6 +809,8 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [
   "| `LOOPOVER_LEDGER_UNCHAINED_WAIVER` | `src/selfhost/preflight.ts` |",
   "| `LOOPOVER_MCP_TOKEN` | `src/selfhost/preflight.ts` |",
   "| `LOOPOVER_METRICS_REPO_LABELS` | `src/server.ts` |",
+  "| `LOOPOVER_POSTHOG_AI_CONTENT` | `src/selfhost/posthog.ts` |",
+  "| `LOOPOVER_POSTHOG_AI_CONTENT_MAX_CHARS` | `src/selfhost/posthog.ts` |",
   "| `LOOPOVER_PUBLIC_SCORE_TERMS_ALLOWED_REPOS` | `src/selfhost/inert-config.ts` |",
   "| `LOOPOVER_PUBLIC_STATS_REPOS` | `src/selfhost/inert-config.ts` |",
   "| `LOOPOVER_REPO_CONFIG_DIR` | `src/server.ts` |",

@@ -1559,6 +1559,19 @@ function isExpectedEmbeddingRoutingError(options: AiRunOptions, error: unknown):
   return requestKind(options) === "embedding" && EXPECTED_EMBEDDING_ROUTING_ERRORS.has(errorMessage(error));
 }
 
+/** The prompt, flattened to the `{role, content}` list PostHog's `$ai_input` expects (#10218).
+ *
+ *  Reuses `contentText`, so an image block is dropped rather than base64-encoded into a telemetry event --
+ *  the same projection the subscription CLIs already apply when building their stdin prompt, so what is
+ *  captured is what the provider was actually given. An embedding request has no messages at all; its
+ *  inputs are the raw texts, reported under the same synthetic `user` role so one property shape covers
+ *  both request kinds. Capture is gated in posthog.ts, so this always builds and is simply ignored when
+ *  the operator has not opted in. */
+function aiContentInput(options: AiRunOptions): Array<{ role: string; content: string }> {
+  if (Array.isArray(options.text)) return options.text.map((text) => ({ role: "user", content: text }));
+  return toMessages(options).map((message) => ({ role: message.role, content: contentText(message.content) }));
+}
+
 async function runProviderWithOtel(
   provider: { name: string; ai: SelfHostAi },
   model: string,
@@ -1619,6 +1632,8 @@ async function runProviderWithOtel(
       totalCostUsd: usage?.costUsd,
       effort: usage?.effort,
       context: { repo: options.repoFullName, pullNumber: options.pullNumber },
+      input: aiContentInput(options),
+      outputText: result.response,
     });
     return usage ? { ...result, usage } : result;
   } catch (error) {
@@ -1635,6 +1650,9 @@ async function runProviderWithOtel(
       isError: true,
       error,
       context: { repo: options.repoFullName, pullNumber: options.pullNumber },
+      // The prompt is what a failure needs most: a `claude_stalled_no_output` or a context-length rejection
+      // is only diagnosable against the input that produced it. There is no completion to report.
+      input: aiContentInput(options),
     });
     incr("loopover_ai_provider_failures_total", { provider: provider.name });
     incr("loopover_ai_provider_request_errors_total", { provider: provider.name, request_kind: requestKindLabel });
@@ -1809,6 +1827,8 @@ export function withAiGenerationCapture(providerName: string, ai: SelfHostAi): S
           outputTokens: usage?.outputTokens,
           totalCostUsd: usage?.costUsd,
           effort: usage?.effort,
+          input: aiContentInput(options),
+          outputText: result.response,
         });
         return usage ? { ...result, usage } : result;
       } catch (error) {
@@ -1819,6 +1839,7 @@ export function withAiGenerationCapture(providerName: string, ai: SelfHostAi): S
           latencyMs: Date.now() - startedAtMs,
           isError: true,
           error,
+          input: aiContentInput(options),
         });
         throw error;
       }

@@ -74,6 +74,37 @@ function processEnvString(env: NodeJS.ProcessEnv, name: string): string | undefi
   return nonBlank(env[name]);
 }
 
+/** Truthy-string env flag, matching the repo-wide convention for a flag-gated capability that ships OFF. */
+const TRUE_ENV_VALUES = new Set(["1", "true", "yes", "on"]);
+
+/** Default ceiling on a single captured message's characters (#10218). A review prompt carries up to 120k
+ *  chars of diff plus a 240k-char aggregate context budget, so an uncapped capture would push events past
+ *  PostHog's own payload limits and be dropped wholesale -- a cap is what makes this survivable at all. */
+const DEFAULT_AI_CONTENT_MAX_CHARS = 10_000;
+
+/** #10218: whether `$ai_input`/`$ai_output_choices` are populated at all. OFF unless
+ *  `LOOPOVER_POSTHOG_AI_CONTENT` is explicitly truthy. Self-host-only, read off real process.env, matching
+ *  this file's precedent for every other self-host-exclusive knob. */
+let aiContentCapture = false;
+let aiContentMaxChars = DEFAULT_AI_CONTENT_MAX_CHARS;
+
+function resolveAiContentCapture(env: NodeJS.ProcessEnv): boolean {
+  return TRUE_ENV_VALUES.has((env.LOOPOVER_POSTHOG_AI_CONTENT ?? "").trim().toLowerCase());
+}
+
+/** Per-message character ceiling. A non-numeric, zero, negative or fractional override falls back to the
+ *  default rather than disabling the cap -- an operator typo must never become "send the whole diff". */
+function resolveAiContentMaxChars(env: NodeJS.ProcessEnv): number {
+  const raw = Number(env.LOOPOVER_POSTHOG_AI_CONTENT_MAX_CHARS);
+  return Number.isInteger(raw) && raw > 0 ? raw : DEFAULT_AI_CONTENT_MAX_CHARS;
+}
+
+/** Truncate to the configured ceiling, marking that it happened so a reader never mistakes a clipped prompt
+ *  for the whole one. */
+function boundedContent(value: string): string {
+  return value.length <= aiContentMaxChars ? value : `${value.slice(0, aiContentMaxChars)}…[truncated]`;
+}
+
 /** Resolve the PostHog release id: explicit override first, then the image-baked self-host version, matching
  *  {@link resolveSentryRelease}'s identical precedence in sentry.ts. */
 export function resolvePostHogRelease(env: NodeJS.ProcessEnv): string | undefined {
@@ -181,6 +212,11 @@ export async function initPostHog(env: NodeJS.ProcessEnv): Promise<boolean> {
   usingCentralPostHogKey = !explicitKey;
   await loadNodeHasher();
   const { PostHog } = await import("posthog-node");
+  // #10218: content capture is OFF unless the operator turns it on. Upgrading an ORB must never start
+  // shipping private PR diffs and model completions to a vendor as a side effect of a version bump, so the
+  // default stays metadata-only exactly as before -- see resolveAiContentCapture.
+  aiContentCapture = resolveAiContentCapture(env);
+  aiContentMaxChars = resolveAiContentMaxChars(env);
   posthogEnvironment = processEnvString(env, "POSTHOG_ENVIRONMENT") ?? "production";
   activeRelease = resolvePostHogRelease(env);
   const host = processEnvString(env, "POSTHOG_HOST") ?? DEFAULT_POSTHOG_HOST;
@@ -406,8 +442,34 @@ export type PostHogAiGenerationEvent = {
   /** Raw caught value on the error path (mirrors capturePostHogError's own `error` param) -- never a
    *  caller-preformatted string. Ignored when `isError` is false. */
   error?: unknown;
+  /** #10218: the prompt messages, already flattened to plain text by the caller (an image block has no
+   *  place in a telemetry event and is dropped upstream). IGNORED unless content capture is enabled --
+   *  passing it is always safe, the gate lives here rather than at every call site. */
+  input?: ReadonlyArray<{ role: string; content: string }> | undefined;
+  /** #10218: the model's completion text. Same gating as {@link PostHogAiGenerationEvent.input}. */
+  outputText?: string | undefined;
 };
 
+/** Build the `$ai_input`/`$ai_output_choices` properties PostHog's LLM-analytics views read (#10218).
+ *  Returns an EMPTY bag when content capture is off, which is the default and the pre-#10218 behavior
+ *  byte-for-byte. Every string is bounded here; the vendor-bound redaction pass still runs afterwards over
+ *  the whole properties bag via `before_send` (scrubPostHogEvent -> scrubRecord), so captured content goes
+ *  through exactly the same credential/vocabulary scrubbing as every other field in this file -- it is not
+ *  a second, weaker path. */
+function aiContentProperties(event: PostHogAiGenerationEvent): Record<string, unknown> {
+  if (!aiContentCapture) return {};
+  const properties: Record<string, unknown> = {};
+  if (event.input && event.input.length > 0) {
+    properties.$ai_input = event.input.map((message) => ({ role: message.role, content: boundedContent(message.content) }));
+  }
+  // PostHog's shape is a list of choices; the self-host providers are all single-completion, so this is
+  // always a one-element list rather than a fabricated multi-choice response.
+  if (event.outputText !== undefined && event.outputText !== "") {
+    properties.$ai_output_choices = [{ role: "assistant", content: boundedContent(event.outputText) }];
+  }
+  return properties;
+}
+
 /** Capture one AI provider attempt as PostHog's `$ai_generation` (`$ai_embedding` for an embedding
  *  request) event (#8296). No-op when PostHog is off -- same contract as every other capture function in
  *  this file. Unlike {@link capturePostHogError}, this never gates on POSTHOG_MIN_SEVERITY: a successful
@@ -451,6 +513,7 @@ export function capturePostHogAiGeneration(event: PostHogAiGenerationEvent): voi
   const properties: Record<string, unknown> = {
     ...operational,
     ...aiTraceProperties(operational),
+    ...aiContentProperties(event),
     // Names the node in the trace tree. Provider-and-kind rather than the tool/feature name, because
     // that is what this function actually knows -- the feature lives in ai_usage_events, not here.
     $ai_span_name: `ai.${event.requestKind}/${nonBlank(event.provider) ?? "unknown"}`,
@@ -566,5 +629,7 @@ export function resetPostHogForTest(): void {
   activeRelease = undefined;
   usingCentralPostHogKey = false;
   centralKeyAnonSecret = undefined;
+  aiContentCapture = false;
+  aiContentMaxChars = DEFAULT_AI_CONTENT_MAX_CHARS;
   resetRedactionScrubForTest();
 }
@@ -40,6 +40,15 @@ describe("self-host PostHog observability docs (#8287)", () => {
     expect(operations).toContain("<provider>-default");
   });
 
+  it("documents content capture as opt-in, with the real env var names and the privacy warning (#10218)", () => {
+    // The page previously promised metadata-only unconditionally. That is now conditional, so the page must
+    // say WHICH var changes it and what enabling it actually sends -- a self-hoster's private diffs.
+    expect(operations).toContain("LOOPOVER_POSTHOG_AI_CONTENT");
+    expect(operations).toContain("LOOPOVER_POSTHOG_AI_CONTENT_MAX_CHARS");
+    expect(operations).toContain("Off by default");
+    expect(operations).toContain("private source code leaving your infrastructure");
+  });
+
   it("documents the cron-monitor heartbeat replacement with the real exported event name", () => {
     expect(operations).toContain("Cron Monitors");
     expect(operations).toContain(POSTHOG_MONITOR_HEARTBEAT_EVENT);

@@ -740,6 +740,75 @@ describe("$ai_generation PostHog capture at the chain chokepoint (#8296)", () =>
     expect(posthogMocks.capture).not.toHaveBeenCalled();
   });
 
+  // #10218: content capture. ai.ts always passes the prompt/completion; posthog.ts decides whether it is
+  // emitted, so these drive the real chain with the flag on.
+  it("captures the flattened prompt and the completion when content capture is enabled (#10218)", async () => {
+    await initPostHog({ POSTHOG_API_KEY: "phc_test_key", LOOPOVER_POSTHOG_AI_CONTENT: "1" } as unknown as NodeJS.ProcessEnv);
+    const provider = { name: "ollama", ai: { run: async () => ({ response: "no blockers found" }) } };
+    await createChainAi([provider]).run("m", {
+      messages: [
+        { role: "system", content: "you are a reviewer" },
+        { role: "user", content: "review this diff" },
+      ],
+    });
+    const { properties } = posthogMocks.capture.mock.calls[0]?.[0];
+    expect(properties.$ai_input).toEqual([
+      { role: "system", content: "you are a reviewer" },
+      { role: "user", content: "review this diff" },
+    ]);
+    expect(properties.$ai_output_choices).toEqual([{ role: "assistant", content: "no blockers found" }]);
+  });
+
+  it("drops an image block rather than base64-encoding it into the event (#10218)", async () => {
+    await initPostHog({ POSTHOG_API_KEY: "phc_test_key", LOOPOVER_POSTHOG_AI_CONTENT: "1" } as unknown as NodeJS.ProcessEnv);
+    const provider = { name: "ollama", ai: { run: async () => ({ response: "ok" }) } };
+    await createChainAi([provider]).run("m", {
+      messages: [
+        {
+          role: "user",
+          content: [
+            { type: "text", text: "describe this" },
+            { type: "image", mimeType: "image/png", data: "AAAABBBBCCCC" },
+          ],
+        },
+      ],
+    });
+    const captured = JSON.stringify(posthogMocks.capture.mock.calls[0]?.[0].properties.$ai_input);
+    expect(captured).toContain("describe this");
+    expect(captured).not.toContain("AAAABBBBCCCC");
+  });
+
+  it("reports an embedding request's raw texts as the input, under a synthetic user role (#10218)", async () => {
+    await initPostHog({ POSTHOG_API_KEY: "phc_test_key", LOOPOVER_POSTHOG_AI_CONTENT: "1" } as unknown as NodeJS.ProcessEnv);
+    const provider = { name: "ollama", ai: { run: async () => ({ data: [[0.1]] }) } };
+    await createChainAi([provider]).run("m", { text: ["chunk one", "chunk two"] });
+    const { properties } = posthogMocks.capture.mock.calls[0]?.[0];
+    expect(properties.$ai_input).toEqual([
+      { role: "user", content: "chunk one" },
+      { role: "user", content: "chunk two" },
+    ]);
+    // An embedding has no completion to report.
+    expect("$ai_output_choices" in properties).toBe(false);
+  });
+
+  it("captures the prompt on the FAILURE path, where it is what makes the failure diagnosable (#10218)", async () => {
+    await initPostHog({ POSTHOG_API_KEY: "phc_test_key", LOOPOVER_POSTHOG_AI_CONTENT: "1" } as unknown as NodeJS.ProcessEnv);
+    const provider = { name: "claude-code", ai: { run: async () => { throw new Error("claude_stalled_no_output"); } } };
+    await expect(createChainAi([provider]).run("m", { prompt: "review this" })).rejects.toThrow(/stalled/);
+    const generation = posthogMocks.capture.mock.calls.find((call) => call[0].event === "$ai_generation")?.[0];
+    expect(generation.properties.$ai_input).toEqual([{ role: "user", content: "review this" }]);
+    expect("$ai_output_choices" in generation.properties).toBe(false);
+  });
+
+  it("emits NO content when the operator has not opted in, even though ai.ts always passes it (#10218)", async () => {
+    await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv);
+    const provider = { name: "ollama", ai: { run: async () => ({ response: "secret completion" }) } };
+    await createChainAi([provider]).run("m", { prompt: "private diff" });
+    const keys = Object.keys(posthogMocks.capture.mock.calls[0]?.[0].properties);
+    expect(keys).not.toContain("$ai_input");
+    expect(keys).not.toContain("$ai_output_choices");
+  });
+
   it("stays a no-op end-to-end when PostHog is unconfigured (no posthog-node client constructed)", async () => {
     const provider = { name: "posthog-unconfigured-provider", ai: { run: async () => ({ response: "ok" }) } };
     await createChainAi([provider]).run("m", { prompt: "x" });