diff --git a/apps/loopover-ui/content/docs/self-hosting-operations.mdx b/apps/loopover-ui/content/docs/self-hosting-operations.mdx index 052ca3516..17de50324 100644 --- a/apps/loopover-ui/content/docs/self-hosting-operations.mdx +++ b/apps/loopover-ui/content/docs/self-hosting-operations.mdx @@ -678,9 +678,24 @@ three standalone bindings (`AI_EMBED`, `AI_VISION`, `AI_ADVISORY`). description: "Metadata only — never the prompt or completion text. $ai_generation's own optional content fields ($ai_input/$ai_output_choices) are never populated, the same redaction posture as every other capture path on this page.", }, + { + title: "Model labelling", + description: + "$ai_model is always the model the provider actually resolved — on the failure path too. The core passes a Workers-AI model id that every self-host provider discards, so it is resolved to the real one (or -default) before capture and never reported verbatim.", + }, + { + title: "Degraded requests", + description: + "selfhost_ai_degraded is emitted for a request that reached NO model: reason=circuit_open (a provider skipped while its breaker is in cooldown) or reason=chain_exhausted (every provider in AI_PROVIDER threw). It shares the $ai_trace_id of the surrounding attempts, and is deliberately NOT an $ai_generation — no model ran, so none is credited with the failure and the spend/token aggregates stay a record of real calls.", + }, ]} /> +A CLI-subscription provider declining an embedding request (`claude_code_no_embed` / `codex_no_embed`) +is routing working as designed — it is how an embed reaches an embed-capable provider — so it is exempt +from both captures. A chain with **no** embed-capable member at all is a real misconfiguration, but it +is reported once at boot by the RAG embed preflight rather than once per chunk batch. + ## PostHog alert classes and runbook Tune PostHog insight alerts for **persistent failure classes**, not one-off diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts index e9d4cd3f2..a779d20ef 100644 --- a/src/selfhost/ai.ts +++ b/src/selfhost/ai.ts @@ -14,7 +14,7 @@ export { assertNoLegacySharedAiEnv } from "./ai-config"; import { wasLoadedFromFile } from "./file-sourced-secrets"; import { getProviderCredentialResolver } from "./provider-credential-registry"; import { incr, observe } from "./metrics"; -import { capturePostHogAiGeneration, type PostHogAiGenerationRequestKind } from "./posthog"; +import { capturePostHogAiDegradation, capturePostHogAiGeneration, type PostHogAiGenerationRequestKind } from "./posthog"; import { withReviewSpan } from "./tracing"; import { delimiter } from "node:path"; @@ -90,6 +90,12 @@ interface AiRunOptions { export type AiResult = { response?: string; data?: number[][]; usage?: AiUsage }; export interface SelfHostAi { run(model: string, options: AiRunOptions): Promise; + /** The model id this adapter's own `run()` WOULD resolve for the same arguments (#10186). Telemetry needs + * this on the FAILURE path, where there is no `usage.model` to read and the raw `model` argument is a + * request-layer placeholder `resolveModel` throws away — see {@link resolveTelemetryModel}. Optional so a + * test double or a future adapter stays a valid `SelfHostAi`; implementers must derive it from the same + * expression `run()` uses, never a parallel copy that can drift. */ + telemetryModel?(model: string, options: AiRunOptions): string; } function toMessages(options: AiRunOptions): Array<{ role: string; content: string | AiContentBlock[] }> { @@ -144,6 +150,24 @@ export function resolveModel(configured: string | undefined, passed: string, pro return providerDefault; } +/** The model label telemetry must report for one provider attempt, on the SUCCESS and the FAILURE path alike + * (#10186). The `model` argument is a REQUEST-layer id: the core passes a Workers-AI model that + * {@link resolveModel} discards for every self-host provider, so reporting it verbatim labels a failure with a + * model that never ran — which is exactly what made a breakdown by model show `@cf/*` ids at a 100% error rate + * on a deployment that has never called Workers AI, while the real failing providers read 0%. + * + * The adapter's own {@link SelfHostAi.telemetryModel} is authoritative because it mirrors that adapter's own + * resolution. Without one (a test double), fall back to the passed id — but never to a `@cf/` placeholder, + * which has no live binding anywhere (`src/services/ai-summaries.ts`) and is wrong by construction. The last + * resort is `-default` rather than a bare "default": codex resolves to `""` on purpose (the CLI picks + * the account default), and `codex-default` still says WHICH provider's default it was. */ +export function resolveTelemetryModel(providerName: string, ai: SelfHostAi, model: string, options: AiRunOptions): string { + const resolved = ai.telemetryModel?.(model, options)?.trim(); + if (resolved) return resolved; + const passed = model.trim(); + return passed && !passed.startsWith("@cf/") ? passed : `${providerName}-default`; +} + function firstConfigured(...values: Array): string | undefined { return values.find((value) => value !== undefined && value.trim() !== ""); } @@ -337,11 +361,22 @@ export function createOpenAiCompatibleAi(opts: { }): SelfHostAi { const base = opts.baseUrl.replace(/\/+$/, ""); const headers = (): Record => ({ "content-type": "application/json", ...(opts.apiKey ? { authorization: `Bearer ${opts.apiKey}` } : {}) }); + // #10186: the two model resolutions this adapter performs, hoisted so `run` and `telemetryModel` below share + // ONE expression each — a telemetry label that re-derived the model separately could drift from the model the + // request actually carried, which is the very failure this issue is about. + const resolvedEmbedModel = (): string => opts.embedModel ?? "bge-m3"; + const resolvedChatModel = (model: string, options: AiRunOptions): string => { + const repoOverride = opts.providerName ? resolveOpenAiCompatibleRepoOverride(opts.providerName, options) : undefined; + return resolveModel(firstConfigured(repoOverride, opts.model), model, opts.defaultModel ?? DEFAULT_OPENAI_COMPATIBLE_CHAT_MODEL); + }; return { + // Branches on the SAME `options.text` discriminator `run` does, so an embed request is labelled with the + // embed model (`bge-m3:latest`) rather than the chat model it never touches. + telemetryModel: (model, options) => (Array.isArray(options.text) ? resolvedEmbedModel() : resolvedChatModel(model, options)), async run(model, options) { // Embedding request — the core's embedTexts passes { text: string[] }; route to /embeddings (for RAG). if (Array.isArray(options.text)) { - const embedModel = opts.embedModel ?? "bge-m3"; + const embedModel = resolvedEmbedModel(); if (options.text.length === 0) return { data: [], usage: buildAiUsage({ provider: opts.providerName, model: embedModel, inputTokens: 0, totalTokens: 0 }) }; const res = await fetch(`${base}/embeddings`, { method: "POST", @@ -373,8 +408,7 @@ export function createOpenAiCompatibleAi(opts: { }), }; } - const repoOverride = opts.providerName ? resolveOpenAiCompatibleRepoOverride(opts.providerName, options) : undefined; - const resolvedModel = resolveModel(firstConfigured(repoOverride, opts.model), model, opts.defaultModel ?? DEFAULT_OPENAI_COMPATIBLE_CHAT_MODEL); + const resolvedModel = resolvedChatModel(model, options); const chatBody = (withResponseFormat: boolean): string => JSON.stringify({ model: resolvedModel, @@ -438,7 +472,11 @@ function toAnthropicMessageContent(content: string | AiContentBlock[]): string | * subscription path). The system message becomes the top-level `system` param; the rest map to user/assistant. */ export function createAnthropicAi(opts: { apiKey: string; model?: string | undefined; baseUrl?: string | undefined }): SelfHostAi { const base = (opts.baseUrl ?? "https://api.anthropic.com").replace(/\/+$/, ""); + // #10186: one expression, shared with `run`'s own `resolvedModel` below. + const resolvedAnthropicModel = (model: string, options: AiRunOptions): string => + resolveModel(firstConfigured(options.anthropicModel, opts.model), model, "claude-sonnet-5"); return { + telemetryModel: resolvedAnthropicModel, async run(model, options) { const msgs = toMessages(options); const system = @@ -451,7 +489,7 @@ export function createAnthropicAi(opts: { apiKey: string; model?: string | undef .map((m) => ({ role: m.role === "assistant" ? "assistant" : "user", content: toAnthropicMessageContent(m.content) })); // Repo override > construction-time env-resolved opts.model (#3902), same priority as the OpenAI-compatible // providers above and the CLI providers' claudeModel/codexModel. - const resolvedModel = resolveModel(firstConfigured(options.anthropicModel, opts.model), model, "claude-sonnet-5"); + const resolvedModel = resolvedAnthropicModel(model, options); const res = await fetch(`${base}/v1/messages`, { method: "POST", headers: { "content-type": "application/json", "x-api-key": opts.apiKey, "anthropic-version": "2023-06-01" }, @@ -1113,14 +1151,19 @@ async function resolveClaudeOauthToken(parentEnv: Record, spawnImpl?: SpawnFn): SelfHostAi { + // #10186: shared with `run`'s own `claudeModel` below so the telemetry label and the `--model` argument are + // the same value by construction. + const resolvedClaudeModel = (model: string, options: AiRunOptions): string => + resolveModel(configuredClaudeModel(parentEnv, options.claudeModel), model, "claude-sonnet-5"); return { + telemetryModel: resolvedClaudeModel, async run(model, options) { // Claude has no embeddings model (CLI or API), so REJECT an embed request and let the provider chain fall // through to an embed-capable provider (ollama/openai-compatible). Without this throw the chain would treat // claude's empty-prompt text answer as "success" and never reach the embed provider → RAG silently breaks. if (options.text) throw new Error("claude_code_no_embed"); const token = await resolveClaudeOauthToken(parentEnv); - const claudeModel = resolveModel(configuredClaudeModel(parentEnv, options.claudeModel), model, "claude-sonnet-5"); + const claudeModel = resolvedClaudeModel(model, options); const effort = resolveEffort(firstConfigured(options.claudeEffort, parentEnv.CLAUDE_AI_EFFORT)); const timeoutMs = resolveClaudeCliTimeoutMs(parentEnv, options.claudeTimeoutMs); // #4994: same clamp reasoning as createCodexAi's identical line — keeps the fast-fail deadline strictly @@ -1238,14 +1281,20 @@ export function createCodexAi( spawnImpl?: SpawnFn, authCheckImpl: (env: Record) => Promise = assertCodexAuthConfigured, ): SelfHostAi { + // #10186: shared with `run`'s own `codexModel` below. Resolves to "" when nothing is configured — which is + // meaningful here (no `--model` is passed and Codex picks the account default), and which + // `resolveTelemetryModel` renders as "codex-default" rather than losing the attribution to a bare "default". + const resolvedCodexModel = (model: string, options: AiRunOptions): string => + resolveModel(configuredCodexModel(parentEnv, options.codexModel), model, ""); return { + telemetryModel: resolvedCodexModel, async run(model, options) { // Codex is chat-only here — reject embed requests so the chain routes them to an embed-capable provider. if (options.text) throw new Error("codex_no_embed"); // codex 0.142+: `exec` is non-interactive — the old `--ask-for-approval` flag was REMOVED (passing it errors). // `--skip-git-repo-check` lets it run outside a git repo. Pass `--model` ONLY when one is explicitly // configured: otherwise Codex selects the account default. - const codexModel = resolveModel(configuredCodexModel(parentEnv, options.codexModel), model, ""); + const codexModel = resolvedCodexModel(model, options); const effort = resolveCodexEffort(firstConfigured(options.codexEffort, parentEnv.CODEX_AI_EFFORT)); const timeoutMs = resolveCodexCliTimeoutMs(parentEnv, options.codexTimeoutMs); // Clamp below timeoutMs so a misconfigured/low CODEX_AI_TIMEOUT_MS (its own floor is 30_000ms, the same as @@ -1425,6 +1474,10 @@ export function createChainAi(providers: Array<{ name: string; ai: SelfHostAi }> async run(model, options) { let lastError: unknown = new Error("no_ai_providers"); const failures: Array<{ provider: string; error: string }> = []; + // #10186: whether EVERY provider declined for an expected embedding-routing reason (claude_code_no_embed / + // codex_no_embed) rather than actually failing -- see the degradation capture below for why that case is + // exempt. Vacuously true for an empty chain, which is then excluded by the `failures.length` guard. + let everyFailureExpectedRouting = true; for (const p of providers) { try { const result = await runProviderWithOtel(p, model, options); @@ -1433,6 +1486,7 @@ export function createChainAi(providers: Array<{ name: string; ai: SelfHostAi }> } catch (error) { lastError = error; failures.push({ provider: p.name, error: errorMessage(error) }); + if (!isExpectedEmbeddingRoutingError(options, error)) everyFailureExpectedRouting = false; console.error( JSON.stringify({ level: "warn", @@ -1448,17 +1502,50 @@ export function createChainAi(providers: Array<{ name: string; ai: SelfHostAi }> } } recordAiProvidersExhausted(); + // The provider that ran LAST is the one `lastError` belongs to; with an empty chain there is none. + const lastProvider = providers[providers.length - 1]; + const exhaustedModel = lastProvider + ? resolveTelemetryModel(lastProvider.name, lastProvider.ai, model, options) + : model || "unknown"; console.error( JSON.stringify({ level: "error", event: "selfhost_ai_providers_exhausted", provider: failures.length === 1 ? failures[0]?.provider : undefined, - model: model || "default", + // #10186: was `model || "default"` -- the same discarded `@cf/` placeholder the $ai_generation sink + // reported, in the one log line that says the whole chain is down. + model: exhaustedModel, providers: failures.map((failure) => failure.provider), failures, error: errorMessage(lastError), }), ); + // #10186: that console line reaches PostHog only as an `$exception` (via forwardStructuredLogToPostHog), + // which the AI observability views do not read at all -- so a chain that is failing EVERY request was + // indistinguishable there from a chain nobody is calling. Emitted per exhausted request, alongside the + // per-provider $ai_generation failures, so "the chain gave up" is countable in its own right. + // + // Exempt for the same reason runProviderWithOtel's own capture is: a chain of CLI-subscription providers + // DECLINING an embed request (claude_code_no_embed) is routing working as designed, not a degradation -- + // it is how an embed request reaches an embed-capable provider. When such a chain has no embed-capable + // member at all the config really is broken, but that is a persistent, per-BATCH-of-chunks condition + // whose own fail-loud signal is shouldWarnRagEmbedUnavailable's boot warning, not one event per chunk. + // One exemption rule, applied at both sinks, rather than two that can disagree. + // + // `lastProvider` is the guard rather than `failures.length`: reaching here with a provider present means + // every one of them threw, so the two conditions are the same condition -- and narrowing on the provider + // keeps the capture from needing an unreachable "no provider" placeholder for its own label. + if (lastProvider && !everyFailureExpectedRouting) { + capturePostHogAiDegradation({ + reason: "chain_exhausted", + provider: lastProvider.name, + model: exhaustedModel, + requestKind: requestKind(options), + providers: failures.map((failure) => failure.provider), + error: lastError, + context: { repo: options.repoFullName, pullNumber: options.pullNumber }, + }); + } throw lastError instanceof Error ? lastError : new Error("all_ai_providers_failed"); }, }; @@ -1480,18 +1567,35 @@ async function runProviderWithOtel( const circuit = aiProviderCircuits.get(provider.name); if (circuit && circuit.cooldownUntil > Date.now()) { incr("loopover_ai_provider_circuit_open_total", { provider: provider.name }); - throw new Error( + const skipped = new Error( circuit.structural ? `circuit_open: provider "${provider.name}" has a structural config error (bad/missing credentials) — skipping until the cooldown expires; fix the underlying config, then restart` : `circuit_open: provider "${provider.name}" is in cooldown after ${AI_PROVIDER_FAILURE_THRESHOLD} consecutive failures — skipping this attempt`, ); + // #10186: a provider in cooldown previously left NOTHING in PostHog -- only a Prometheus counter and this + // throw. Once the breaker latches, that provider's $ai_generation stream simply stops, which reads exactly + // like "nobody called it" rather than "it is broken and being skipped". The Prometheus counter lives on the + // ORB's own stack; PostHog is where the AI observability views are, so it needs its own signal. + capturePostHogAiDegradation({ + reason: "circuit_open", + provider: provider.name, + model: resolveTelemetryModel(provider.name, provider.ai, model, options), + requestKind: requestKind(options), + error: skipped, + context: { repo: options.repoFullName, pullNumber: options.pullNumber }, + }); + throw skipped; } const requestKindLabel = requestKind(options); + // #10186: resolved ONCE, before the call, so the failure path below labels the attempt with the model that was + // actually about to run instead of the `@cf/` request-layer placeholder it used to report. The OTel span + // attribute reads from the same value — it carried the identical lie. + const telemetryModel = resolveTelemetryModel(provider.name, provider.ai, model, options); const startedAtMs = Date.now(); try { const result = await withReviewSpan( "selfhost.ai.provider", - { "ai.provider": provider.name, "ai.model": model || "default", "ai.request_kind": requestKindLabel }, + { "ai.provider": provider.name, "ai.model": telemetryModel, "ai.request_kind": requestKindLabel }, () => provider.ai.run(model, options), ); observe("loopover_ai_provider_request_duration_seconds", (Date.now() - startedAtMs) / 1000, { @@ -1500,11 +1604,13 @@ async function runProviderWithOtel( }); aiProviderCircuits.delete(provider.name); const usage = result.usage - ? { ...result.usage, provider: result.usage.provider ?? provider.name, model: result.usage.model ?? (model || "default") } + ? { ...result.usage, provider: result.usage.provider ?? provider.name, model: result.usage.model ?? telemetryModel } : undefined; capturePostHogAiGeneration({ provider: usage?.provider ?? provider.name, - model: usage?.model ?? (model || "default"), + // `usage.model` is what the provider REPORTED running and stays authoritative; `telemetryModel` is the + // same resolution computed up-front, so the two paths now agree for a given configured provider. + model: usage?.model ?? telemetryModel, requestKind: requestKindLabel, latencyMs: Date.now() - startedAtMs, isError: false, @@ -1523,7 +1629,7 @@ async function runProviderWithOtel( if (isExpectedEmbeddingRoutingError(options, error)) throw error; capturePostHogAiGeneration({ provider: provider.name, - model: model || "default", + model: telemetryModel, requestKind: requestKindLabel, latencyMs: Date.now() - startedAtMs, isError: true, @@ -1684,15 +1790,18 @@ export function withAiGenerationCapture(providerName: string, ai: SelfHostAi): S return { async run(model, options) { const requestKindLabel: PostHogAiGenerationRequestKind = requestKind(options); + // #10186: same up-front resolution runProviderWithOtel does — these bindings hit the identical failure-path + // mislabelling (AI_VISION's 2 failures reported the passed "visual-vision" id, not the model it resolves). + const telemetryModel = resolveTelemetryModel(providerName, ai, model, options); const startedAtMs = Date.now(); try { const result = await ai.run(model, options); const usage = result.usage - ? { ...result.usage, provider: result.usage.provider ?? providerName, model: result.usage.model ?? (model || "default") } + ? { ...result.usage, provider: result.usage.provider ?? providerName, model: result.usage.model ?? telemetryModel } : undefined; capturePostHogAiGeneration({ provider: usage?.provider ?? providerName, - model: usage?.model ?? (model || "default"), + model: usage?.model ?? telemetryModel, requestKind: requestKindLabel, latencyMs: Date.now() - startedAtMs, isError: false, @@ -1705,7 +1814,7 @@ export function withAiGenerationCapture(providerName: string, ai: SelfHostAi): S } catch (error) { capturePostHogAiGeneration({ provider: providerName, - model: model || "default", + model: telemetryModel, requestKind: requestKindLabel, latencyMs: Date.now() - startedAtMs, isError: true, diff --git a/src/selfhost/posthog.ts b/src/selfhost/posthog.ts index 6b1b08c54..8abc9ceb7 100644 --- a/src/selfhost/posthog.ts +++ b/src/selfhost/posthog.ts @@ -415,28 +415,42 @@ export type PostHogAiGenerationEvent = { * exception by the caller's own existing `selfhost_ai_provider_failed` log line (forwarded via * {@link forwardStructuredLogToPostHog}) -- this event exists for spend/latency/failure ANALYTICS, a * parallel concern to error tracking, not a substitute gate for it. */ +/** #10185: the OTel trace a capture already runs inside IS the trace PostHog should group by. + * operationalProperties has computed it (as plain `trace_id`/`span_id`) since #8296, but + * {@link capturePostHogAiGeneration} then minted a fresh randomUUID() for $ai_trace_id and threw it away -- + * so every generation became its own single-event trace and the review pipeline that produced them was + * never visible as one thing (13,428 events across 13,428 traces on the live project). + * + * withReviewPipelineSpan (./review-tracing.ts) wraps a whole review, so every provider attempt inside it -- + * both dual-review legs, every retry, every self-consistency run, the RAG embeddings -- shares that span's + * trace id and now nests under one PostHog trace. + * + * randomUUID() stays the fallback, NOT an error: AI_EMBED/AI_VISION/AI_ADVISORY and any call made outside a + * review span legitimately have no ambient trace, and an orphan trace is still a valid (if less useful) + * event. `$ai_span_id` is only set when there IS a real span to name. Shared (#10186) so the degradation + * event below lands in the SAME trace as the attempts around it instead of re-deriving this. */ +function aiTraceProperties(operational: Record): Record { + const traceId = typeof operational.trace_id === "string" ? operational.trace_id : undefined; + const spanId = typeof operational.span_id === "string" ? operational.span_id : undefined; + return { $ai_trace_id: traceId ?? randomUUID(), ...(spanId === undefined ? {} : { $ai_span_id: spanId }) }; +} + +/** #10185: a real group, so "which repo costs the most in AI spend" is answerable in PostHog rather than only + * in the ai_usage_events SQL table. Deliberately reads the ALREADY-PROCESSED `repo` off + * {@link operationalProperties} rather than the caller's raw context: under the shared central key that value + * is HMAC-anonymized, and when the anon secret has not been injected the key is dropped entirely. Reusing it + * means the group inherits that fail-closed behavior for free -- a raw private repo name can never reach the + * group index by this path. */ +function repoGroup(operational: Record): { groups?: { repo: string } } { + return typeof operational.repo === "string" ? { groups: { repo: operational.repo } } : {}; +} + export function capturePostHogAiGeneration(event: PostHogAiGenerationEvent): void { if (!active || !client) return; const operational = operationalProperties(event.context); - // #10185: the OTel trace this call already runs inside IS the trace PostHog should group by. - // operationalProperties has computed it (as plain `trace_id`/`span_id`) since #8296, but this - // function then minted a fresh randomUUID() for $ai_trace_id and threw it away -- so every - // generation became its own single-event trace and the review pipeline that produced them was - // never visible as one thing (13,428 events across 13,428 traces on the live project). - // - // withReviewPipelineSpan (./review-tracing.ts) wraps a whole review, so every provider attempt - // inside it -- both dual-review legs, every retry, every self-consistency run, the RAG embeddings - // -- shares that span's trace id and now nests under one PostHog trace. - // - // randomUUID() stays the fallback, NOT an error: AI_EMBED/AI_VISION/AI_ADVISORY and any call made - // outside a review span legitimately have no ambient trace, and an orphan trace is still a valid - // (if less useful) event. `$ai_span_id` is only set when there IS a real span to name. - const traceId = typeof operational.trace_id === "string" ? operational.trace_id : undefined; - const spanId = typeof operational.span_id === "string" ? operational.span_id : undefined; const properties: Record = { ...operational, - $ai_trace_id: traceId ?? randomUUID(), - ...(spanId === undefined ? {} : { $ai_span_id: spanId }), + ...aiTraceProperties(operational), // Names the node in the trace tree. Provider-and-kind rather than the tool/feature name, because // that is what this function actually knows -- the feature lives in ai_usage_events, not here. $ai_span_name: `ai.${event.requestKind}/${nonBlank(event.provider) ?? "unknown"}`, @@ -460,13 +474,67 @@ export function capturePostHogAiGeneration(event: PostHogAiGenerationEvent): voi distinctId: POSTHOG_DISTINCT_ID, event: event.requestKind === "embedding" ? "$ai_embedding" : "$ai_generation", properties, - // #10185: a real group, so "which repo costs the most in AI spend" is answerable in PostHog - // rather than only in the ai_usage_events SQL table. Deliberately reads the ALREADY-PROCESSED - // `repo` off operationalProperties rather than event.context: under the shared central key that - // value is HMAC-anonymized, and when the anon secret has not been injected the key is dropped - // entirely. Reusing it means the group inherits that fail-closed behavior for free -- a raw - // private repo name can never reach the group index by this path. - ...(typeof operational.repo === "string" ? { groups: { repo: operational.repo } } : {}), + ...repoGroup(operational), + }); +} + +/** Why a provider attempt never reached a model (#10186). `circuit_open`: the provider was SKIPPED because its + * breaker is in cooldown. `chain_exhausted`: every provider in the chain threw, so the caller degrades. */ +export type PostHogAiDegradationReason = "circuit_open" | "chain_exhausted"; + +export const POSTHOG_AI_DEGRADED_EVENT = "selfhost_ai_degraded"; + +/** One AI request that produced NO generation. Same metadata-only posture as + * {@link PostHogAiGenerationEvent}: provider/model ids, the reason, and an already-redacted error string. */ +export type PostHogAiDegradationEvent = { + reason: PostHogAiDegradationReason; + /** The provider that was skipped (`circuit_open`), or the last one tried (`chain_exhausted`). */ + provider: string; + /** The model that WOULD have run, resolved by ai.ts's `resolveTelemetryModel` -- never the raw `@cf/` + * request-layer placeholder, for exactly the reason #10186 exists. */ + model: string; + requestKind: PostHogAiGenerationRequestKind; + /** Every provider tried, for `chain_exhausted` -- names WHICH chain failed, not just its last member. */ + providers?: readonly string[] | undefined; + context?: Record | undefined; + error?: unknown; +}; + +/** Capture an AI request that degraded WITHOUT any model being called (#10186). No-op when PostHog is off. + * + * Deliberately NOT a `$ai_generation` with `$ai_is_error: true`: no model ran, so booking it against one + * would credit that model with a failure it never had -- the same false signal this issue's own headline bug + * produces -- and would drag zero-token, zero-cost rows into the spend and token aggregates. It carries the + * shared `$ai_trace_id` instead, so a degraded request still joins to the real attempts around it in the same + * trace, and PostHog's LLM-analytics views stay a faithful record of calls that actually happened. + * + * Both reasons were previously invisible to PostHog as anything but an absence: a provider in cooldown emits + * a Prometheus counter and throws, and chain exhaustion writes a console line. Neither produced an `$ai_*` + * event, so a fully degraded provider looked exactly like a provider with no traffic. */ +export function capturePostHogAiDegradation(event: PostHogAiDegradationEvent): void { + if (!active || !client) return; + const operational = operationalProperties(event.context); + const properties: Record = { + ...operational, + ...aiTraceProperties(operational), + $ai_span_name: `ai.${event.requestKind}/${nonBlank(event.provider) ?? "unknown"}/${event.reason}`, + $ai_model: nonBlank(event.model) ?? "unknown", + $ai_provider: nonBlank(event.provider) ?? "unknown", + $ai_is_error: true, + reason: event.reason, + request_kind: event.requestKind, + environment: posthogEnvironment, + }; + if (event.providers && event.providers.length > 0) properties.providers = [...event.providers]; + if (event.error !== undefined) { + const error = event.error instanceof Error ? event.error : new Error(String(event.error)); + properties.$ai_error = error.message.slice(0, 500); + } + client.capture({ + distinctId: POSTHOG_DISTINCT_ID, + event: POSTHOG_AI_DEGRADED_EVENT, + properties, + ...repoGroup(operational), }); } diff --git a/test/unit/docs-selfhost-posthog-observability.test.ts b/test/unit/docs-selfhost-posthog-observability.test.ts index 1fbac42c1..39d0ef3f3 100644 --- a/test/unit/docs-selfhost-posthog-observability.test.ts +++ b/test/unit/docs-selfhost-posthog-observability.test.ts @@ -1,7 +1,7 @@ import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; -import { POSTHOG_MONITOR_HEARTBEAT_EVENT } from "../../src/selfhost/posthog"; +import { POSTHOG_AI_DEGRADED_EVENT, POSTHOG_MONITOR_HEARTBEAT_EVENT } from "../../src/selfhost/posthog"; // Drift guard (#8287, #1468 -- 2026-07-25 Sentry removal): self-host PostHog docs must stay aligned with the // exported monitor-heartbeat event name. Sentry's own docs test (docs-selfhost-sentry-observability.test.ts) @@ -26,6 +26,20 @@ describe("self-host PostHog observability docs (#8287)", () => { expect(operations).toContain("exception autocapture"); }); + it("documents the degraded-request event with its real exported name and both reasons (#10186)", () => { + // Same drift guard as the heartbeat below: an operator reading this page must be able to search + // PostHog for the exact event name the code emits. + expect(operations).toContain(POSTHOG_AI_DEGRADED_EVENT); + for (const reason of ["circuit_open", "chain_exhausted"]) { + expect(operations).toContain(reason); + } + }); + + it("documents that $ai_model is the resolved model on the failure path too (#10186)", () => { + expect(operations).toContain("Model labelling"); + expect(operations).toContain("-default"); + }); + it("documents the cron-monitor heartbeat replacement with the real exported event name", () => { expect(operations).toContain("Cron Monitors"); expect(operations).toContain(POSTHOG_MONITOR_HEARTBEAT_EVENT); diff --git a/test/unit/selfhost-ai.test.ts b/test/unit/selfhost-ai.test.ts index 0db562f4d..3f231bf75 100644 --- a/test/unit/selfhost-ai.test.ts +++ b/test/unit/selfhost-ai.test.ts @@ -745,6 +745,132 @@ describe("$ai_generation PostHog capture at the chain chokepoint (#8296)", () => await createChainAi([provider]).run("m", { prompt: "x" }); expect(posthogMocks.capture).not.toHaveBeenCalled(); }); + + // #10186: the headline bug. Every one of the 33 failed AI events on the live project was labelled with a + // model that never ran, so a breakdown by model showed `@cf/*` ids at a 100% error rate on a deployment + // that has never called Workers AI, while the real failing providers read 0%. + it("REGRESSION (#10186): the FAILURE path reports the model the provider would have run, not the discarded `@cf/` placeholder", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + const provider = { + name: "claude-code", + ai: { + telemetryModel: () => "claude-sonnet-5", + run: async () => { throw new Error("claude_code_error_429"); }, + }, + }; + await expect(createChainAi([provider]).run("@cf/openai/gpt-oss-120b", { prompt: "x" })).rejects.toThrow(/429/); + const generation = posthogMocks.capture.mock.calls.find((call) => call[0].event === "$ai_generation")?.[0]; + expect(generation.properties.$ai_model).toBe("claude-sonnet-5"); + expect(generation.properties.$ai_error).toBe("claude_code_error_429"); + }); + + it("REGRESSION (#10186): the success and the failure path agree on the model for the SAME configured provider", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + // The deliverable stated as an invariant: same provider, same passed placeholder, same label either way. + const telemetryModel = (): string => "qwen3:8b"; + await createChainAi([{ name: "ollama", ai: { telemetryModel, run: async () => ({ response: "ok" }) } }]) + .run("@cf/meta/llama-3.1-8b-instruct-fp8-fast", { prompt: "x" }); + await expect( + createChainAi([{ name: "ollama", ai: { telemetryModel, run: async () => { throw new Error("down"); } } }]) + .run("@cf/meta/llama-3.1-8b-instruct-fp8-fast", { prompt: "x" }), + ).rejects.toThrow("down"); + const generations = posthogMocks.capture.mock.calls.filter((call) => call[0].event === "$ai_generation"); + expect(generations[0]?.[0].properties.$ai_is_error).toBe(false); + expect(generations[1]?.[0].properties.$ai_is_error).toBe(true); + expect(generations[0]?.[0].properties.$ai_model).toBe("qwen3:8b"); + expect(generations[1]?.[0].properties.$ai_model).toBe("qwen3:8b"); + }); + + it("an embedding request is labelled with the EMBED model, not the chat model (#10186)", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + // The live `@cf/baai/bge-m3` mislabelling: 8 embed failures reported a Workers-AI id for a call that + // would have run bge-m3:latest. + const ai = createOpenAiCompatibleAi({ baseUrl: "http://localhost:11434/v1", model: "qwen3:8b", embedModel: "bge-m3:latest", providerName: "ollama" }); + vi.stubGlobal("fetch", vi.fn(async () => new Response("nope", { status: 400 }))); + await expect(createChainAi([{ name: "ollama", ai }]).run("@cf/baai/bge-m3", { text: ["chunk"] })).rejects.toThrow(/ai_embed_http_400/); + const generation = posthogMocks.capture.mock.calls.find((call) => call[0].event === "$ai_embedding")?.[0]; + expect(generation.properties.$ai_model).toBe("bge-m3:latest"); + }); + + // #10186 (second half): the degradation paths, which previously left NOTHING in PostHog -- so a provider + // that was failing or being skipped for every request looked exactly like a provider with no traffic. + it("captures a circuit_open degradation when a provider in cooldown is SKIPPED", async () => { + vi.useFakeTimers(); + try { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + const provider = { + name: "claude-code", + ai: { telemetryModel: () => "claude-sonnet-5", run: async () => { throw new Error("claude_code_error_429"); } }, + }; + const ai = createChainAi([provider]); + // Three consecutive failures latch the breaker (AI_PROVIDER_FAILURE_THRESHOLD). + for (let i = 0; i < 3; i += 1) await expect(ai.run("m", { prompt: "x" })).rejects.toThrow(); + posthogMocks.capture.mockClear(); + await expect(ai.run("m", { prompt: "x" })).rejects.toThrow(/circuit_open/); + const degraded = posthogMocks.capture.mock.calls.find((call) => call[0].event === "selfhost_ai_degraded")?.[0]; + expect(degraded.properties.reason).toBe("circuit_open"); + expect(degraded.properties.$ai_provider).toBe("claude-code"); + // Labelled with the model it WOULD have run — the skip is still attributable to a real model. + expect(degraded.properties.$ai_model).toBe("claude-sonnet-5"); + expect(degraded.properties.$ai_error).toMatch(/circuit_open/); + // The skipped attempt is NOT also booked as a generation: no model was called. + expect(posthogMocks.capture.mock.calls.some((call) => call[0].event === "$ai_generation")).toBe(false); + } finally { + vi.useRealTimers(); + } + }); + + it("captures a chain_exhausted degradation naming every provider that was tried", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + const ai = createChainAi([ + { name: "claude-code", ai: { telemetryModel: () => "claude-sonnet-5", run: async () => { throw new Error("claude_code_error_429"); } } }, + { name: "ollama", ai: { telemetryModel: () => "qwen3:8b", run: async () => { throw new Error("ai_http_500"); } } }, + ]); + await expect(ai.run("@cf/openai/gpt-oss-120b", { prompt: "x", repoFullName: "owner/repo" })).rejects.toThrow(/ai_http_500/); + const degraded = posthogMocks.capture.mock.calls.find((call) => call[0].event === "selfhost_ai_degraded")?.[0]; + expect(degraded.properties.reason).toBe("chain_exhausted"); + expect(degraded.properties.providers).toEqual(["claude-code", "ollama"]); + // Attributed to the provider that ran LAST, which is the one `lastError` belongs to. + expect(degraded.properties.$ai_provider).toBe("ollama"); + expect(degraded.properties.$ai_model).toBe("qwen3:8b"); + expect(degraded.properties.repo).toBe("owner/repo"); + // Both per-provider failures are still captured individually alongside it. + expect(posthogMocks.capture.mock.calls.filter((call) => call[0].event === "$ai_generation")).toHaveLength(2); + }); + + it("does NOT capture chain_exhausted when EVERY provider merely declined an embed request (one exemption rule, both sinks)", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + // A CLI-subscription chain declining an embed is routing working as designed, not a degradation. Its own + // fail-loud signal is shouldWarnRagEmbedUnavailable's boot warning, not one event per chunk batch. + const ai = createChainAi([ + { name: "claude-code", ai: { run: async () => { throw new Error("claude_code_no_embed"); } } }, + { name: "codex", ai: { run: async () => { throw new Error("codex_no_embed"); } } }, + ]); + await expect(ai.run("m", { text: ["chunk"] })).rejects.toThrow(); + expect(posthogMocks.capture).not.toHaveBeenCalled(); + }); + + it("DOES capture chain_exhausted when a routing decline is mixed with a real failure", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + // The exemption is all-or-nothing: one genuine failure means the chain really did degrade. + const ai = createChainAi([ + { name: "claude-code", ai: { run: async () => { throw new Error("claude_code_no_embed"); } } }, + { name: "ollama", ai: { telemetryModel: () => "bge-m3:latest", run: async () => { throw new Error("ai_embed_http_400"); } } }, + ]); + await expect(ai.run("m", { text: ["chunk"] })).rejects.toThrow(/ai_embed_http_400/); + const degraded = posthogMocks.capture.mock.calls.find((call) => call[0].event === "selfhost_ai_degraded")?.[0]; + expect(degraded.properties.reason).toBe("chain_exhausted"); + expect(degraded.properties.$ai_model).toBe("bge-m3:latest"); + }); + + it("an EMPTY chain exhausts without a degradation event, and labels its log line from the passed model or 'unknown'", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + // There is no provider to attribute to and nothing was skipped -- `no_ai_providers` is a configuration + // state the boot preflight owns, not a runtime degradation of a working chain. + await expect(createChainAi([]).run("real-model", { prompt: "x" })).rejects.toThrow(/no_ai_providers/); + await expect(createChainAi([]).run("", { prompt: "x" })).rejects.toThrow(/no_ai_providers/); + expect(posthogMocks.capture).not.toHaveBeenCalled(); + }); }); describe("withAiGenerationCapture (#8296 — AI_EMBED/AI_VISION/AI_ADVISORY, ungoverned single-provider bindings)", () => { @@ -793,7 +919,7 @@ describe("withAiGenerationCapture (#8296 — AI_EMBED/AI_VISION/AI_ADVISORY, ung await ai.run("", { prompt: "x" }); const { properties } = posthogMocks.capture.mock.calls[0]?.[0]; expect(properties.$ai_provider).toBe("ai_vision"); - expect(properties.$ai_model).toBe("default"); + expect(properties.$ai_model).toBe("ai_vision-default"); expect(properties.$ai_input_tokens).toBe(3); }); @@ -804,20 +930,65 @@ describe("withAiGenerationCapture (#8296 — AI_EMBED/AI_VISION/AI_ADVISORY, ung expect(posthogMocks.capture.mock.calls[0]?.[0].properties.$ai_model).toBe("configured-model"); }); - it("falls back to 'default' model on the failure path when the model arg is empty", async () => { + it("falls back to '-default' on the failure path when the model arg is empty (#10186)", async () => { await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); const ai = withAiGenerationCapture("ai_embed", { run: async () => { throw new Error("down"); } }); await expect(ai.run("", { text: ["chunk"] })).rejects.toThrow("down"); - expect(posthogMocks.capture.mock.calls[0]?.[0].properties.$ai_model).toBe("default"); + // #10186: a bare "default" said nothing about WHICH binding failed; every unlabelled failure across + // AI_EMBED/AI_VISION/AI_ADVISORY collapsed into one indistinguishable bucket. + expect(posthogMocks.capture.mock.calls[0]?.[0].properties.$ai_model).toBe("ai_embed-default"); }); - it("falls back to 'default' model when the result carries no usage at all AND the model arg is empty", async () => { + it("falls back to '-default' when the result carries no usage at all AND the model arg is empty (#10186)", async () => { await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); const ai = withAiGenerationCapture("ai_advisory", { run: async () => ({ response: "ok" }) }); await ai.run("", { prompt: "x" }); const { properties } = posthogMocks.capture.mock.calls[0]?.[0]; expect(properties.$ai_provider).toBe("ai_advisory"); - expect(properties.$ai_model).toBe("default"); + expect(properties.$ai_model).toBe("ai_advisory-default"); + }); + + it("REGRESSION (#10186): a `@cf/` placeholder never reaches $ai_model, on the success OR the failure path", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + // Workers AI has no live binding anywhere on a self-host (src/services/ai-summaries.ts), so the core's + // `@cf/`-prefixed id is a request-layer placeholder resolveModel discards -- reporting it labels the call + // with a model that never ran. The adapter here implements telemetryModel, as every real one does. + const ai = withAiGenerationCapture("ai_vision", { + telemetryModel: () => "qwen3-vl:8b-instruct", + run: async () => { throw new Error("down"); }, + }); + await expect(ai.run("@cf/openai/gpt-oss-120b", { prompt: "x" })).rejects.toThrow("down"); + expect(posthogMocks.capture.mock.calls[0]?.[0].properties.$ai_model).toBe("qwen3-vl:8b-instruct"); + }); + + it("REGRESSION (#10186): with no telemetryModel to consult, a `@cf/` id still degrades to '-default' rather than passing through", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + const ai = withAiGenerationCapture("ai_embed", { run: async () => { throw new Error("down"); } }); + await expect(ai.run("@cf/baai/bge-m3", { text: ["chunk"] })).rejects.toThrow("down"); + expect(posthogMocks.capture.mock.calls[0]?.[0].properties.$ai_model).toBe("ai_embed-default"); + }); + + it("prefers an adapter's own telemetryModel over a truthy non-@cf model arg on the failure path (#10186)", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + // "visual-vision" is a real live example: a passed id that is not `@cf/`-prefixed but is still NOT the + // model the binding resolves once its own configured model wins in resolveModel. + const ai = withAiGenerationCapture("ai_vision", { + telemetryModel: () => "qwen3-vl:8b-instruct", + run: async () => { throw new Error("down"); }, + }); + await expect(ai.run("visual-vision", { prompt: "x" })).rejects.toThrow("down"); + expect(posthogMocks.capture.mock.calls[0]?.[0].properties.$ai_model).toBe("qwen3-vl:8b-instruct"); + }); + + it("ignores a telemetryModel that resolves to blank, falling through to the model arg (#10186)", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + // Codex's own resolution returns "" when nothing is configured (the CLI picks the account default). + const ai = withAiGenerationCapture("ai_advisory", { + telemetryModel: () => " ", + run: async () => ({ response: "ok" }), + }); + await ai.run("real-model", { prompt: "x" }); + expect(posthogMocks.capture.mock.calls[0]?.[0].properties.$ai_model).toBe("real-model"); }); }); @@ -1202,7 +1373,7 @@ describe("routeProviders (#dual-ai-combiner — address one provider by name for }); }); - it("provider routing falls back to \"default\" when both the adapter's usage AND the requested model are empty", async () => { + it("provider routing falls back to \"-default\" when both the adapter's usage AND the requested model are empty (#10186)", async () => { const ai = createChainAi([ { name: "codex", @@ -1213,7 +1384,7 @@ describe("routeProviders (#dual-ai-combiner — address one provider by name for ]); await expect(ai.run("", { prompt: "x" })).resolves.toMatchObject({ response: "ok", - usage: { provider: "codex", model: "default", inputTokens: 3 }, + usage: { provider: "codex", model: "codex-default", inputTokens: 3 }, }); }); diff --git a/test/unit/selfhost-posthog.test.ts b/test/unit/selfhost-posthog.test.ts index 247c15ef6..a9df1e2f6 100644 --- a/test/unit/selfhost-posthog.test.ts +++ b/test/unit/selfhost-posthog.test.ts @@ -24,6 +24,7 @@ vi.mock("../../src/selfhost/otel", () => ({ })); import { + capturePostHogAiDegradation, capturePostHogAiGeneration, capturePostHogError, capturePostHogReviewFailure, @@ -31,6 +32,7 @@ import { forwardStructuredLogToPostHog, initPostHog, installPostHogStructuredLogForwarding, + POSTHOG_AI_DEGRADED_EVENT, POSTHOG_MONITOR_HEARTBEAT_EVENT, resetPostHogForTest, resolvePostHogRelease, @@ -765,6 +767,113 @@ describe("capturePostHogAiGeneration (#8296)", () => { }); }); +describe("capturePostHogAiDegradation (#10186 — a request that reached NO model)", () => { + const BASE = { + reason: "circuit_open" as const, + provider: "claude-code", + model: "claude-sonnet-5", + requestKind: "review" as const, + }; + + it("is a no-op when PostHog is unconfigured", () => { + capturePostHogAiDegradation(BASE); + expect(mocks.capture).not.toHaveBeenCalled(); + }); + + it("emits a NATIVE event, never a $ai_generation — no model ran, so none may be blamed", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogAiDegradation(BASE); + const call = mocks.capture.mock.calls[0]?.[0]; + // Booking this as $ai_generation would credit claude-sonnet-5 with a failure it never had -- the exact + // false signal #10186 exists to remove -- and drag a zero-token, zero-cost row into the spend aggregates. + expect(call.event).toBe(POSTHOG_AI_DEGRADED_EVENT); + expect(call.event).not.toBe("$ai_generation"); + expect(call.properties.reason).toBe("circuit_open"); + expect(call.properties.request_kind).toBe("review"); + expect(call.properties.$ai_model).toBe("claude-sonnet-5"); + expect(call.properties.$ai_provider).toBe("claude-code"); + expect(call.properties.$ai_is_error).toBe(true); + expect(call.properties.environment).toBe("production"); + // Never a spend/token figure: this request consumed neither. + expect("$ai_total_cost_usd" in call.properties).toBe(false); + expect("$ai_input_tokens" in call.properties).toBe(false); + }); + + it("shares the ambient OTel trace so a degraded request joins the real attempts around it", async () => { + otelMocks.currentOtelTraceIds.mockReturnValue({ trace_id: "review-trace-1", span_id: "provider-span-1" }); + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogAiDegradation(BASE); + const { properties } = mocks.capture.mock.calls[0]?.[0]; + expect(properties.$ai_trace_id).toBe("review-trace-1"); + expect(properties.$ai_span_id).toBe("provider-span-1"); + expect(properties.$ai_span_name).toBe("ai.review/claude-code/circuit_open"); + }); + + it("falls back to a minted trace id, and omits $ai_span_id, outside any span", async () => { + otelMocks.currentOtelTraceIds.mockReturnValue(undefined); + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogAiDegradation(BASE); + const { properties } = mocks.capture.mock.calls[0]?.[0]; + expect(properties.$ai_trace_id).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i); + expect("$ai_span_id" in properties).toBe(false); + }); + + it("names the chain that gave up, not just its last member", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogAiDegradation({ ...BASE, reason: "chain_exhausted", providers: ["claude-code", "ollama"] }); + const { properties } = mocks.capture.mock.calls[0]?.[0]; + expect(properties.reason).toBe("chain_exhausted"); + expect(properties.providers).toEqual(["claude-code", "ollama"]); + }); + + it("omits `providers` when absent, and when present-but-empty", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogAiDegradation(BASE); + capturePostHogAiDegradation({ ...BASE, providers: [] }); + expect("providers" in mocks.capture.mock.calls[0]?.[0].properties).toBe(false); + expect("providers" in mocks.capture.mock.calls[1]?.[0].properties).toBe(false); + }); + + it("records a bounded error message from an Error, and from a non-Error thrown value", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogAiDegradation({ ...BASE, error: new Error("y".repeat(600)) }); + capturePostHogAiDegradation({ ...BASE, error: "just a string" }); + expect(mocks.capture.mock.calls[0]?.[0].properties.$ai_error).toHaveLength(500); + expect(mocks.capture.mock.calls[1]?.[0].properties.$ai_error).toBe("just a string"); + }); + + it("omits $ai_error entirely when no error was supplied", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogAiDegradation(BASE); + expect("$ai_error" in mocks.capture.mock.calls[0]?.[0].properties).toBe(false); + }); + + it("falls back to 'unknown' for a blank provider/model rather than emitting an empty label", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogAiDegradation({ ...BASE, provider: " ", model: "" }); + const { properties } = mocks.capture.mock.calls[0]?.[0]; + expect(properties.$ai_provider).toBe("unknown"); + expect(properties.$ai_model).toBe("unknown"); + expect(properties.$ai_span_name).toBe("ai.review/unknown/circuit_open"); + }); + + it("stamps the repo group when there is a repo, and omits `groups` when there is not", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogAiDegradation({ ...BASE, context: { repo: "owner/repo", pullNumber: 7 } }); + capturePostHogAiDegradation(BASE); + expect(mocks.capture.mock.calls[0]?.[0].groups).toEqual({ repo: "owner/repo" }); + expect("groups" in mocks.capture.mock.calls[1]?.[0]).toBe(false); + }); + + it("never carries prompt/response content", async () => { + await initPostHog({ POSTHOG_API_KEY: "phc_test_key" } as unknown as NodeJS.ProcessEnv); + capturePostHogAiDegradation({ ...BASE, error: new Error("boom") }); + const keys = Object.keys(mocks.capture.mock.calls[0]?.[0].properties); + expect(keys).not.toContain("$ai_input"); + expect(keys).not.toContain("$ai_output_choices"); + }); +}); + describe("flushPostHog / shutdownPostHog", () => { it("flushPostHog is a no-op when unconfigured", async () => { await flushPostHog();