From 9c4bc144e341ad2cfb588d623d70c0ccf5f17151 Mon Sep 17 00:00:00 2001 From: Ross <144740362+ross0x01@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:34:39 -0500 Subject: [PATCH 1/3] feat: add paid model quality retention experiments --- .../webhook/__tests__/route.test.ts | 17 ++ app/api/subscription/webhook/route.ts | 39 +++ lib/__tests__/paid-funnel.test.ts | 17 ++ lib/analytics/paid-funnel.ts | 19 ++ lib/api/__tests__/chat-logger.test.ts | 50 ++++ lib/api/chat-handler.ts | 34 +-- lib/api/chat-logger.ts | 18 ++ .../__tests__/deepseek-v4-pro-0813.test.ts | 122 -------- .../__tests__/paid-model-quality.test.ts | 255 +++++++++++++++++ lib/experiments/deepseek-v4-pro-0813.ts | 132 --------- lib/experiments/paid-model-quality.ts | 270 ++++++++++++++++++ trigger/agent-long.ts | 34 +-- 12 files changed, 721 insertions(+), 286 deletions(-) delete mode 100644 lib/experiments/__tests__/deepseek-v4-pro-0813.test.ts create mode 100644 lib/experiments/__tests__/paid-model-quality.test.ts delete mode 100644 lib/experiments/deepseek-v4-pro-0813.ts create mode 100644 lib/experiments/paid-model-quality.ts diff --git a/app/api/subscription/webhook/__tests__/route.test.ts b/app/api/subscription/webhook/__tests__/route.test.ts index ee152d0b8..5bc57e85f 100644 --- a/app/api/subscription/webhook/__tests__/route.test.ts +++ b/app/api/subscription/webhook/__tests__/route.test.ts @@ -1698,6 +1698,14 @@ describe("POST /api/subscription/webhook", () => { }), ); } + expect(mockPostHogEvent).toHaveBeenCalledWith( + "invoice_paid", + expect.objectContaining({ + subscription_mrr_dollars: 29, + attributed_mrr_dollars: 29, + retained_mrr_dollars: 29, + }), + ); }); it("emits recovery when invoice.paid arrives before the failure webhook", async () => { @@ -2063,12 +2071,18 @@ describe("POST /api/subscription/webhook", () => { userId: "user_paid", tier: "pro-plus", org_id: "org_hackerai", + churn_type: "voluntary", + voluntary_churn: true, + involuntary_churn: false, $set: { subscription_tier: "free" }, }), ); expect(mockPostHogEvent).toHaveBeenCalledWith( PAID_FUNNEL_EVENTS.cancellationCompleted, expect.objectContaining({ + churn_type: "voluntary", + voluntary_churn: true, + involuntary_churn: false, $insert_id: cancellationCompletionInsertId("sub_hackerai_deleted"), }), ); @@ -2647,6 +2661,9 @@ describe("POST /api/subscription/webhook", () => { org_id: "org_deleted_payment_failed", tier: "ultra", cancellation_reason: "payment_failed", + churn_type: "involuntary", + voluntary_churn: false, + involuntary_churn: true, stripe_event_id: "evt_subscription_deleted_payment_failed", $set: { subscription_tier: "free" }, }), diff --git a/app/api/subscription/webhook/route.ts b/app/api/subscription/webhook/route.ts index 49ebf408b..42bb739da 100644 --- a/app/api/subscription/webhook/route.ts +++ b/app/api/subscription/webhook/route.ts @@ -24,6 +24,7 @@ import { billingPaymentRecoveryInsertId, cancellationCompletionInsertId, paidFunnelProperties, + subscriptionChurnHealthProperties, } from "@/lib/analytics/paid-funnel"; import { logStripeWebhookMissingSignature, @@ -756,6 +757,15 @@ function emitInvoicePaidRevenueAnalytics({ invoicePrice.lookup_key, ); const attributedRevenueDollars = amountPaidDollars / userIds.length; + const subscriptionMrr = subscriptionMrrDollars({ + price: invoicePrice, + quantity: subscription.items?.data[0]?.quantity ?? 1, + fallbackTotalIntervalAmountDollars: amountPaidDollars, + }); + const attributedMrrDollars = + subscriptionMrr === undefined + ? undefined + : subscriptionMrr / userIds.length; for (const uid of userIds) { phLogger.event( @@ -774,6 +784,9 @@ function emitInvoicePaidRevenueAnalytics({ invoice.attempt_count > 1 && { recovery_result: "recovered" }), amount_paid_dollars: amountPaidDollars, attributed_revenue_dollars: attributedRevenueDollars, + subscription_mrr_dollars: subscriptionMrr, + attributed_mrr_dollars: attributedMrrDollars, + retained_mrr_dollars: attributedMrrDollars, user_count: userIds.length, currency: invoice.currency, stripe_event_id: stripeEventId, @@ -2146,6 +2159,14 @@ async function recordCancellationCompleted(args: { args.subscription.metadata, args.price?.lookup_key, ); + const subscriptionMrr = subscriptionMrrDollars({ + price: args.price, + quantity: args.subscription.items?.data[0]?.quantity ?? 1, + }); + const attributedMrrDollars = + subscriptionMrr === undefined + ? undefined + : subscriptionMrr / args.userIds.length; let updatedCount = 0; try { @@ -2189,6 +2210,12 @@ async function recordCancellationCompleted(args: { billing_interval: priceBillingInterval(args.price), billing_interval_count: args.price?.recurring?.interval_count, cancellation_reason: stripeCancellationReason, + churn_type: "voluntary", + voluntary_churn: true, + involuntary_churn: false, + subscription_mrr_dollars: subscriptionMrr, + attributed_mrr_dollars: attributedMrrDollars, + at_risk_mrr_dollars: attributedMrrDollars, cancellation_completion_type: args.completionType, cancel_at_period_end: args.subscription.cancel_at_period_end, stripe_customer_id: args.customerId, @@ -2258,6 +2285,14 @@ async function handleSubscriptionDeleted( } const cancellationReason = subscription.cancellation_details?.reason ?? null; + const subscriptionMrr = subscriptionMrrDollars({ + price, + quantity: subscription.items?.data[0]?.quantity ?? 1, + }); + const attributedMrrDollars = + subscriptionMrr === undefined + ? undefined + : subscriptionMrr / userIds.length; console.log( `[Subscription Webhook] subscription.deleted: tier ${tier ?? "unknown"} cancelled for ${userIds.length} user(s) (reason: ${cancellationReason ?? "none"})`, ); @@ -2278,6 +2313,10 @@ async function handleSubscriptionDeleted( tier, org_id: orgId, cancellation_reason: cancellationReason, + ...subscriptionChurnHealthProperties(cancellationReason), + subscription_mrr_dollars: subscriptionMrr, + attributed_mrr_dollars: attributedMrrDollars, + lost_mrr_dollars: attributedMrrDollars, stripe_event_id: stripeEventId, stripe_event_type: "customer.subscription.deleted", $insert_id: `subscription_cancelled:${stripeEventId}:${uid}`, diff --git a/lib/__tests__/paid-funnel.test.ts b/lib/__tests__/paid-funnel.test.ts index 083be7525..956e93353 100644 --- a/lib/__tests__/paid-funnel.test.ts +++ b/lib/__tests__/paid-funnel.test.ts @@ -3,10 +3,27 @@ import { checkoutStartedInsertId, normalizePaidFunnelLabel, paidFunnelProperties, + subscriptionChurnHealthProperties, upgradeCtaImpressionInsertId, } from "@/lib/analytics/paid-funnel"; describe("paid funnel analytics helpers", () => { + it.each([ + ["cancellation_requested", "voluntary", true, false], + ["payment_failed", "involuntary", false, true], + ["payment_disputed", "dispute", false, false], + [null, "unknown", false, false], + ] as const)( + "classifies Stripe cancellation reason %p as %s", + (reason, churnType, voluntaryChurn, involuntaryChurn) => { + expect(subscriptionChurnHealthProperties(reason)).toEqual({ + churn_type: churnType, + voluntary_churn: voluntaryChurn, + involuntary_churn: involuntaryChurn, + }); + }, + ); + it("keeps the paid funnel event version authoritative", () => { expect( paidFunnelProperties({ diff --git a/lib/analytics/paid-funnel.ts b/lib/analytics/paid-funnel.ts index 3811486dd..26c362fd6 100644 --- a/lib/analytics/paid-funnel.ts +++ b/lib/analytics/paid-funnel.ts @@ -150,3 +150,22 @@ export function paidFunnelProperties(properties: Record = {}) { paid_funnel_event_version: PAID_FUNNEL_EVENT_VERSION, }; } + +export function subscriptionChurnHealthProperties( + reason: string | null | undefined, +) { + const churnType = + reason === "payment_failed" + ? "involuntary" + : reason === "cancellation_requested" + ? "voluntary" + : reason === "payment_disputed" + ? "dispute" + : "unknown"; + + return { + churn_type: churnType, + voluntary_churn: churnType === "voluntary", + involuntary_churn: churnType === "involuntary", + }; +} diff --git a/lib/api/__tests__/chat-logger.test.ts b/lib/api/__tests__/chat-logger.test.ts index eadbbe93c..65e0a8242 100644 --- a/lib/api/__tests__/chat-logger.test.ts +++ b/lib/api/__tests__/chat-logger.test.ts @@ -638,6 +638,56 @@ describe("captureAgentCompletionAnalytics", () => { }, }); }); + + it("adds a cross-mode model-quality outcome only for an active experiment", () => { + const capture = jest.fn(); + + captureAgentCompletionAnalytics({ + posthog: { capture } as any, + userId: "user_123", + chatId: "chat_123", + endpoint: "/api/chat", + mode: "ask", + subscription: "pro-plus", + sandboxInfo: null, + outcome: "success", + chatLogger: undefined, + selectedModel: "model-deepseek-v4-pro", + configuredModelId: "deepseek/deepseek-v4-pro", + responseModel: "deepseek/deepseek-v4-pro", + fallbackServed: false, + finishReason: "stop", + activeModelStreamDurationMs: 9_000, + requestToFirstModelChunkMs: 700, + providerRecoveryAttempts: 0, + experiment: { + key: "paid_standard_model_quality_v1", + variant: "test", + }, + }); + + expect(capture).toHaveBeenCalledTimes(1); + expect(capture).toHaveBeenCalledWith({ + distinctId: "user_123", + event: "paid_model_quality_run_completed", + properties: expect.objectContaining({ + experiment_key: "paid_standard_model_quality_v1", + experiment_variant: "test", + "$feature/paid_standard_model_quality_v1": "test", + subscription_tier: "pro-plus", + mode: "ask", + selected_model: "model-deepseek-v4-pro", + configured_model: "deepseek/deepseek-v4-pro", + outcome: "success", + successful_run: true, + fallback_served: false, + active_model_stream_duration_ms: 9_000, + request_to_first_model_chunk_ms: 700, + provider_recovery_attempts: 0, + $process_person_profile: false, + }), + }); + }); }); describe("captureUsageCost", () => { diff --git a/lib/api/chat-handler.ts b/lib/api/chat-handler.ts index 6b4a09293..402461842 100644 --- a/lib/api/chat-handler.ts +++ b/lib/api/chat-handler.ts @@ -158,11 +158,11 @@ import { PAID_FUNNEL_EVENTS } from "@/lib/analytics/paid-funnel"; import { readAnalyticsRequestContext } from "@/lib/analytics/request-context"; import { buildAgentStepLimitTelemetry } from "@/lib/analytics/agent-step-limit-telemetry"; import { - captureDeepSeekV4Pro0813ExperimentExposure, - evaluateDeepSeekV4Pro0813Experiment, - getActiveDeepSeekV4Pro0813ExperimentAssignment, - getDeepSeekV4Pro0813ExperimentContext, -} from "@/lib/experiments/deepseek-v4-pro-0813"; + capturePaidModelQualityExperimentExposure, + evaluatePaidModelQualityExperiment, + getActivePaidModelQualityExperimentAssignment, + getPaidModelQualityExperimentContext, +} from "@/lib/experiments/paid-model-quality"; import { isEligibleForDirectGlmVision } from "@/lib/chat/auxiliary-vision-eligibility"; import { capturePaidDailyFreeAllowanceServerEvent, @@ -461,15 +461,17 @@ export const createChatHandler = () => { ); } - const deepSeekV4Pro0813Experiment = - await evaluateDeepSeekV4Pro0813Experiment({ + const paidModelQualityExperiment = + await evaluatePaidModelQualityExperiment({ posthog: (posthog ??= PostHogClient()), userId, + mode, + subscription, selectedModel, requestId: req.headers.get("x-vercel-id") ?? undefined, }); - if (deepSeekV4Pro0813Experiment) { - selectedModel = deepSeekV4Pro0813Experiment.modelKey; + if (paidModelQualityExperiment) { + selectedModel = paidModelQualityExperiment.modelKey; } const notesEnabled = (subscription !== "free" || isAgentMode(mode)) && @@ -606,13 +608,13 @@ export const createChatHandler = () => { }); } - const activeDeepSeekV4Pro0813Experiment = - getActiveDeepSeekV4Pro0813ExperimentAssignment( - deepSeekV4Pro0813Experiment, + const activePaidModelQualityExperiment = + getActivePaidModelQualityExperimentAssignment( + paidModelQualityExperiment, selectedModel, ); - const routingExperimentContext = getDeepSeekV4Pro0813ExperimentContext( - activeDeepSeekV4Pro0813Experiment, + const routingExperimentContext = getPaidModelQualityExperimentContext( + activePaidModelQualityExperiment, ); const freeMonthlyBudgetSnapshot = @@ -1476,7 +1478,7 @@ export const createChatHandler = () => { let result; try { - captureDeepSeekV4Pro0813ExperimentExposure({ + capturePaidModelQualityExperimentExposure({ posthog, userId, subscription, @@ -1484,7 +1486,7 @@ export const createChatHandler = () => { selectedModelOverride, selectedModel, configuredModel: configuredModelId, - assignment: activeDeepSeekV4Pro0813Experiment, + assignment: activePaidModelQualityExperiment, }); result = await createStream(selectedModel); } catch (error) { diff --git a/lib/api/chat-logger.ts b/lib/api/chat-logger.ts index 9c2943419..693409d41 100644 --- a/lib/api/chat-logger.ts +++ b/lib/api/chat-logger.ts @@ -34,6 +34,7 @@ import { getExperimentAnalyticsProperties, type ExperimentAnalyticsContext, } from "@/lib/analytics/experiment-context"; +import { capturePaidModelQualityRun } from "@/lib/experiments/paid-model-quality"; import type { AgentStepLimitTelemetry } from "@/lib/analytics/agent-step-limit-telemetry"; import { buildAgentPerformanceDiagnostics } from "@/lib/analytics/agent-performance-diagnostics"; import { @@ -1628,6 +1629,23 @@ export function captureAgentCompletionAnalytics( args: AgentCompletionAnalyticsArgs, ) { const { posthog, userId, mode, subscription, sandboxInfo, outcome } = args; + capturePaidModelQualityRun({ + posthog, + userId, + experiment: args.experiment, + subscription, + mode, + selectedModel: args.selectedModel, + configuredModel: args.configuredModelId, + responseModel: args.responseModel, + outcome, + finishReason: args.finishReason, + fallbackServed: args.fallbackServed, + activeModelStreamDurationMs: args.activeModelStreamDurationMs, + requestToFirstModelChunkMs: args.requestToFirstModelChunkMs, + providerRecoveryAttempts: args.providerRecoveryAttempts, + stepLimitReached: args.stepLimitTelemetry?.stepLimitReached, + }); captureAgentRun({ posthog, userId, diff --git a/lib/experiments/__tests__/deepseek-v4-pro-0813.test.ts b/lib/experiments/__tests__/deepseek-v4-pro-0813.test.ts deleted file mode 100644 index 7b9e1b2bb..000000000 --- a/lib/experiments/__tests__/deepseek-v4-pro-0813.test.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { - captureDeepSeekV4Pro0813ExperimentExposure, - DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY, - DEEPSEEK_V4_PRO_0813_EXPOSURE_EVENT, - DEEPSEEK_V4_PRO_0813_FEATURE_PROPERTY, - evaluateDeepSeekV4Pro0813Experiment, - getActiveDeepSeekV4Pro0813ExperimentAssignment, - getDeepSeekV4Pro0813ExperimentContext, - isEligibleForDeepSeekV4Pro0813Experiment, -} from "@/lib/experiments/deepseek-v4-pro-0813"; - -describe("DeepSeek V4 Pro 0813 experiment", () => { - it("only evaluates requests already resolved to the current DeepSeek V4 Pro route", () => { - expect( - isEligibleForDeepSeekV4Pro0813Experiment("model-deepseek-v4-pro"), - ).toBe(true); - expect(isEligibleForDeepSeekV4Pro0813Experiment("model-grok-4.5")).toBe( - false, - ); - expect(isEligibleForDeepSeekV4Pro0813Experiment("agent-model-free")).toBe( - false, - ); - }); - - it.each([ - ["control", "model-deepseek-v4-pro"], - ["test", "model-deepseek-v4-pro-0813"], - ] as const)("maps %s to %s", async (variant, modelKey) => { - const evaluateFlags = jest.fn(async () => ({ getFlag: () => variant })); - - await expect( - evaluateDeepSeekV4Pro0813Experiment({ - posthog: { evaluateFlags } as never, - userId: "user-1", - selectedModel: "model-deepseek-v4-pro", - }), - ).resolves.toEqual({ - key: DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY, - variant, - modelKey, - }); - }); - - it("does not evaluate ineligible routes", async () => { - const evaluateFlags = jest.fn(); - - await expect( - evaluateDeepSeekV4Pro0813Experiment({ - posthog: { evaluateFlags } as never, - userId: "user-1", - selectedModel: "model-grok-4.5", - }), - ).resolves.toBeUndefined(); - expect(evaluateFlags).not.toHaveBeenCalled(); - }); - - it("fails closed when evaluation returns an unknown value", async () => { - await expect( - evaluateDeepSeekV4Pro0813Experiment({ - posthog: { - evaluateFlags: jest.fn(async () => ({ getFlag: () => true })), - } as never, - userId: "user-1", - selectedModel: "model-deepseek-v4-pro", - }), - ).resolves.toBeUndefined(); - }); - - it("captures a privacy-safe custom exposure from the provider surface", () => { - const capture = jest.fn(); - const assignment = { - key: DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY, - variant: "test" as const, - modelKey: "model-deepseek-v4-pro-0813" as const, - }; - - captureDeepSeekV4Pro0813ExperimentExposure({ - posthog: { capture } as never, - userId: "user-1", - subscription: "pro", - mode: "agent", - selectedModelOverride: "hackerai-standard", - selectedModel: assignment.modelKey, - configuredModel: "deepseek/deepseek-v4-pro-0813", - assignment, - }); - - expect(capture).toHaveBeenCalledWith({ - distinctId: "user-1", - event: DEEPSEEK_V4_PRO_0813_EXPOSURE_EVENT, - properties: { - experiment_key: DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY, - experiment_variant: "test", - [DEEPSEEK_V4_PRO_0813_FEATURE_PROPERTY]: "test", - subscription: "pro", - subscription_tier: "pro", - mode: "agent", - selected_model: "model-deepseek-v4-pro-0813", - selected_model_override: "hackerai-standard", - configured_model: "deepseek/deepseek-v4-pro-0813", - exposure_surface: "provider_request", - $process_person_profile: false, - }, - }); - expect(getDeepSeekV4Pro0813ExperimentContext(assignment)).toEqual({ - key: DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY, - variant: "test", - }); - expect( - getActiveDeepSeekV4Pro0813ExperimentAssignment( - assignment, - "model-deepseek-v4-pro-0813", - ), - ).toBe(assignment); - expect( - getActiveDeepSeekV4Pro0813ExperimentAssignment( - assignment, - "model-grok-4.5", - ), - ).toBeUndefined(); - }); -}); diff --git a/lib/experiments/__tests__/paid-model-quality.test.ts b/lib/experiments/__tests__/paid-model-quality.test.ts new file mode 100644 index 000000000..dad95b6b0 --- /dev/null +++ b/lib/experiments/__tests__/paid-model-quality.test.ts @@ -0,0 +1,255 @@ +import { + capturePaidModelQualityExperimentExposure, + capturePaidModelQualityRun, + evaluatePaidModelQualityExperiment, + getActivePaidModelQualityExperimentAssignment, + getEligiblePaidModelQualityRoute, + getPaidModelQualityExperimentContext, + PAID_MODEL_QUALITY_EXPOSURE_EVENT, + PAID_MODEL_QUALITY_RUN_EVENT, + PAID_PRO_MODEL_QUALITY_EXPERIMENT_KEY, + PAID_STANDARD_MODEL_QUALITY_EXPERIMENT_KEY, +} from "@/lib/experiments/paid-model-quality"; + +describe("paid model quality experiments", () => { + it.each([ + ["ask", "model-deepseek-v4-flash-0731"], + ["agent", "model-glm-5.3-flash-agent"], + ] as const)( + "recognizes the current %s Standard route", + (mode, selectedModel) => { + expect( + getEligiblePaidModelQualityRoute({ + mode, + subscription: "pro", + selectedModel, + }), + ).toMatchObject({ + key: PAID_STANDARD_MODEL_QUALITY_EXPERIMENT_KEY, + route: "standard", + controlModelKey: selectedModel, + previousModelKey: "model-deepseek-v4-pro", + }); + }, + ); + + it("recognizes the current Pro route", () => { + expect( + getEligiblePaidModelQualityRoute({ + mode: "agent", + subscription: "pro-plus", + selectedModel: "model-deepseek-v4-pro-0813", + }), + ).toMatchObject({ + key: PAID_PRO_MODEL_QUALITY_EXPERIMENT_KEY, + route: "pro", + previousModelKey: "model-grok-4.6-pro", + }); + }); + + it.each([ + ["free", "agent", "model-glm-5.3-flash-agent"], + ["team", "agent", "model-glm-5.3-flash-agent"], + ["ultra", "agent", "model-deepseek-v4-pro-0813"], + ["pro", "agent", "model-grok-4.5"], + ["pro", "ask", "model-glm-5.3-flash-agent"], + ] as const)( + "excludes %s %s route %s", + (subscription, mode, selectedModel) => { + expect( + getEligiblePaidModelQualityRoute({ + mode, + subscription, + selectedModel, + }), + ).toBeUndefined(); + }, + ); + + it.each([ + ["ask", "model-deepseek-v4-flash-0731", "test", "model-deepseek-v4-pro"], + [ + "agent", + "model-glm-5.3-flash-agent", + "control", + "model-glm-5.3-flash-agent", + ], + ["agent", "model-deepseek-v4-pro-0813", "test", "model-grok-4.6-pro"], + ] as const)( + "maps %s %s variant %s to %s", + async (mode, selectedModel, variant, modelKey) => { + const evaluateFlags = jest.fn(async () => ({ getFlag: () => variant })); + + await expect( + evaluatePaidModelQualityExperiment({ + posthog: { evaluateFlags } as never, + userId: "user-1", + mode, + subscription: "pro", + selectedModel, + }), + ).resolves.toMatchObject({ variant, modelKey }); + }, + ); + + it("evaluates only the flag for the resolved route", async () => { + const evaluateFlags = jest.fn(async () => ({ getFlag: () => "control" })); + + await evaluatePaidModelQualityExperiment({ + posthog: { evaluateFlags } as never, + userId: "user-1", + mode: "agent", + subscription: "pro-plus", + selectedModel: "model-deepseek-v4-pro-0813", + }); + + expect(evaluateFlags).toHaveBeenCalledWith("user-1", { + flagKeys: [PAID_PRO_MODEL_QUALITY_EXPERIMENT_KEY], + }); + }); + + it("fails closed for unknown values and evaluation errors", async () => { + await expect( + evaluatePaidModelQualityExperiment({ + posthog: { + evaluateFlags: jest.fn(async () => ({ getFlag: () => true })), + } as never, + userId: "user-1", + mode: "ask", + subscription: "pro", + selectedModel: "model-deepseek-v4-flash-0731", + }), + ).resolves.toBeUndefined(); + + const warn = jest.spyOn(console, "warn").mockImplementation(); + await expect( + evaluatePaidModelQualityExperiment({ + posthog: { + evaluateFlags: jest.fn(async () => { + throw new Error("unavailable"); + }), + } as never, + userId: "user-1", + mode: "ask", + subscription: "pro", + selectedModel: "model-deepseek-v4-flash-0731", + }), + ).resolves.toBeUndefined(); + expect(warn).toHaveBeenCalled(); + warn.mockRestore(); + }); + + it("drops attribution if a later limit-rescue route replaces the assignment", () => { + const assignment = { + key: PAID_STANDARD_MODEL_QUALITY_EXPERIMENT_KEY, + route: "standard" as const, + variant: "test" as const, + controlModelKey: "model-glm-5.3-flash-agent" as const, + modelKey: "model-deepseek-v4-pro" as const, + }; + + expect( + getActivePaidModelQualityExperimentAssignment( + assignment, + "model-deepseek-v4-pro", + ), + ).toBe(assignment); + expect( + getActivePaidModelQualityExperimentAssignment( + assignment, + "agent-model-free", + ), + ).toBeUndefined(); + expect(getPaidModelQualityExperimentContext(assignment)).toEqual({ + key: PAID_STANDARD_MODEL_QUALITY_EXPERIMENT_KEY, + variant: "test", + }); + }); + + it("captures a privacy-safe provider-boundary exposure", () => { + const capture = jest.fn(); + const assignment = { + key: PAID_STANDARD_MODEL_QUALITY_EXPERIMENT_KEY, + route: "standard" as const, + variant: "test" as const, + controlModelKey: "model-glm-5.3-flash-agent" as const, + modelKey: "model-deepseek-v4-pro" as const, + }; + + capturePaidModelQualityExperimentExposure({ + posthog: { capture } as never, + userId: "user-1", + subscription: "pro-plus", + mode: "agent", + selectedModelOverride: "hackerai-standard", + selectedModel: assignment.modelKey, + configuredModel: "deepseek/deepseek-v4-pro", + assignment, + }); + + expect(capture).toHaveBeenCalledWith({ + distinctId: "user-1", + event: PAID_MODEL_QUALITY_EXPOSURE_EVENT, + properties: { + experiment_key: PAID_STANDARD_MODEL_QUALITY_EXPERIMENT_KEY, + experiment_variant: "test", + [`$feature/${PAID_STANDARD_MODEL_QUALITY_EXPERIMENT_KEY}`]: "test", + experiment_route: "standard", + subscription: "pro-plus", + subscription_tier: "pro-plus", + mode: "agent", + control_model: "model-glm-5.3-flash-agent", + selected_model: "model-deepseek-v4-pro", + selected_model_override: "hackerai-standard", + configured_model: "deepseek/deepseek-v4-pro", + exposure_surface: "provider_request", + $process_person_profile: false, + }, + }); + }); + + it.each([ + ["ask", "success", "length", false, true], + ["agent", "success", "stop", false, true], + ["agent", "success", "length", true, false], + ["agent", "error", "error", false, false], + ] as const)( + "classifies %s outcome=%s finish=%s stepLimit=%s as successful=%s", + (mode, outcome, finishReason, stepLimitReached, successfulRun) => { + const capture = jest.fn(); + + capturePaidModelQualityRun({ + posthog: { capture } as never, + userId: "user-1", + experiment: { + key: PAID_PRO_MODEL_QUALITY_EXPERIMENT_KEY, + variant: "control", + }, + subscription: "pro", + mode, + selectedModel: "model-deepseek-v4-pro-0813", + configuredModel: "deepseek/deepseek-v4-pro-0813", + responseModel: "deepseek/deepseek-v4-pro-0813", + outcome, + finishReason, + fallbackServed: false, + activeModelStreamDurationMs: 12_000, + requestToFirstModelChunkMs: 800, + providerRecoveryAttempts: 0, + stepLimitReached, + }); + + expect(capture).toHaveBeenCalledWith({ + distinctId: "user-1", + event: PAID_MODEL_QUALITY_RUN_EVENT, + properties: expect.objectContaining({ + experiment_key: PAID_PRO_MODEL_QUALITY_EXPERIMENT_KEY, + experiment_variant: "control", + successful_run: successfulRun, + outcome, + mode, + }), + }); + }, + ); +}); diff --git a/lib/experiments/deepseek-v4-pro-0813.ts b/lib/experiments/deepseek-v4-pro-0813.ts deleted file mode 100644 index 424aadfcc..000000000 --- a/lib/experiments/deepseek-v4-pro-0813.ts +++ /dev/null @@ -1,132 +0,0 @@ -import type { PostHog } from "posthog-node"; -import type { ExperimentAnalyticsContext } from "@/lib/analytics/experiment-context"; -import type { ModelName } from "@/lib/ai/providers"; -import type { ChatMode, SelectedModel, SubscriptionTier } from "@/types"; - -export const DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY = - "deepseek_v4_pro_0813_model_v1"; -export const DEEPSEEK_V4_PRO_0813_EXPOSURE_EVENT = - "hac68_deepseek_v4_pro_0813_experiment_exposed"; -export const DEEPSEEK_V4_PRO_0813_FEATURE_PROPERTY = `$feature/${DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY}`; - -export type DeepSeekV4Pro0813ExperimentVariant = "control" | "test"; -export type DeepSeekV4Pro0813ModelKey = - "model-deepseek-v4-pro" | "model-deepseek-v4-pro-0813"; - -export type DeepSeekV4Pro0813ExperimentAssignment = { - key: typeof DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY; - variant: DeepSeekV4Pro0813ExperimentVariant; - modelKey: DeepSeekV4Pro0813ModelKey; -}; - -export function isEligibleForDeepSeekV4Pro0813Experiment( - selectedModel: ModelName, -): selectedModel is "model-deepseek-v4-pro" { - return selectedModel === "model-deepseek-v4-pro"; -} - -export async function evaluateDeepSeekV4Pro0813Experiment({ - posthog, - userId, - selectedModel, - requestId, -}: { - posthog: Pick | null; - userId: string; - selectedModel: ModelName; - requestId?: string; -}): Promise { - if (!posthog || !isEligibleForDeepSeekV4Pro0813Experiment(selectedModel)) { - return undefined; - } - - try { - const flags = await posthog.evaluateFlags(userId, { - flagKeys: [DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY], - }); - const variant = flags.getFlag(DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY); - - if (variant !== "control" && variant !== "test") { - return undefined; - } - - return { - key: DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY, - variant, - modelKey: - variant === "test" - ? "model-deepseek-v4-pro-0813" - : "model-deepseek-v4-pro", - }; - } catch (error) { - console.warn( - JSON.stringify({ - timestamp: new Date().toISOString(), - level: "warn", - event: "deepseek_v4_pro_0813_experiment_evaluation_failed", - service: "model-routing-experiment", - environment: - process.env.VERCEL_ENV ?? process.env.NODE_ENV ?? "unknown", - request_id: requestId ?? "unavailable", - user_id: userId, - flag_key: DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY, - error_name: error instanceof Error ? error.name : "UnknownError", - }), - ); - return undefined; - } -} - -export function captureDeepSeekV4Pro0813ExperimentExposure({ - posthog, - userId, - subscription, - mode, - selectedModelOverride, - selectedModel, - configuredModel, - assignment, -}: { - posthog: Pick | null; - userId: string; - subscription: SubscriptionTier; - mode: ChatMode; - selectedModelOverride?: SelectedModel; - selectedModel: ModelName; - configuredModel: string; - assignment?: DeepSeekV4Pro0813ExperimentAssignment; -}): void { - if (!posthog || !assignment) return; - - posthog.capture({ - distinctId: userId, - event: DEEPSEEK_V4_PRO_0813_EXPOSURE_EVENT, - properties: { - experiment_key: assignment.key, - experiment_variant: assignment.variant, - [DEEPSEEK_V4_PRO_0813_FEATURE_PROPERTY]: assignment.variant, - subscription, - subscription_tier: subscription, - mode, - selected_model: selectedModel, - selected_model_override: selectedModelOverride ?? "auto", - configured_model: configuredModel, - exposure_surface: "provider_request", - $process_person_profile: false, - }, - }); -} - -export function getDeepSeekV4Pro0813ExperimentContext( - assignment: DeepSeekV4Pro0813ExperimentAssignment | undefined, -): ExperimentAnalyticsContext | undefined { - if (!assignment) return undefined; - return { key: assignment.key, variant: assignment.variant }; -} - -export function getActiveDeepSeekV4Pro0813ExperimentAssignment( - assignment: DeepSeekV4Pro0813ExperimentAssignment | undefined, - selectedModel: ModelName, -): DeepSeekV4Pro0813ExperimentAssignment | undefined { - return assignment?.modelKey === selectedModel ? assignment : undefined; -} diff --git a/lib/experiments/paid-model-quality.ts b/lib/experiments/paid-model-quality.ts new file mode 100644 index 000000000..b3f4a824a --- /dev/null +++ b/lib/experiments/paid-model-quality.ts @@ -0,0 +1,270 @@ +import type { PostHog } from "posthog-node"; +import type { ExperimentAnalyticsContext } from "@/lib/analytics/experiment-context"; +import type { ModelName } from "@/lib/ai/providers"; +import type { ChatMode, SelectedModel, SubscriptionTier } from "@/types"; + +export const PAID_STANDARD_MODEL_QUALITY_EXPERIMENT_KEY = + "paid_standard_model_quality_v1"; +export const PAID_PRO_MODEL_QUALITY_EXPERIMENT_KEY = + "paid_pro_model_quality_v1"; +export const PAID_MODEL_QUALITY_EXPOSURE_EVENT = + "paid_model_quality_experiment_exposed"; +export const PAID_MODEL_QUALITY_RUN_EVENT = "paid_model_quality_run_completed"; + +export type PaidModelQualityRoute = "standard" | "pro"; +export type PaidModelQualityExperimentVariant = "control" | "test"; + +export type PaidModelQualityExperimentAssignment = { + key: + | typeof PAID_STANDARD_MODEL_QUALITY_EXPERIMENT_KEY + | typeof PAID_PRO_MODEL_QUALITY_EXPERIMENT_KEY; + route: PaidModelQualityRoute; + variant: PaidModelQualityExperimentVariant; + controlModelKey: ModelName; + modelKey: ModelName; +}; + +type EligibleRoute = Pick< + PaidModelQualityExperimentAssignment, + "key" | "route" | "controlModelKey" +> & { + previousModelKey: ModelName; +}; + +function isTargetSubscription(subscription: SubscriptionTier): boolean { + return subscription === "pro" || subscription === "pro-plus"; +} + +export function getEligiblePaidModelQualityRoute({ + mode, + subscription, + selectedModel, +}: { + mode: ChatMode; + subscription: SubscriptionTier; + selectedModel: ModelName; +}): EligibleRoute | undefined { + if (!isTargetSubscription(subscription)) return undefined; + + const standardControlModel = + mode === "agent" + ? "model-glm-5.3-flash-agent" + : "model-deepseek-v4-flash-0731"; + + if (selectedModel === standardControlModel) { + return { + key: PAID_STANDARD_MODEL_QUALITY_EXPERIMENT_KEY, + route: "standard", + controlModelKey: standardControlModel, + previousModelKey: "model-deepseek-v4-pro", + }; + } + + if (selectedModel === "model-deepseek-v4-pro-0813") { + return { + key: PAID_PRO_MODEL_QUALITY_EXPERIMENT_KEY, + route: "pro", + controlModelKey: "model-deepseek-v4-pro-0813", + previousModelKey: "model-grok-4.6-pro", + }; + } + + return undefined; +} + +export async function evaluatePaidModelQualityExperiment({ + posthog, + userId, + mode, + subscription, + selectedModel, + requestId, +}: { + posthog: Pick | null; + userId: string; + mode: ChatMode; + subscription: SubscriptionTier; + selectedModel: ModelName; + requestId?: string; +}): Promise { + const eligibleRoute = getEligiblePaidModelQualityRoute({ + mode, + subscription, + selectedModel, + }); + if (!posthog || !eligibleRoute) return undefined; + + try { + const flags = await posthog.evaluateFlags(userId, { + flagKeys: [eligibleRoute.key], + }); + const variant = flags.getFlag(eligibleRoute.key); + + if (variant !== "control" && variant !== "test") { + return undefined; + } + + return { + key: eligibleRoute.key, + route: eligibleRoute.route, + variant, + controlModelKey: eligibleRoute.controlModelKey, + modelKey: + variant === "test" + ? eligibleRoute.previousModelKey + : eligibleRoute.controlModelKey, + }; + } catch (error) { + console.warn( + JSON.stringify({ + timestamp: new Date().toISOString(), + level: "warn", + event: "paid_model_quality_experiment_evaluation_failed", + service: "model-routing-experiment", + environment: + process.env.VERCEL_ENV ?? + process.env.TRIGGER_ENV ?? + process.env.NODE_ENV ?? + "unknown", + request_id: requestId ?? "unavailable", + user_id: userId, + flag_key: eligibleRoute.key, + experiment_route: eligibleRoute.route, + error_name: error instanceof Error ? error.name : "UnknownError", + }), + ); + return undefined; + } +} + +export function getActivePaidModelQualityExperimentAssignment( + assignment: PaidModelQualityExperimentAssignment | undefined, + selectedModel: ModelName, +): PaidModelQualityExperimentAssignment | undefined { + return assignment?.modelKey === selectedModel ? assignment : undefined; +} + +export function getPaidModelQualityExperimentContext( + assignment: PaidModelQualityExperimentAssignment | undefined, +): ExperimentAnalyticsContext | undefined { + if (!assignment) return undefined; + return { key: assignment.key, variant: assignment.variant }; +} + +export function capturePaidModelQualityExperimentExposure({ + posthog, + userId, + subscription, + mode, + selectedModelOverride, + selectedModel, + configuredModel, + assignment, +}: { + posthog: Pick | null; + userId: string; + subscription: SubscriptionTier; + mode: ChatMode; + selectedModelOverride?: SelectedModel; + selectedModel: ModelName; + configuredModel: string; + assignment?: PaidModelQualityExperimentAssignment; +}): void { + if (!posthog || !assignment) return; + + posthog.capture({ + distinctId: userId, + event: PAID_MODEL_QUALITY_EXPOSURE_EVENT, + properties: { + experiment_key: assignment.key, + experiment_variant: assignment.variant, + [`$feature/${assignment.key}`]: assignment.variant, + experiment_route: assignment.route, + subscription, + subscription_tier: subscription, + mode, + control_model: assignment.controlModelKey, + selected_model: selectedModel, + selected_model_override: selectedModelOverride ?? "auto", + configured_model: configuredModel, + exposure_surface: "provider_request", + $process_person_profile: false, + }, + }); +} + +export function capturePaidModelQualityRun({ + posthog, + userId, + experiment, + subscription, + mode, + selectedModel, + configuredModel, + responseModel, + outcome, + finishReason, + fallbackServed, + activeModelStreamDurationMs, + requestToFirstModelChunkMs, + providerRecoveryAttempts, + stepLimitReached, +}: { + posthog: Pick | null; + userId: string; + experiment?: ExperimentAnalyticsContext; + subscription: string; + mode: ChatMode; + selectedModel: string; + configuredModel: string; + responseModel?: string; + outcome: "success" | "error" | "aborted"; + finishReason?: string; + fallbackServed?: boolean; + activeModelStreamDurationMs?: number; + requestToFirstModelChunkMs?: number; + providerRecoveryAttempts?: number; + stepLimitReached?: boolean; +}): void { + if (!posthog || !experiment) return; + + const successfulRun = + outcome === "success" && + (mode === "ask" || (finishReason === "stop" && stepLimitReached !== true)); + + posthog.capture({ + distinctId: userId, + event: PAID_MODEL_QUALITY_RUN_EVENT, + properties: { + experiment_key: experiment.key, + experiment_variant: experiment.variant, + [`$feature/${experiment.key}`]: experiment.variant, + subscription, + subscription_tier: subscription, + mode, + selected_model: selectedModel, + configured_model: configuredModel, + ...(responseModel && { response_model: responseModel }), + outcome, + successful_run: successfulRun, + ...(mode === "agent" && { natural_completion: successfulRun }), + ...(finishReason && { finish_reason: finishReason }), + ...(fallbackServed !== undefined && { + fallback_served: fallbackServed, + }), + ...(activeModelStreamDurationMs !== undefined && { + active_model_stream_duration_ms: activeModelStreamDurationMs, + }), + ...(requestToFirstModelChunkMs !== undefined && { + request_to_first_model_chunk_ms: requestToFirstModelChunkMs, + }), + ...(providerRecoveryAttempts !== undefined && { + provider_recovery_attempts: providerRecoveryAttempts, + }), + ...(stepLimitReached !== undefined && { + step_limit_reached: stepLimitReached, + }), + $process_person_profile: false, + }, + }); +} diff --git a/trigger/agent-long.ts b/trigger/agent-long.ts index e673b18cc..9e610fe4d 100644 --- a/trigger/agent-long.ts +++ b/trigger/agent-long.ts @@ -145,11 +145,11 @@ import { } from "@/lib/api/agent-endpoints"; import { phLogger } from "@/lib/posthog/server"; import { - captureDeepSeekV4Pro0813ExperimentExposure, - evaluateDeepSeekV4Pro0813Experiment, - getActiveDeepSeekV4Pro0813ExperimentAssignment, - getDeepSeekV4Pro0813ExperimentContext, -} from "@/lib/experiments/deepseek-v4-pro-0813"; + capturePaidModelQualityExperimentExposure, + evaluatePaidModelQualityExperiment, + getActivePaidModelQualityExperimentAssignment, + getPaidModelQualityExperimentContext, +} from "@/lib/experiments/paid-model-quality"; import { isEligibleForDirectGlmVision } from "@/lib/chat/auxiliary-vision-eligibility"; import type { AgentAutoReviewAssignment } from "@/lib/experiments/agent-auto-review"; import { PAID_FUNNEL_EVENTS } from "@/lib/analytics/paid-funnel"; @@ -2653,15 +2653,17 @@ export const agentLongTask = task({ ); } - const deepSeekV4Pro0813Experiment = - await evaluateDeepSeekV4Pro0813Experiment({ + const paidModelQualityExperiment = + await evaluatePaidModelQualityExperiment({ posthog, userId, + mode, + subscription, selectedModel, requestId: ctx.run.id, }); - if (deepSeekV4Pro0813Experiment) { - selectedModel = deepSeekV4Pro0813Experiment.modelKey; + if (paidModelQualityExperiment) { + selectedModel = paidModelQualityExperiment.modelKey; } const notesEnabled = userCustomization?.include_notes ?? true; @@ -2934,14 +2936,14 @@ export const agentLongTask = task({ }); } - const activeDeepSeekV4Pro0813Experiment = - getActiveDeepSeekV4Pro0813ExperimentAssignment( - deepSeekV4Pro0813Experiment, + const activePaidModelQualityExperiment = + getActivePaidModelQualityExperimentAssignment( + paidModelQualityExperiment, selectedModel, ); const routingExperimentContext = - getDeepSeekV4Pro0813ExperimentContext( - activeDeepSeekV4Pro0813Experiment, + getPaidModelQualityExperimentContext( + activePaidModelQualityExperiment, ); const freeMonthlyBudgetSnapshot = @@ -4453,7 +4455,7 @@ export const agentLongTask = task({ let result; try { - captureDeepSeekV4Pro0813ExperimentExposure({ + capturePaidModelQualityExperimentExposure({ posthog, userId, subscription, @@ -4461,7 +4463,7 @@ export const agentLongTask = task({ selectedModelOverride, selectedModel, configuredModel: configuredModelId, - assignment: activeDeepSeekV4Pro0813Experiment, + assignment: activePaidModelQualityExperiment, }); result = await createStream(selectedModel); } catch (error) { From ecf8d29616b8a680105e1e05d0d3f2b700849845 Mon Sep 17 00:00:00 2001 From: Ross <144740362+ross0x01@users.noreply.github.com> Date: Wed, 2 Sep 2026 22:40:23 -0500 Subject: [PATCH 2/3] fix: use invoice quantity for paid MRR --- .../webhook/__tests__/route.test.ts | 15 +++++--- app/api/subscription/webhook/route.ts | 36 +++++++++++++------ 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/app/api/subscription/webhook/__tests__/route.test.ts b/app/api/subscription/webhook/__tests__/route.test.ts index 5bc57e85f..4a61c21ff 100644 --- a/app/api/subscription/webhook/__tests__/route.test.ts +++ b/app/api/subscription/webhook/__tests__/route.test.ts @@ -199,9 +199,11 @@ function subscriptionInvoiceLine( subscriptionId: string, priceId: string, amount: number, + quantity?: number, ) { return { amount, + ...(quantity !== undefined && { quantity }), subscription: subscriptionId, parent: { type: "subscription_item_details", @@ -1601,7 +1603,7 @@ describe("POST /api/subscription/webhook", () => { object: { id: "in_historical_price", customer: "cus_historical_price", - amount_paid: 2900, + amount_paid: 5800, currency: "usd", billing_reason: "subscription_create", parent: { @@ -1614,7 +1616,8 @@ describe("POST /api/subscription/webhook", () => { subscriptionInvoiceLine( "sub_historical_price", "price_pro_29", - 2900, + 5800, + 2, ), ], }, @@ -1678,6 +1681,8 @@ describe("POST /api/subscription/webhook", () => { expect.objectContaining({ stripePriceId: "price_pro_29", plan: "pro-monthly-plan-29-experiment", + quantity: 2, + mrrDollars: 58, }), ); expect(mockConvexMutation).toHaveBeenCalledWith( @@ -1701,9 +1706,9 @@ describe("POST /api/subscription/webhook", () => { expect(mockPostHogEvent).toHaveBeenCalledWith( "invoice_paid", expect.objectContaining({ - subscription_mrr_dollars: 29, - attributed_mrr_dollars: 29, - retained_mrr_dollars: 29, + subscription_mrr_dollars: 58, + attributed_mrr_dollars: 58, + retained_mrr_dollars: 58, }), ); }); diff --git a/app/api/subscription/webhook/route.ts b/app/api/subscription/webhook/route.ts index 42bb739da..30a6803fd 100644 --- a/app/api/subscription/webhook/route.ts +++ b/app/api/subscription/webhook/route.ts @@ -145,11 +145,11 @@ function invoiceLineIsProration(line: Stripe.InvoiceLineItem): boolean { ); } -/** Return the immutable Price ID recorded on a subscription invoice line. */ -async function invoiceSubscriptionPriceId( +/** Return immutable billing details recorded on a subscription invoice line. */ +async function invoiceSubscriptionBillingDetails( invoice: Stripe.Invoice, subscriptionId: string, -): Promise { +): Promise<{ priceId: string; quantity?: number } | undefined> { const lines = await invoiceLineItems(invoice); const candidates = lines.filter( (line) => invoiceLineSubscriptionId(line) === subscriptionId, @@ -163,7 +163,15 @@ async function invoiceSubscriptionPriceId( ); const selectedLine = recurringLine ?? candidates.find((line) => invoiceLinePriceId(line)); - return selectedLine ? invoiceLinePriceId(selectedLine) : undefined; + const priceId = selectedLine ? invoiceLinePriceId(selectedLine) : undefined; + return priceId + ? { + priceId, + ...(typeof selectedLine?.quantity === "number" && { + quantity: selectedLine.quantity, + }), + } + : undefined; } /** @@ -643,6 +651,7 @@ async function recordSubscriptionRevenue({ orgId, tier, subscription, + invoiceQuantity, reason, }: { invoice: Stripe.Invoice; @@ -652,6 +661,7 @@ async function recordSubscriptionRevenue({ orgId?: string; tier: SubscriptionTier; subscription: Stripe.Subscription; + invoiceQuantity?: number; reason: string; }) { const grossRevenueDollars = centsToDollars( @@ -668,7 +678,7 @@ async function recordSubscriptionRevenue({ reason === "subscription_create" || reason === "subscription_cycle" ? subscriptionMrrDollars({ price: invoicePrice, - quantity: item?.quantity ?? 1, + quantity: invoiceQuantity ?? item?.quantity ?? 1, fallbackTotalIntervalAmountDollars: grossRevenueDollars, }) : undefined; @@ -696,7 +706,7 @@ async function recordSubscriptionRevenue({ stripeInvoiceId: invoice.id, stripePriceId: invoicePrice.id, plan: invoicePrice.lookup_key ?? tier, - quantity: item?.quantity, + quantity: invoiceQuantity ?? item?.quantity, userCount: userIds.length, description: reason, }), @@ -721,7 +731,7 @@ async function recordSubscriptionRevenue({ stripeInvoiceId: invoice.id, stripePriceId: invoicePrice.id, plan: invoicePrice.lookup_key ?? tier, - quantity: item?.quantity, + quantity: invoiceQuantity ?? item?.quantity, userCount: userIds.length, description: reason, }), @@ -739,6 +749,7 @@ function emitInvoicePaidRevenueAnalytics({ orgId, tier, subscription, + invoiceQuantity, }: { invoice: Stripe.Invoice; invoicePrice: Stripe.Price; @@ -748,6 +759,7 @@ function emitInvoicePaidRevenueAnalytics({ orgId?: string; tier: SubscriptionTier; subscription: Stripe.Subscription; + invoiceQuantity?: number; }) { const amountPaidDollars = centsToDollars(invoice.amount_paid); if (amountPaidDollars <= 0 || userIds.length === 0) return; @@ -759,7 +771,7 @@ function emitInvoicePaidRevenueAnalytics({ const attributedRevenueDollars = amountPaidDollars / userIds.length; const subscriptionMrr = subscriptionMrrDollars({ price: invoicePrice, - quantity: subscription.items?.data[0]?.quantity ?? 1, + quantity: invoiceQuantity ?? subscription.items?.data[0]?.quantity ?? 1, fallbackTotalIntervalAmountDollars: amountPaidDollars, }); const attributedMrrDollars = @@ -1053,17 +1065,19 @@ async function handleInvoicePaid( const { tier, subscription } = resolved; const entitlementItem = subscription.items?.data[0]; const entitlementPrice = entitlementItem?.price; - const invoicePriceId = await invoiceSubscriptionPriceId( + const invoiceBillingDetails = await invoiceSubscriptionBillingDetails( invoice, subscriptionId, ); - if (!invoicePriceId) { + if (!invoiceBillingDetails) { phLogger.warn("invoice_paid_historical_price_missing", { stripe_invoice_id: invoice.id, stripe_subscription_id: subscriptionId, }); throw new Error("Historical subscription Price missing from paid invoice"); } + const { priceId: invoicePriceId, quantity: invoiceQuantity } = + invoiceBillingDetails; let invoicePrice: Stripe.Price; if (entitlementPrice?.id === invoicePriceId) { @@ -1145,6 +1159,7 @@ async function handleInvoicePaid( orgId: orgId ?? undefined, tier, subscription, + invoiceQuantity, reason: resetMode.reason, }); } catch (error) { @@ -1168,6 +1183,7 @@ async function handleInvoicePaid( orgId: orgId ?? undefined, tier, subscription, + invoiceQuantity, }); if (resetMode.mode === "skip") { From 12e1f11f0ad378c8a0eb61a1aca7369913cc1b42 Mon Sep 17 00:00:00 2001 From: Ross <144740362+ross0x01@users.noreply.github.com> Date: Thu, 3 Sep 2026 10:23:17 -0500 Subject: [PATCH 3/3] chore: remove paid model quality experiments --- lib/api/__tests__/chat-logger.test.ts | 50 ---- lib/api/chat-handler.ts | 34 ++- lib/api/chat-logger.ts | 18 -- .../__tests__/deepseek-v4-pro-0813.test.ts | 122 ++++++++ .../__tests__/paid-model-quality.test.ts | 255 ----------------- lib/experiments/deepseek-v4-pro-0813.ts | 132 +++++++++ lib/experiments/paid-model-quality.ts | 270 ------------------ trigger/agent-long.ts | 34 ++- 8 files changed, 286 insertions(+), 629 deletions(-) create mode 100644 lib/experiments/__tests__/deepseek-v4-pro-0813.test.ts delete mode 100644 lib/experiments/__tests__/paid-model-quality.test.ts create mode 100644 lib/experiments/deepseek-v4-pro-0813.ts delete mode 100644 lib/experiments/paid-model-quality.ts diff --git a/lib/api/__tests__/chat-logger.test.ts b/lib/api/__tests__/chat-logger.test.ts index 65e0a8242..eadbbe93c 100644 --- a/lib/api/__tests__/chat-logger.test.ts +++ b/lib/api/__tests__/chat-logger.test.ts @@ -638,56 +638,6 @@ describe("captureAgentCompletionAnalytics", () => { }, }); }); - - it("adds a cross-mode model-quality outcome only for an active experiment", () => { - const capture = jest.fn(); - - captureAgentCompletionAnalytics({ - posthog: { capture } as any, - userId: "user_123", - chatId: "chat_123", - endpoint: "/api/chat", - mode: "ask", - subscription: "pro-plus", - sandboxInfo: null, - outcome: "success", - chatLogger: undefined, - selectedModel: "model-deepseek-v4-pro", - configuredModelId: "deepseek/deepseek-v4-pro", - responseModel: "deepseek/deepseek-v4-pro", - fallbackServed: false, - finishReason: "stop", - activeModelStreamDurationMs: 9_000, - requestToFirstModelChunkMs: 700, - providerRecoveryAttempts: 0, - experiment: { - key: "paid_standard_model_quality_v1", - variant: "test", - }, - }); - - expect(capture).toHaveBeenCalledTimes(1); - expect(capture).toHaveBeenCalledWith({ - distinctId: "user_123", - event: "paid_model_quality_run_completed", - properties: expect.objectContaining({ - experiment_key: "paid_standard_model_quality_v1", - experiment_variant: "test", - "$feature/paid_standard_model_quality_v1": "test", - subscription_tier: "pro-plus", - mode: "ask", - selected_model: "model-deepseek-v4-pro", - configured_model: "deepseek/deepseek-v4-pro", - outcome: "success", - successful_run: true, - fallback_served: false, - active_model_stream_duration_ms: 9_000, - request_to_first_model_chunk_ms: 700, - provider_recovery_attempts: 0, - $process_person_profile: false, - }), - }); - }); }); describe("captureUsageCost", () => { diff --git a/lib/api/chat-handler.ts b/lib/api/chat-handler.ts index 2ac270344..6e2d22568 100644 --- a/lib/api/chat-handler.ts +++ b/lib/api/chat-handler.ts @@ -159,11 +159,11 @@ import { PAID_FUNNEL_EVENTS } from "@/lib/analytics/paid-funnel"; import { readAnalyticsRequestContext } from "@/lib/analytics/request-context"; import { buildAgentStepLimitTelemetry } from "@/lib/analytics/agent-step-limit-telemetry"; import { - capturePaidModelQualityExperimentExposure, - evaluatePaidModelQualityExperiment, - getActivePaidModelQualityExperimentAssignment, - getPaidModelQualityExperimentContext, -} from "@/lib/experiments/paid-model-quality"; + captureDeepSeekV4Pro0813ExperimentExposure, + evaluateDeepSeekV4Pro0813Experiment, + getActiveDeepSeekV4Pro0813ExperimentAssignment, + getDeepSeekV4Pro0813ExperimentContext, +} from "@/lib/experiments/deepseek-v4-pro-0813"; import { isEligibleForDirectGlmVision } from "@/lib/chat/auxiliary-vision-eligibility"; import { capturePaidDailyFreeAllowanceServerEvent, @@ -476,17 +476,15 @@ export const createChatHandler = () => { ); } - const paidModelQualityExperiment = - await evaluatePaidModelQualityExperiment({ + const deepSeekV4Pro0813Experiment = + await evaluateDeepSeekV4Pro0813Experiment({ posthog: (posthog ??= PostHogClient()), userId, - mode, - subscription, selectedModel, requestId: req.headers.get("x-vercel-id") ?? undefined, }); - if (paidModelQualityExperiment) { - selectedModel = paidModelQualityExperiment.modelKey; + if (deepSeekV4Pro0813Experiment) { + selectedModel = deepSeekV4Pro0813Experiment.modelKey; } const notesEnabled = (subscription !== "free" || isAgentMode(mode)) && @@ -623,13 +621,13 @@ export const createChatHandler = () => { }); } - const activePaidModelQualityExperiment = - getActivePaidModelQualityExperimentAssignment( - paidModelQualityExperiment, + const activeDeepSeekV4Pro0813Experiment = + getActiveDeepSeekV4Pro0813ExperimentAssignment( + deepSeekV4Pro0813Experiment, selectedModel, ); - const routingExperimentContext = getPaidModelQualityExperimentContext( - activePaidModelQualityExperiment, + const routingExperimentContext = getDeepSeekV4Pro0813ExperimentContext( + activeDeepSeekV4Pro0813Experiment, ); const freeMonthlyBudgetSnapshot = @@ -1493,7 +1491,7 @@ export const createChatHandler = () => { let result; try { - capturePaidModelQualityExperimentExposure({ + captureDeepSeekV4Pro0813ExperimentExposure({ posthog, userId, subscription, @@ -1501,7 +1499,7 @@ export const createChatHandler = () => { selectedModelOverride, selectedModel, configuredModel: configuredModelId, - assignment: activePaidModelQualityExperiment, + assignment: activeDeepSeekV4Pro0813Experiment, }); result = await createStream(selectedModel); } catch (error) { diff --git a/lib/api/chat-logger.ts b/lib/api/chat-logger.ts index a0d50ba5d..0fd61b9ac 100644 --- a/lib/api/chat-logger.ts +++ b/lib/api/chat-logger.ts @@ -34,7 +34,6 @@ import { getExperimentAnalyticsProperties, type ExperimentAnalyticsContext, } from "@/lib/analytics/experiment-context"; -import { capturePaidModelQualityRun } from "@/lib/experiments/paid-model-quality"; import type { AgentStepLimitTelemetry } from "@/lib/analytics/agent-step-limit-telemetry"; import { buildAgentPerformanceDiagnostics } from "@/lib/analytics/agent-performance-diagnostics"; import { @@ -1636,23 +1635,6 @@ export function captureAgentCompletionAnalytics( args: AgentCompletionAnalyticsArgs, ) { const { posthog, userId, mode, subscription, sandboxInfo, outcome } = args; - capturePaidModelQualityRun({ - posthog, - userId, - experiment: args.experiment, - subscription, - mode, - selectedModel: args.selectedModel, - configuredModel: args.configuredModelId, - responseModel: args.responseModel, - outcome, - finishReason: args.finishReason, - fallbackServed: args.fallbackServed, - activeModelStreamDurationMs: args.activeModelStreamDurationMs, - requestToFirstModelChunkMs: args.requestToFirstModelChunkMs, - providerRecoveryAttempts: args.providerRecoveryAttempts, - stepLimitReached: args.stepLimitTelemetry?.stepLimitReached, - }); captureAgentRun({ posthog, userId, diff --git a/lib/experiments/__tests__/deepseek-v4-pro-0813.test.ts b/lib/experiments/__tests__/deepseek-v4-pro-0813.test.ts new file mode 100644 index 000000000..7b9e1b2bb --- /dev/null +++ b/lib/experiments/__tests__/deepseek-v4-pro-0813.test.ts @@ -0,0 +1,122 @@ +import { + captureDeepSeekV4Pro0813ExperimentExposure, + DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY, + DEEPSEEK_V4_PRO_0813_EXPOSURE_EVENT, + DEEPSEEK_V4_PRO_0813_FEATURE_PROPERTY, + evaluateDeepSeekV4Pro0813Experiment, + getActiveDeepSeekV4Pro0813ExperimentAssignment, + getDeepSeekV4Pro0813ExperimentContext, + isEligibleForDeepSeekV4Pro0813Experiment, +} from "@/lib/experiments/deepseek-v4-pro-0813"; + +describe("DeepSeek V4 Pro 0813 experiment", () => { + it("only evaluates requests already resolved to the current DeepSeek V4 Pro route", () => { + expect( + isEligibleForDeepSeekV4Pro0813Experiment("model-deepseek-v4-pro"), + ).toBe(true); + expect(isEligibleForDeepSeekV4Pro0813Experiment("model-grok-4.5")).toBe( + false, + ); + expect(isEligibleForDeepSeekV4Pro0813Experiment("agent-model-free")).toBe( + false, + ); + }); + + it.each([ + ["control", "model-deepseek-v4-pro"], + ["test", "model-deepseek-v4-pro-0813"], + ] as const)("maps %s to %s", async (variant, modelKey) => { + const evaluateFlags = jest.fn(async () => ({ getFlag: () => variant })); + + await expect( + evaluateDeepSeekV4Pro0813Experiment({ + posthog: { evaluateFlags } as never, + userId: "user-1", + selectedModel: "model-deepseek-v4-pro", + }), + ).resolves.toEqual({ + key: DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY, + variant, + modelKey, + }); + }); + + it("does not evaluate ineligible routes", async () => { + const evaluateFlags = jest.fn(); + + await expect( + evaluateDeepSeekV4Pro0813Experiment({ + posthog: { evaluateFlags } as never, + userId: "user-1", + selectedModel: "model-grok-4.5", + }), + ).resolves.toBeUndefined(); + expect(evaluateFlags).not.toHaveBeenCalled(); + }); + + it("fails closed when evaluation returns an unknown value", async () => { + await expect( + evaluateDeepSeekV4Pro0813Experiment({ + posthog: { + evaluateFlags: jest.fn(async () => ({ getFlag: () => true })), + } as never, + userId: "user-1", + selectedModel: "model-deepseek-v4-pro", + }), + ).resolves.toBeUndefined(); + }); + + it("captures a privacy-safe custom exposure from the provider surface", () => { + const capture = jest.fn(); + const assignment = { + key: DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY, + variant: "test" as const, + modelKey: "model-deepseek-v4-pro-0813" as const, + }; + + captureDeepSeekV4Pro0813ExperimentExposure({ + posthog: { capture } as never, + userId: "user-1", + subscription: "pro", + mode: "agent", + selectedModelOverride: "hackerai-standard", + selectedModel: assignment.modelKey, + configuredModel: "deepseek/deepseek-v4-pro-0813", + assignment, + }); + + expect(capture).toHaveBeenCalledWith({ + distinctId: "user-1", + event: DEEPSEEK_V4_PRO_0813_EXPOSURE_EVENT, + properties: { + experiment_key: DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY, + experiment_variant: "test", + [DEEPSEEK_V4_PRO_0813_FEATURE_PROPERTY]: "test", + subscription: "pro", + subscription_tier: "pro", + mode: "agent", + selected_model: "model-deepseek-v4-pro-0813", + selected_model_override: "hackerai-standard", + configured_model: "deepseek/deepseek-v4-pro-0813", + exposure_surface: "provider_request", + $process_person_profile: false, + }, + }); + expect(getDeepSeekV4Pro0813ExperimentContext(assignment)).toEqual({ + key: DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY, + variant: "test", + }); + expect( + getActiveDeepSeekV4Pro0813ExperimentAssignment( + assignment, + "model-deepseek-v4-pro-0813", + ), + ).toBe(assignment); + expect( + getActiveDeepSeekV4Pro0813ExperimentAssignment( + assignment, + "model-grok-4.5", + ), + ).toBeUndefined(); + }); +}); diff --git a/lib/experiments/__tests__/paid-model-quality.test.ts b/lib/experiments/__tests__/paid-model-quality.test.ts deleted file mode 100644 index dad95b6b0..000000000 --- a/lib/experiments/__tests__/paid-model-quality.test.ts +++ /dev/null @@ -1,255 +0,0 @@ -import { - capturePaidModelQualityExperimentExposure, - capturePaidModelQualityRun, - evaluatePaidModelQualityExperiment, - getActivePaidModelQualityExperimentAssignment, - getEligiblePaidModelQualityRoute, - getPaidModelQualityExperimentContext, - PAID_MODEL_QUALITY_EXPOSURE_EVENT, - PAID_MODEL_QUALITY_RUN_EVENT, - PAID_PRO_MODEL_QUALITY_EXPERIMENT_KEY, - PAID_STANDARD_MODEL_QUALITY_EXPERIMENT_KEY, -} from "@/lib/experiments/paid-model-quality"; - -describe("paid model quality experiments", () => { - it.each([ - ["ask", "model-deepseek-v4-flash-0731"], - ["agent", "model-glm-5.3-flash-agent"], - ] as const)( - "recognizes the current %s Standard route", - (mode, selectedModel) => { - expect( - getEligiblePaidModelQualityRoute({ - mode, - subscription: "pro", - selectedModel, - }), - ).toMatchObject({ - key: PAID_STANDARD_MODEL_QUALITY_EXPERIMENT_KEY, - route: "standard", - controlModelKey: selectedModel, - previousModelKey: "model-deepseek-v4-pro", - }); - }, - ); - - it("recognizes the current Pro route", () => { - expect( - getEligiblePaidModelQualityRoute({ - mode: "agent", - subscription: "pro-plus", - selectedModel: "model-deepseek-v4-pro-0813", - }), - ).toMatchObject({ - key: PAID_PRO_MODEL_QUALITY_EXPERIMENT_KEY, - route: "pro", - previousModelKey: "model-grok-4.6-pro", - }); - }); - - it.each([ - ["free", "agent", "model-glm-5.3-flash-agent"], - ["team", "agent", "model-glm-5.3-flash-agent"], - ["ultra", "agent", "model-deepseek-v4-pro-0813"], - ["pro", "agent", "model-grok-4.5"], - ["pro", "ask", "model-glm-5.3-flash-agent"], - ] as const)( - "excludes %s %s route %s", - (subscription, mode, selectedModel) => { - expect( - getEligiblePaidModelQualityRoute({ - mode, - subscription, - selectedModel, - }), - ).toBeUndefined(); - }, - ); - - it.each([ - ["ask", "model-deepseek-v4-flash-0731", "test", "model-deepseek-v4-pro"], - [ - "agent", - "model-glm-5.3-flash-agent", - "control", - "model-glm-5.3-flash-agent", - ], - ["agent", "model-deepseek-v4-pro-0813", "test", "model-grok-4.6-pro"], - ] as const)( - "maps %s %s variant %s to %s", - async (mode, selectedModel, variant, modelKey) => { - const evaluateFlags = jest.fn(async () => ({ getFlag: () => variant })); - - await expect( - evaluatePaidModelQualityExperiment({ - posthog: { evaluateFlags } as never, - userId: "user-1", - mode, - subscription: "pro", - selectedModel, - }), - ).resolves.toMatchObject({ variant, modelKey }); - }, - ); - - it("evaluates only the flag for the resolved route", async () => { - const evaluateFlags = jest.fn(async () => ({ getFlag: () => "control" })); - - await evaluatePaidModelQualityExperiment({ - posthog: { evaluateFlags } as never, - userId: "user-1", - mode: "agent", - subscription: "pro-plus", - selectedModel: "model-deepseek-v4-pro-0813", - }); - - expect(evaluateFlags).toHaveBeenCalledWith("user-1", { - flagKeys: [PAID_PRO_MODEL_QUALITY_EXPERIMENT_KEY], - }); - }); - - it("fails closed for unknown values and evaluation errors", async () => { - await expect( - evaluatePaidModelQualityExperiment({ - posthog: { - evaluateFlags: jest.fn(async () => ({ getFlag: () => true })), - } as never, - userId: "user-1", - mode: "ask", - subscription: "pro", - selectedModel: "model-deepseek-v4-flash-0731", - }), - ).resolves.toBeUndefined(); - - const warn = jest.spyOn(console, "warn").mockImplementation(); - await expect( - evaluatePaidModelQualityExperiment({ - posthog: { - evaluateFlags: jest.fn(async () => { - throw new Error("unavailable"); - }), - } as never, - userId: "user-1", - mode: "ask", - subscription: "pro", - selectedModel: "model-deepseek-v4-flash-0731", - }), - ).resolves.toBeUndefined(); - expect(warn).toHaveBeenCalled(); - warn.mockRestore(); - }); - - it("drops attribution if a later limit-rescue route replaces the assignment", () => { - const assignment = { - key: PAID_STANDARD_MODEL_QUALITY_EXPERIMENT_KEY, - route: "standard" as const, - variant: "test" as const, - controlModelKey: "model-glm-5.3-flash-agent" as const, - modelKey: "model-deepseek-v4-pro" as const, - }; - - expect( - getActivePaidModelQualityExperimentAssignment( - assignment, - "model-deepseek-v4-pro", - ), - ).toBe(assignment); - expect( - getActivePaidModelQualityExperimentAssignment( - assignment, - "agent-model-free", - ), - ).toBeUndefined(); - expect(getPaidModelQualityExperimentContext(assignment)).toEqual({ - key: PAID_STANDARD_MODEL_QUALITY_EXPERIMENT_KEY, - variant: "test", - }); - }); - - it("captures a privacy-safe provider-boundary exposure", () => { - const capture = jest.fn(); - const assignment = { - key: PAID_STANDARD_MODEL_QUALITY_EXPERIMENT_KEY, - route: "standard" as const, - variant: "test" as const, - controlModelKey: "model-glm-5.3-flash-agent" as const, - modelKey: "model-deepseek-v4-pro" as const, - }; - - capturePaidModelQualityExperimentExposure({ - posthog: { capture } as never, - userId: "user-1", - subscription: "pro-plus", - mode: "agent", - selectedModelOverride: "hackerai-standard", - selectedModel: assignment.modelKey, - configuredModel: "deepseek/deepseek-v4-pro", - assignment, - }); - - expect(capture).toHaveBeenCalledWith({ - distinctId: "user-1", - event: PAID_MODEL_QUALITY_EXPOSURE_EVENT, - properties: { - experiment_key: PAID_STANDARD_MODEL_QUALITY_EXPERIMENT_KEY, - experiment_variant: "test", - [`$feature/${PAID_STANDARD_MODEL_QUALITY_EXPERIMENT_KEY}`]: "test", - experiment_route: "standard", - subscription: "pro-plus", - subscription_tier: "pro-plus", - mode: "agent", - control_model: "model-glm-5.3-flash-agent", - selected_model: "model-deepseek-v4-pro", - selected_model_override: "hackerai-standard", - configured_model: "deepseek/deepseek-v4-pro", - exposure_surface: "provider_request", - $process_person_profile: false, - }, - }); - }); - - it.each([ - ["ask", "success", "length", false, true], - ["agent", "success", "stop", false, true], - ["agent", "success", "length", true, false], - ["agent", "error", "error", false, false], - ] as const)( - "classifies %s outcome=%s finish=%s stepLimit=%s as successful=%s", - (mode, outcome, finishReason, stepLimitReached, successfulRun) => { - const capture = jest.fn(); - - capturePaidModelQualityRun({ - posthog: { capture } as never, - userId: "user-1", - experiment: { - key: PAID_PRO_MODEL_QUALITY_EXPERIMENT_KEY, - variant: "control", - }, - subscription: "pro", - mode, - selectedModel: "model-deepseek-v4-pro-0813", - configuredModel: "deepseek/deepseek-v4-pro-0813", - responseModel: "deepseek/deepseek-v4-pro-0813", - outcome, - finishReason, - fallbackServed: false, - activeModelStreamDurationMs: 12_000, - requestToFirstModelChunkMs: 800, - providerRecoveryAttempts: 0, - stepLimitReached, - }); - - expect(capture).toHaveBeenCalledWith({ - distinctId: "user-1", - event: PAID_MODEL_QUALITY_RUN_EVENT, - properties: expect.objectContaining({ - experiment_key: PAID_PRO_MODEL_QUALITY_EXPERIMENT_KEY, - experiment_variant: "control", - successful_run: successfulRun, - outcome, - mode, - }), - }); - }, - ); -}); diff --git a/lib/experiments/deepseek-v4-pro-0813.ts b/lib/experiments/deepseek-v4-pro-0813.ts new file mode 100644 index 000000000..424aadfcc --- /dev/null +++ b/lib/experiments/deepseek-v4-pro-0813.ts @@ -0,0 +1,132 @@ +import type { PostHog } from "posthog-node"; +import type { ExperimentAnalyticsContext } from "@/lib/analytics/experiment-context"; +import type { ModelName } from "@/lib/ai/providers"; +import type { ChatMode, SelectedModel, SubscriptionTier } from "@/types"; + +export const DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY = + "deepseek_v4_pro_0813_model_v1"; +export const DEEPSEEK_V4_PRO_0813_EXPOSURE_EVENT = + "hac68_deepseek_v4_pro_0813_experiment_exposed"; +export const DEEPSEEK_V4_PRO_0813_FEATURE_PROPERTY = `$feature/${DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY}`; + +export type DeepSeekV4Pro0813ExperimentVariant = "control" | "test"; +export type DeepSeekV4Pro0813ModelKey = + "model-deepseek-v4-pro" | "model-deepseek-v4-pro-0813"; + +export type DeepSeekV4Pro0813ExperimentAssignment = { + key: typeof DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY; + variant: DeepSeekV4Pro0813ExperimentVariant; + modelKey: DeepSeekV4Pro0813ModelKey; +}; + +export function isEligibleForDeepSeekV4Pro0813Experiment( + selectedModel: ModelName, +): selectedModel is "model-deepseek-v4-pro" { + return selectedModel === "model-deepseek-v4-pro"; +} + +export async function evaluateDeepSeekV4Pro0813Experiment({ + posthog, + userId, + selectedModel, + requestId, +}: { + posthog: Pick | null; + userId: string; + selectedModel: ModelName; + requestId?: string; +}): Promise { + if (!posthog || !isEligibleForDeepSeekV4Pro0813Experiment(selectedModel)) { + return undefined; + } + + try { + const flags = await posthog.evaluateFlags(userId, { + flagKeys: [DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY], + }); + const variant = flags.getFlag(DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY); + + if (variant !== "control" && variant !== "test") { + return undefined; + } + + return { + key: DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY, + variant, + modelKey: + variant === "test" + ? "model-deepseek-v4-pro-0813" + : "model-deepseek-v4-pro", + }; + } catch (error) { + console.warn( + JSON.stringify({ + timestamp: new Date().toISOString(), + level: "warn", + event: "deepseek_v4_pro_0813_experiment_evaluation_failed", + service: "model-routing-experiment", + environment: + process.env.VERCEL_ENV ?? process.env.NODE_ENV ?? "unknown", + request_id: requestId ?? "unavailable", + user_id: userId, + flag_key: DEEPSEEK_V4_PRO_0813_EXPERIMENT_KEY, + error_name: error instanceof Error ? error.name : "UnknownError", + }), + ); + return undefined; + } +} + +export function captureDeepSeekV4Pro0813ExperimentExposure({ + posthog, + userId, + subscription, + mode, + selectedModelOverride, + selectedModel, + configuredModel, + assignment, +}: { + posthog: Pick | null; + userId: string; + subscription: SubscriptionTier; + mode: ChatMode; + selectedModelOverride?: SelectedModel; + selectedModel: ModelName; + configuredModel: string; + assignment?: DeepSeekV4Pro0813ExperimentAssignment; +}): void { + if (!posthog || !assignment) return; + + posthog.capture({ + distinctId: userId, + event: DEEPSEEK_V4_PRO_0813_EXPOSURE_EVENT, + properties: { + experiment_key: assignment.key, + experiment_variant: assignment.variant, + [DEEPSEEK_V4_PRO_0813_FEATURE_PROPERTY]: assignment.variant, + subscription, + subscription_tier: subscription, + mode, + selected_model: selectedModel, + selected_model_override: selectedModelOverride ?? "auto", + configured_model: configuredModel, + exposure_surface: "provider_request", + $process_person_profile: false, + }, + }); +} + +export function getDeepSeekV4Pro0813ExperimentContext( + assignment: DeepSeekV4Pro0813ExperimentAssignment | undefined, +): ExperimentAnalyticsContext | undefined { + if (!assignment) return undefined; + return { key: assignment.key, variant: assignment.variant }; +} + +export function getActiveDeepSeekV4Pro0813ExperimentAssignment( + assignment: DeepSeekV4Pro0813ExperimentAssignment | undefined, + selectedModel: ModelName, +): DeepSeekV4Pro0813ExperimentAssignment | undefined { + return assignment?.modelKey === selectedModel ? assignment : undefined; +} diff --git a/lib/experiments/paid-model-quality.ts b/lib/experiments/paid-model-quality.ts deleted file mode 100644 index b3f4a824a..000000000 --- a/lib/experiments/paid-model-quality.ts +++ /dev/null @@ -1,270 +0,0 @@ -import type { PostHog } from "posthog-node"; -import type { ExperimentAnalyticsContext } from "@/lib/analytics/experiment-context"; -import type { ModelName } from "@/lib/ai/providers"; -import type { ChatMode, SelectedModel, SubscriptionTier } from "@/types"; - -export const PAID_STANDARD_MODEL_QUALITY_EXPERIMENT_KEY = - "paid_standard_model_quality_v1"; -export const PAID_PRO_MODEL_QUALITY_EXPERIMENT_KEY = - "paid_pro_model_quality_v1"; -export const PAID_MODEL_QUALITY_EXPOSURE_EVENT = - "paid_model_quality_experiment_exposed"; -export const PAID_MODEL_QUALITY_RUN_EVENT = "paid_model_quality_run_completed"; - -export type PaidModelQualityRoute = "standard" | "pro"; -export type PaidModelQualityExperimentVariant = "control" | "test"; - -export type PaidModelQualityExperimentAssignment = { - key: - | typeof PAID_STANDARD_MODEL_QUALITY_EXPERIMENT_KEY - | typeof PAID_PRO_MODEL_QUALITY_EXPERIMENT_KEY; - route: PaidModelQualityRoute; - variant: PaidModelQualityExperimentVariant; - controlModelKey: ModelName; - modelKey: ModelName; -}; - -type EligibleRoute = Pick< - PaidModelQualityExperimentAssignment, - "key" | "route" | "controlModelKey" -> & { - previousModelKey: ModelName; -}; - -function isTargetSubscription(subscription: SubscriptionTier): boolean { - return subscription === "pro" || subscription === "pro-plus"; -} - -export function getEligiblePaidModelQualityRoute({ - mode, - subscription, - selectedModel, -}: { - mode: ChatMode; - subscription: SubscriptionTier; - selectedModel: ModelName; -}): EligibleRoute | undefined { - if (!isTargetSubscription(subscription)) return undefined; - - const standardControlModel = - mode === "agent" - ? "model-glm-5.3-flash-agent" - : "model-deepseek-v4-flash-0731"; - - if (selectedModel === standardControlModel) { - return { - key: PAID_STANDARD_MODEL_QUALITY_EXPERIMENT_KEY, - route: "standard", - controlModelKey: standardControlModel, - previousModelKey: "model-deepseek-v4-pro", - }; - } - - if (selectedModel === "model-deepseek-v4-pro-0813") { - return { - key: PAID_PRO_MODEL_QUALITY_EXPERIMENT_KEY, - route: "pro", - controlModelKey: "model-deepseek-v4-pro-0813", - previousModelKey: "model-grok-4.6-pro", - }; - } - - return undefined; -} - -export async function evaluatePaidModelQualityExperiment({ - posthog, - userId, - mode, - subscription, - selectedModel, - requestId, -}: { - posthog: Pick | null; - userId: string; - mode: ChatMode; - subscription: SubscriptionTier; - selectedModel: ModelName; - requestId?: string; -}): Promise { - const eligibleRoute = getEligiblePaidModelQualityRoute({ - mode, - subscription, - selectedModel, - }); - if (!posthog || !eligibleRoute) return undefined; - - try { - const flags = await posthog.evaluateFlags(userId, { - flagKeys: [eligibleRoute.key], - }); - const variant = flags.getFlag(eligibleRoute.key); - - if (variant !== "control" && variant !== "test") { - return undefined; - } - - return { - key: eligibleRoute.key, - route: eligibleRoute.route, - variant, - controlModelKey: eligibleRoute.controlModelKey, - modelKey: - variant === "test" - ? eligibleRoute.previousModelKey - : eligibleRoute.controlModelKey, - }; - } catch (error) { - console.warn( - JSON.stringify({ - timestamp: new Date().toISOString(), - level: "warn", - event: "paid_model_quality_experiment_evaluation_failed", - service: "model-routing-experiment", - environment: - process.env.VERCEL_ENV ?? - process.env.TRIGGER_ENV ?? - process.env.NODE_ENV ?? - "unknown", - request_id: requestId ?? "unavailable", - user_id: userId, - flag_key: eligibleRoute.key, - experiment_route: eligibleRoute.route, - error_name: error instanceof Error ? error.name : "UnknownError", - }), - ); - return undefined; - } -} - -export function getActivePaidModelQualityExperimentAssignment( - assignment: PaidModelQualityExperimentAssignment | undefined, - selectedModel: ModelName, -): PaidModelQualityExperimentAssignment | undefined { - return assignment?.modelKey === selectedModel ? assignment : undefined; -} - -export function getPaidModelQualityExperimentContext( - assignment: PaidModelQualityExperimentAssignment | undefined, -): ExperimentAnalyticsContext | undefined { - if (!assignment) return undefined; - return { key: assignment.key, variant: assignment.variant }; -} - -export function capturePaidModelQualityExperimentExposure({ - posthog, - userId, - subscription, - mode, - selectedModelOverride, - selectedModel, - configuredModel, - assignment, -}: { - posthog: Pick | null; - userId: string; - subscription: SubscriptionTier; - mode: ChatMode; - selectedModelOverride?: SelectedModel; - selectedModel: ModelName; - configuredModel: string; - assignment?: PaidModelQualityExperimentAssignment; -}): void { - if (!posthog || !assignment) return; - - posthog.capture({ - distinctId: userId, - event: PAID_MODEL_QUALITY_EXPOSURE_EVENT, - properties: { - experiment_key: assignment.key, - experiment_variant: assignment.variant, - [`$feature/${assignment.key}`]: assignment.variant, - experiment_route: assignment.route, - subscription, - subscription_tier: subscription, - mode, - control_model: assignment.controlModelKey, - selected_model: selectedModel, - selected_model_override: selectedModelOverride ?? "auto", - configured_model: configuredModel, - exposure_surface: "provider_request", - $process_person_profile: false, - }, - }); -} - -export function capturePaidModelQualityRun({ - posthog, - userId, - experiment, - subscription, - mode, - selectedModel, - configuredModel, - responseModel, - outcome, - finishReason, - fallbackServed, - activeModelStreamDurationMs, - requestToFirstModelChunkMs, - providerRecoveryAttempts, - stepLimitReached, -}: { - posthog: Pick | null; - userId: string; - experiment?: ExperimentAnalyticsContext; - subscription: string; - mode: ChatMode; - selectedModel: string; - configuredModel: string; - responseModel?: string; - outcome: "success" | "error" | "aborted"; - finishReason?: string; - fallbackServed?: boolean; - activeModelStreamDurationMs?: number; - requestToFirstModelChunkMs?: number; - providerRecoveryAttempts?: number; - stepLimitReached?: boolean; -}): void { - if (!posthog || !experiment) return; - - const successfulRun = - outcome === "success" && - (mode === "ask" || (finishReason === "stop" && stepLimitReached !== true)); - - posthog.capture({ - distinctId: userId, - event: PAID_MODEL_QUALITY_RUN_EVENT, - properties: { - experiment_key: experiment.key, - experiment_variant: experiment.variant, - [`$feature/${experiment.key}`]: experiment.variant, - subscription, - subscription_tier: subscription, - mode, - selected_model: selectedModel, - configured_model: configuredModel, - ...(responseModel && { response_model: responseModel }), - outcome, - successful_run: successfulRun, - ...(mode === "agent" && { natural_completion: successfulRun }), - ...(finishReason && { finish_reason: finishReason }), - ...(fallbackServed !== undefined && { - fallback_served: fallbackServed, - }), - ...(activeModelStreamDurationMs !== undefined && { - active_model_stream_duration_ms: activeModelStreamDurationMs, - }), - ...(requestToFirstModelChunkMs !== undefined && { - request_to_first_model_chunk_ms: requestToFirstModelChunkMs, - }), - ...(providerRecoveryAttempts !== undefined && { - provider_recovery_attempts: providerRecoveryAttempts, - }), - ...(stepLimitReached !== undefined && { - step_limit_reached: stepLimitReached, - }), - $process_person_profile: false, - }, - }); -} diff --git a/trigger/agent-long.ts b/trigger/agent-long.ts index 9e610fe4d..e673b18cc 100644 --- a/trigger/agent-long.ts +++ b/trigger/agent-long.ts @@ -145,11 +145,11 @@ import { } from "@/lib/api/agent-endpoints"; import { phLogger } from "@/lib/posthog/server"; import { - capturePaidModelQualityExperimentExposure, - evaluatePaidModelQualityExperiment, - getActivePaidModelQualityExperimentAssignment, - getPaidModelQualityExperimentContext, -} from "@/lib/experiments/paid-model-quality"; + captureDeepSeekV4Pro0813ExperimentExposure, + evaluateDeepSeekV4Pro0813Experiment, + getActiveDeepSeekV4Pro0813ExperimentAssignment, + getDeepSeekV4Pro0813ExperimentContext, +} from "@/lib/experiments/deepseek-v4-pro-0813"; import { isEligibleForDirectGlmVision } from "@/lib/chat/auxiliary-vision-eligibility"; import type { AgentAutoReviewAssignment } from "@/lib/experiments/agent-auto-review"; import { PAID_FUNNEL_EVENTS } from "@/lib/analytics/paid-funnel"; @@ -2653,17 +2653,15 @@ export const agentLongTask = task({ ); } - const paidModelQualityExperiment = - await evaluatePaidModelQualityExperiment({ + const deepSeekV4Pro0813Experiment = + await evaluateDeepSeekV4Pro0813Experiment({ posthog, userId, - mode, - subscription, selectedModel, requestId: ctx.run.id, }); - if (paidModelQualityExperiment) { - selectedModel = paidModelQualityExperiment.modelKey; + if (deepSeekV4Pro0813Experiment) { + selectedModel = deepSeekV4Pro0813Experiment.modelKey; } const notesEnabled = userCustomization?.include_notes ?? true; @@ -2936,14 +2934,14 @@ export const agentLongTask = task({ }); } - const activePaidModelQualityExperiment = - getActivePaidModelQualityExperimentAssignment( - paidModelQualityExperiment, + const activeDeepSeekV4Pro0813Experiment = + getActiveDeepSeekV4Pro0813ExperimentAssignment( + deepSeekV4Pro0813Experiment, selectedModel, ); const routingExperimentContext = - getPaidModelQualityExperimentContext( - activePaidModelQualityExperiment, + getDeepSeekV4Pro0813ExperimentContext( + activeDeepSeekV4Pro0813Experiment, ); const freeMonthlyBudgetSnapshot = @@ -4455,7 +4453,7 @@ export const agentLongTask = task({ let result; try { - capturePaidModelQualityExperimentExposure({ + captureDeepSeekV4Pro0813ExperimentExposure({ posthog, userId, subscription, @@ -4463,7 +4461,7 @@ export const agentLongTask = task({ selectedModelOverride, selectedModel, configuredModel: configuredModelId, - assignment: activePaidModelQualityExperiment, + assignment: activeDeepSeekV4Pro0813Experiment, }); result = await createStream(selectedModel); } catch (error) {