docs(benchmarking): Refresh Sentry corpus benchmark results

package.json

@@ -22,6 +22,7 @@
     "update-pricing-pr": "pnpm --filter @sentry/warden update-pricing-pr",
     "generate:jsonl-schema": "pnpm --filter @sentry/warden generate:jsonl-schema",
     "docs": "pnpm --filter warden-docs dev",
+    "docs:check": "pnpm -C packages/docs check",
     "prepare": "simple-git-hooks",
     "pack:warden": "pnpm --filter @sentry/warden pack"
   },

packages/docs/package.json

@@ -5,6 +5,7 @@
   "scripts": {
     "dev": "astro dev",
     "build": "astro build",
+    "check": "node scripts/validate-benchmark-results.mjs && astro build",
     "preview": "astro preview"
   },
   "dependencies": {

packages/docs/scripts/validate-benchmark-results.mjs

@@ -0,0 +1,156 @@
+import {readdirSync, readFileSync} from "node:fs";
+import {join} from "node:path";
+import {fileURLToPath} from "node:url";
+
+const docsRoot = fileURLToPath(new URL("..", import.meta.url));
+const resultsDir = join(docsRoot, "src/data/benchmarking/results");
+const corpusPath = join(docsRoot, "src/data/benchmarking/sentry-vulnerability-corpus.json");
+
+const corpus = JSON.parse(readFileSync(corpusPath, "utf8"));
+const corpusIds = new Set(corpus.findings.map((finding) => finding.id));
+const errors = [];
+
+const sum = (items, field) =>
+  items.reduce((total, item) => total + (Number(item[field]) || 0), 0);
+
+const nearlyEqual = (left, right, epsilon = 0.01) =>
+  Math.abs((Number(left) || 0) - (Number(right) || 0)) <= epsilon;
+
+const isStableComparison = (result) =>
+  result.corpusId === "sentry-vulnerability-corpus" &&
+  result.targetMode === "all-corpus-files-by-sha" &&
+  !result.supersededBy &&
+  result.summary?.chunksFailed === 0 &&
+  result.summary?.chunksAnalyzed === result.summary?.chunksTotal &&
+  Boolean(result.timing?.analysisChunkMs);
+
+for (const filename of readdirSync(resultsDir).filter((file) => file.endsWith(".json"))) {
+  const result = JSON.parse(readFileSync(join(resultsDir, filename), "utf8"));
+  const label = `${filename} (${result.runId ?? "missing runId"})`;
+
+  if (result.scores) {
+    if (!result.scoring) {
+      errors.push(`${label}: has scores but no scoring summary`);
+    }
+
+    if (result.summary?.findingsTotal !== result.scores.length) {
+      errors.push(
+        `${label}: summary.findingsTotal=${result.summary?.findingsTotal} but scores.length=${result.scores.length}`,
+      );
+    }
+
+    const matchedCorpusIds = new Set();
+    for (const score of result.scores) {
+      for (const corpusId of score.matchedCorpusIds ?? []) {
+        if (!corpusIds.has(corpusId)) {
+          errors.push(`${label}: score ${score.findingId} references unknown corpus id ${corpusId}`);
+        }
+        matchedCorpusIds.add(corpusId);
+      }
+    }
+
+    if (result.scoring) {
+      if (matchedCorpusIds.size !== result.scoring.knownFound) {
+        errors.push(
+          `${label}: scoring.knownFound=${result.scoring.knownFound} but unique matched corpus ids=${matchedCorpusIds.size}`,
+        );
+      }
+
+      const expectedKnownMissed =
+        result.scoring.knownFindingCount - result.scoring.knownFound;
+      if (result.scoring.knownMissed !== expectedKnownMissed) {
+        errors.push(
+          `${label}: scoring.knownMissed=${result.scoring.knownMissed} but expected ${expectedKnownMissed}`,
+        );
+      }
+
+      const expectedRate = Number(
+        (result.scoring.knownFound / result.scoring.knownFindingCount).toFixed(4),
+      );
+      if (
+        result.scoring.knownFoundRate !== undefined &&
+        result.scoring.knownFoundRate !== expectedRate
+      ) {
+        errors.push(
+          `${label}: scoring.knownFoundRate=${result.scoring.knownFoundRate} but expected ${expectedRate}`,
+        );
+      }
+    }
+  }
+
+  if (result.shards?.length && result.summary && !result.supersededBy) {
+    const checks = [
+      ["chunksTotal", "chunksTotal"],
+      ["chunksAnalyzed", "chunksAnalyzed"],
+      ["chunksFailed", "chunksFailed"],
+      ["filesAnalyzed", "filesAnalyzed"],
+      ["findingsTotal", "findingsTotal"],
+      ["targetFileCount", "targetFileCount"],
+    ];
+
+    for (const [summaryField, shardField] of checks) {
+      const shardTotal = sum(result.shards, shardField);
+      if (result.summary[summaryField] !== shardTotal) {
+        errors.push(
+          `${label}: summary.${summaryField}=${result.summary[summaryField]} but shard total=${shardTotal}`,
+        );
+      }
+    }
+
+    const shardDuration = sum(result.shards, "durationMs");
+    if (!nearlyEqual(result.summary.durationMs, shardDuration, 1000)) {
+      errors.push(
+        `${label}: summary.durationMs=${result.summary.durationMs} but shard total=${shardDuration}`,
+      );
+    }
+
+    const shardCost = sum(result.shards, "costUSD");
+    if (!nearlyEqual(result.summary.costUSD, shardCost)) {
+      errors.push(
+        `${label}: summary.costUSD=${result.summary.costUSD} but shard total=${shardCost}`,
+      );
+    }
+  }
+
+  if (
+    result.traceCapture?.enabled &&
+    ["complete", "full"].includes(result.traceCapture.coverage)
+  ) {
+    if (!result.traceSummaries) {
+      errors.push(`${label}: traceCapture is ${result.traceCapture.coverage} but traceSummaries is missing`);
+    } else if (result.traceSummaries.length !== result.summary?.chunksTotal) {
+      errors.push(
+        `${label}: traceSummaries.length=${result.traceSummaries.length} but chunksTotal=${result.summary?.chunksTotal}`,
+      );
+    }
+
+    const failedTraces = (result.traceSummaries ?? []).filter(
+      (trace) => trace.status !== "success",
+    );
+    if (failedTraces.length > 0) {
+      errors.push(`${label}: ${failedTraces.length} trace summaries are not success`);
+    }
+  }
+
+  if (isStableComparison(result)) {
+    if (!result.scoring) {
+      errors.push(`${label}: stable comparison rows must be scored`);
+    }
+
+    if (result.timing.analysisChunkMs.count !== result.summary.chunksTotal) {
+      errors.push(
+        `${label}: timing.analysisChunkMs.count=${result.timing.analysisChunkMs.count} but chunksTotal=${result.summary.chunksTotal}`,
+      );
+    }
+  }
+}
+
+if (errors.length > 0) {
+  console.error("Benchmark result validation failed:");
+  for (const error of errors) {
+    console.error(`- ${error}`);
+  }
+  process.exit(1);
+}
+
+console.log("Benchmark result validation passed.");

packages/docs/src/components/BenchmarkRuns.astro

@@ -28,12 +28,16 @@ type BenchmarkResult = {
   summary: {
     analysisCostUSD: number;
     auxiliaryCostUSD: number;
+    chunksAnalyzed: number;
+    chunksFailed: number;
+    chunksTotal: number;
     costUSD: number;
     durationMs: number;
     findingsTotal: number;
     inputTokens?: number;
     outputTokens?: number;
   };
+  supersededBy?: string;
   targetMode: string;
   timing?: {
     analysisChunkMs?: TimingDistribution;
@@ -171,6 +175,9 @@ const comparisonResults = Object.values(resultModules)
     (result) =>
       result.corpusId === "sentry-vulnerability-corpus" &&
       result.targetMode === "all-corpus-files-by-sha" &&
+      !result.supersededBy &&
+      result.summary.chunksFailed === 0 &&
+      result.summary.chunksAnalyzed === result.summary.chunksTotal &&
       result.timing?.analysisChunkMs,
   )
   .sort(compareResults);
@@ -185,7 +192,7 @@ const comparisonResults = Object.values(resultModules)
         <span role="columnheader">Run</span>
         <span role="columnheader">Known</span>
         <span role="columnheader">Findings</span>
-        <span role="columnheader">Run Cost</span>
+        <span role="columnheader">Recorded Cost</span>
       </div>
       {comparisonResults.map((result) => {
         const scoring = result.scoring;
@@ -213,7 +220,7 @@ const comparisonResults = Object.values(resultModules)
               <strong class="metric-value">{result.summary.findingsTotal}</strong>
             </div>
             <div class="metric-cell" role="cell">
-              <span class="cell-label">Run cost</span>
+              <span class="cell-label">Recorded cost</span>
               <strong class="metric-value">{formatCost(result.summary.costUSD)}</strong>
             </div>
           </article>
@@ -230,7 +237,7 @@ const comparisonResults = Object.values(resultModules)
           <div class="run-table cost-table" role="table" aria-label="Cost breakdown">
             <div class="run-header" role="row">
               <span role="columnheader">Run</span>
-              <span role="columnheader">Run Cost</span>
+              <span role="columnheader">Recorded Cost</span>
               <span role="columnheader">Input Tokens</span>
               <span role="columnheader">Output Tokens</span>
             </div>
@@ -241,7 +248,7 @@ const comparisonResults = Object.values(resultModules)
                   {runMeta(result) && <p>{runMeta(result)}</p>}
                 </div>
                 <div class="metric-cell" role="cell">
-                  <span class="cell-label">Run cost</span>
+                  <span class="cell-label">Recorded cost</span>
                   <strong class="metric-value">{formatCost(result.summary.costUSD)}</strong>
                 </div>
                 <div class="metric-cell" role="cell">

packages/docs/src/content/docs/benchmarking.mdx

@@ -29,8 +29,8 @@ Sentry repository.
 The score table is the headline. The cost and timing tables below it are
 operational context for understanding why two runs with similar scores may look
 very different to operate. This matrix only shows stable comparison runs with
-per-chunk timing metadata; older incomplete runs remain in the result data but
-are hidden here.
+per-chunk timing metadata and no failed chunks; older incomplete or partial
+runs remain in the result data but are hidden here.
 
 <BenchmarkRuns />
 
@@ -59,24 +59,27 @@ Total findings is the amount of review output Warden produced before scoring.
 A higher number can be good if it finds more real vulnerabilities, but it also
 means more human review.
 
-Cost is the provider-reported cost persisted in the result metadata, not cost
-per finding. The cost table shows one run-cost column plus the persisted input
-and output token totals. Total cost can include Warden's post-analysis verifier
-and other auxiliary model calls, but those calls may use auxiliary or synthesis
-models rather than the model being benchmarked. Because of that, auxiliary cost
-is operational context, not a useful comparison dimension for this matrix. The
-raw JSONL logs are kept outside the docs until they have been reviewed for
-sensitive data. Current runs scan the same 156 analysis chunks; failed chunks
-are shown in the run table when present. When the raw artifacts preserve
-verifier usage, it is included under `auxiliaryUsage.verification`; some run
-shapes only persist the final total or per-chunk analysis usage.
+Recorded cost is the provider-reported cost persisted in the result metadata,
+not cost per finding. The cost table shows one recorded-cost column plus the
+persisted input and output token totals. Recorded cost can include Warden's
+post-analysis verifier and other auxiliary model calls, but those calls may use
+auxiliary or synthesis models rather than the model being benchmarked. Because
+of that, auxiliary cost is operational context, not a useful comparison
+dimension for this matrix. The raw JSONL logs are kept outside the docs until
+they have been reviewed for sensitive data. Current displayed runs scan the
+same 156 analysis chunks with zero failed chunks. Rows with failed chunks stay
+out of the stable matrix until they are rerun or explicitly recorded as partial.
+When the raw artifacts preserve verifier usage, it is included under
+`auxiliaryUsage.verification`; some run shapes only persist the final total or
+per-chunk analysis usage.
 
 Some run shapes persist only per-chunk analysis usage. The Opus 4.6 high-effort
-Pi run completed with no failed chunks, but its live CLI shard
-summaries showed approximately $38.43 total including auxiliary/post-processing
-work while the persisted JSONL artifacts contain $30.89 of scan cost. Until
-the artifact format preserves that auxiliary usage exactly, the table uses the
-persisted JSONL cost and the run note records the gap.
+Pi run completed with no failed chunks, but its live CLI shard summaries showed
+approximately $38.43 total including auxiliary/post-processing work while the
+persisted JSONL artifacts contain $30.89 of scan cost. Until the artifact
+format preserves that auxiliary usage exactly, the table uses the persisted
+JSONL cost and the run note records the gap. Treat recorded cost as operational
+accounting for a row, not normalized model pricing.
 
 Treat duration as an operational measurement, not a stable model quality
 metric. P50 and P90 come from per-analysis-chunk `durationMs` records in the
@@ -95,14 +98,30 @@ corpus and target files are identical.
 Pi runs without an explicit Warden `--effort` use Pi's default thinking level,
 which is currently medium.
 
+The Opus 4.7 and 4.8 Pi runs currently have an unusual shape. They complete
+cleanly, but many no-finding chunks are very short. In the traced Opus 4.8 Pi
+rerun, chunks with findings averaged 4.3 turns and 3.1k output tokens, while
+no-finding chunks averaged 1.8 turns and 824 output tokens. Eighty-one of 137
+no-finding chunks were one-turn scans, and 32 of 68 missed corpus entries were
+covered by one-turn no-finding scans. The Opus 4.7 run predates trace capture,
+but its low token use and short timings show a similar pattern.
+
+Treat this as a hypothesis, not a model diagnosis. Through Pi, the traces
+suggest many no-finding chunks terminate early. The miss pattern is consistent
+with under-exploration of cross-file authorization and data-boundary invariants,
+not a proven inability to reason about a found issue.
+
 ## Current Takeaway
 
-In the current agent-verified rows, GPT 5.5 on Pi with explicit high effort
-found 41 of 86. Sonnet 4.6 found 23 of 86 through the Claude SDK, and Opus 4.6
-on Pi with explicit high effort also found 23 of 86. Opus 4.8 found 16 of 86 on
-Pi at Pi's default level, 14 of 86 on Pi with explicit high effort,
-and 17 of 86 through the Claude SDK. Opus 4.7 on Pi found 6 of 86 at Pi's
-default level.
+In the current clean, agent-verified rows, GPT 5.5 on Pi found 41 of 86 with
+explicit high effort and 28 of 86 with explicit low effort. Sonnet 4.6 found 25
+of 86 on Pi at Pi's default level. Opus 4.6 on Pi with explicit high effort
+found 23 of 86. Opus 4.8 found 18 of 86 on Pi at Pi's default level, 14 of 86
+on Pi with explicit high effort, and 17 of 86 through the Claude SDK. Opus 4.7
+on Pi found 6 of 86 at Pi's default level.
+
+A Claude SDK Sonnet 4.6 run found 23 of 86, but one chunk failed, so it is kept
+in the result data and omitted from the stable matrix until rerun cleanly.
 
 Use those numbers as a relative comparison for this corpus. They are not a
 general pass rate for Sentry.