From fa5e335da88cd6e9f1b943ce1bf367a958b9dde1 Mon Sep 17 00:00:00 2001 From: David Baker Effendi Date: Fri, 4 Sep 2026 17:07:29 +0200 Subject: [PATCH 01/14] Separate overview charts by analyzer scope --- docs/src/components/LandingResults.astro | 6 +- docs/src/components/LatencyRanking.astro | 28 ++++- docs/src/components/SnapshotEvolution.astro | 107 +++++++++++--------- docs/src/data/snapshots.ts | 27 +++++ 4 files changed, 113 insertions(+), 55 deletions(-) diff --git a/docs/src/components/LandingResults.astro b/docs/src/components/LandingResults.astro index 808211a06..7804d0090 100644 --- a/docs/src/components/LandingResults.astro +++ b/docs/src/components/LandingResults.astro @@ -934,7 +934,8 @@ const snapshotPage = `/snapshots/${snapshot.slug}`; - + +

Modeling matrix — is the model surface load-bearing?

@@ -1117,7 +1118,8 @@ const snapshotPage = `/snapshots/${snapshot.slug}`; against the correctness sections above, not through them.

- + +

Kernels at a glance

diff --git a/docs/src/components/LatencyRanking.astro b/docs/src/components/LatencyRanking.astro index 7be7ae142..8f062e668 100644 --- a/docs/src/components/LatencyRanking.astro +++ b/docs/src/components/LatencyRanking.astro @@ -42,9 +42,11 @@ import { } from '../data/invocation-overhead'; import { currentSnapshot, + analyzerCohort, snapshotByVersion, vendorColorClass, vendorName, + type AnalyzerCohort, } from '../data/snapshots'; // Self-references to the latency page follow whichever snapshot is current, @@ -68,11 +70,14 @@ interface Props { * does not, and the chart may not appear there without them. */ stamp?: boolean; + /** Optional overview facet; detailed latency pages keep the full field. */ + cohort?: AnalyzerCohort; } const { kernelViews = true, stamp = false, + cohort, snapshotVersion = currentSnapshot.version, } = Astro.props; const snapshot = snapshotByVersion(snapshotVersion); @@ -83,7 +88,15 @@ const ranking = latencyRanking(model); // The axis is computed across *every* view, including the kernel views this // render may not draw, so the same duration sits at the same place on the // landing page and on the tier page. -const panelsToRender = kernelViews ? ranking.views : ranking.views.slice(0, 1); +const panelsToRender = (kernelViews ? ranking.views : ranking.views.slice(0, 1)).map( + (view) => ({ + ...view, + entries: + cohort === undefined + ? view.entries + : view.entries.filter((entry) => analyzerCohort(entry.tool) === cohort), + }), +); // ---- Geometry ------------------------------------------------------------- const width = 820; @@ -255,7 +268,7 @@ function layout(view: (typeof ranking.views)[number]) { // A suffix keeps element identifiers unique when both renders of this // component end up in one document, and keeps the radio group of one render // from capturing the labels of another. -const scope = kernelViews ? 'k' : 'g'; +const scope = kernelViews ? 'k' : cohort ?? 'g'; const panels = panelsToRender.map((view) => ({ view, ...layout(view), @@ -294,9 +307,10 @@ const toggleCss = panels .join('\n'); const general = ranking.views[0]!; -const fastest = general.entries[0]!; -const slowest = general.entries.at(-1)!; -const decomposedCount = general.entries.filter( +const displayedGeneral = panelsToRender[0]!; +const fastest = displayedGeneral.entries[0]!; +const slowest = displayedGeneral.entries.at(-1)!; +const decomposedCount = displayedGeneral.entries.filter( (entry) => entry.phases.length > 0, ).length; const environment = model.environments[0]; @@ -322,6 +336,10 @@ const belowThreshold = overheadPublished.filter( ); --- +{cohort && ( +

{cohort === 'generalist' ? 'Generalists' : 'Specialists'}

+)} +
{kernelViews && panels.map((panel, index) => ( diff --git a/docs/src/components/SnapshotEvolution.astro b/docs/src/components/SnapshotEvolution.astro index 02869e335..42f18f5f4 100644 --- a/docs/src/components/SnapshotEvolution.astro +++ b/docs/src/components/SnapshotEvolution.astro @@ -8,12 +8,27 @@ // kernels only, `core` tiers only, one series per analyzer, no combined score. import { snapshots, + analyzerCohort, coreKernelPopulations, vendorColorClass, vendorName, vendorOrder, + type AnalyzerCohort, } from '../data/snapshots'; +interface Props { + cohort: AnalyzerCohort; + heading?: boolean; +} + +const { cohort, heading = false } = Astro.props; +const cohortLabel = cohort === 'generalist' ? 'Generalists' : 'Specialists'; +const cohortDescription = + cohort === 'generalist' + ? 'analyzers spanning several language ecosystems' + : 'analyzers deliberately focused on one ecosystem or a small related family'; +const scope = cohort === 'generalist' ? 'generalists' : 'specialists'; + interface SeriesPoint { version: string; slug: string; @@ -123,6 +138,7 @@ const series: Series[] = [...byTool.entries()] vendorOrder(left.tool) - vendorOrder(right.tool) || left.name.localeCompare(right.name), ); +const cohortSeries = series.filter((entry) => analyzerCohort(entry.tool) === cohort); // ---- Geometry ------------------------------------------------------------- const height = 440; @@ -149,7 +165,7 @@ const bandMargin = 24; const preferredSpacing = 13; const band = Math.max( 104, - preferredSpacing * Math.max(series.length - 1, 1) + bandMargin, + preferredSpacing * Math.max(cohortSeries.length - 1, 1) + bandMargin, ); const plotWidth = band * columns.length; const width = padLeft + plotWidth + padRight; @@ -185,10 +201,10 @@ const y = (value: number) => padTop + plotHeight * (1 - value / axisMax); // tightens them, so no previously published layout moves. const fannedOffset = (index: number, preferred: number) => { const spacing = - series.length <= 1 + cohortSeries.length <= 1 ? 0 - : Math.min(preferred, (band - bandMargin) / (series.length - 1)); - return (index - (series.length - 1) / 2) * spacing; + : Math.min(preferred, (band - bandMargin) / (cohortSeries.length - 1)); + return (index - (cohortSeries.length - 1) / 2) * spacing; }; const seriesOffset = (index: number) => fannedOffset(index, preferredSpacing); @@ -225,7 +241,7 @@ const percentOffset = (index: number) => fannedOffset(index, 30); // When the fan is tighter than a `k/n` label is wide, those labels are dealt // out over three rows so neighbours cannot overprint each other. const percentSpacing = - series.length <= 1 ? 0 : Math.abs(percentOffset(1) - percentOffset(0)); + cohortSeries.length <= 1 ? 0 : Math.abs(percentOffset(1) - percentOffset(0)); const labelRows = percentSpacing >= 26 ? 1 : 3; const labelRow = (index: number) => index % labelRows; /** Marker area grows with the share of the snapshot's kernels covered. */ @@ -288,7 +304,7 @@ const evidenceHref = (slug: string) => `/snapshots/${slug}/evidence/`; const rows = columns.flatMap((column) => - series + cohortSeries .map((entry) => ({ column, entry, @@ -313,12 +329,12 @@ const sentenceList = (items: string[]) => ? items.join(' and ') : `${items.slice(0, -1).join(', ')}, and ${items.at(-1)}`; const debutList = sentenceList( - series + cohortSeries .filter((entry) => entry.points[0]!.version !== first.version) .map((entry) => `${entry.name} first appears in ${entry.points[0]!.version}`), ); /** Analyzers whose line is a single point: present in exactly one freeze. */ -const singlePoint = series.filter((entry) => entry.points.length === 1); +const singlePoint = cohortSeries.filter((entry) => entry.points.length === 1); const singlePointList = sentenceList(singlePoint.map((entry) => entry.name)); // The snapshot that expanded the population the most: the freeze where a @@ -333,10 +349,15 @@ const biggestJump = columns .sort((left, right) => right.growth - left.growth)[0]!; --- -

How the benchmark and its analyzers evolved

+{heading &&

How the benchmark and its analyzers evolved

} + +

{cohortLabel}

- One series per analyzer, across every published freeze. The vertical axis + One series per {cohort === 'generalist' ? 'generalist' : 'specialist'} analyzer, + across every published freeze. This panel contains {cohortDescription}; the + other cohort is drawn separately because breadth and specialization are not + directly comparable. The vertical axis counts decisive-correct assertions — true positives plus true negatives — on the benchmark-controlled kernel population, the same no-pooling population the cards at the top of this page read. The @@ -352,28 +373,28 @@ const biggestJump = columns

- - - + + +
@@ -431,7 +452,7 @@ const biggestJump = columns ))} - {series.map((entry, entryIndex) => { + {cohortSeries.map((entry, entryIndex) => { const offset = seriesOffset(entryIndex); return ( @@ -554,7 +575,7 @@ const biggestJump = columns y2={yPercent(0)} /> - {series.map((entry, entryIndex) => { + {cohortSeries.map((entry, entryIndex) => { const offset = percentOffset(entryIndex); return ( @@ -641,7 +662,7 @@ const biggestJump = columns
    - {series.map((entry) => ( + {cohortSeries.map((entry) => (
  • {entry.name} · decisive-correct @@ -658,7 +679,7 @@ const biggestJump = columns
    - {series.map((entry) => ( + {cohortSeries.map((entry) => (
  • {entry.name} · decisive-correct ÷ its covered population @@ -890,26 +911,16 @@ const biggestJump = columns .view-toggle label:hover { color: var(--sl-color-white); } - #evolution-view-absolute:checked - ~ .view-toggle - label[for='evolution-view-absolute'], - #evolution-view-normalized:checked - ~ .view-toggle - label[for='evolution-view-normalized'], - #evolution-view-table:checked ~ .view-toggle label[for='evolution-view-table'] { + .absolute-radio:checked ~ .view-toggle .absolute-label, + .normalized-radio:checked ~ .view-toggle .normalized-label, + .table-radio:checked ~ .view-toggle .table-label { background: var(--sl-color-gray-6, var(--sl-color-hairline)); color: var(--sl-color-white); box-shadow: inset 0 0 0 1px var(--sl-color-hairline); } - #evolution-view-absolute:focus-visible - ~ .view-toggle - label[for='evolution-view-absolute'], - #evolution-view-normalized:focus-visible - ~ .view-toggle - label[for='evolution-view-normalized'], - #evolution-view-table:focus-visible - ~ .view-toggle - label[for='evolution-view-table'] { + .absolute-radio:focus-visible ~ .view-toggle .absolute-label, + .normalized-radio:focus-visible ~ .view-toggle .normalized-label, + .table-radio:focus-visible ~ .view-toggle .table-label { outline: 2px solid var(--sl-color-accent-high); outline-offset: 2px; } @@ -919,12 +930,12 @@ const biggestJump = columns height. It stays in the DOM either way — only `display` moves — and the charts' own ``/`<desc>` state the essentials plus where the exact figures live, so nothing is only reachable through the table. */ - #evolution-view-absolute:checked ~ .view-normalized, - #evolution-view-absolute:checked ~ .view-table, - #evolution-view-normalized:checked ~ .view-absolute, - #evolution-view-normalized:checked ~ .view-table, - #evolution-view-table:checked ~ .view-absolute, - #evolution-view-table:checked ~ .view-normalized { + .absolute-radio:checked ~ .view-normalized, + .absolute-radio:checked ~ .view-table, + .normalized-radio:checked ~ .view-absolute, + .normalized-radio:checked ~ .view-table, + .table-radio:checked ~ .view-absolute, + .table-radio:checked ~ .view-normalized { display: none; } .chart-frame { diff --git a/docs/src/data/snapshots.ts b/docs/src/data/snapshots.ts index be1890b19..0d6b32013 100644 --- a/docs/src/data/snapshots.ts +++ b/docs/src/data/snapshots.ts @@ -346,6 +346,33 @@ export function vendorOrder(tool: string): number { return index === -1 ? Object.keys(vendorColorClasses).length : index; } +/** + * Comparison cohort for overview-level cross-analyzer figures. + * + * Generalists publish benchmark-controlled kernels across several language + * ecosystems; specialists deliberately concentrate on one ecosystem or a + * small related family. The distinction is presentation metadata, not a + * score, and keeps unlike product scopes out of the same aggregate panel. + */ +export type AnalyzerCohort = 'generalist' | 'specialist'; + +const analyzerCohorts: Readonly<Record<string, AnalyzerCohort>> = { + bifrost: 'generalist', + codeql: 'generalist', + joern: 'generalist', + semgrep: 'generalist', + opentaint: 'specialist', + infer: 'specialist', + flowdroid: 'specialist', + pysa: 'specialist', +}; + +export function analyzerCohort(tool: string): AnalyzerCohort { + // Unknown future adapters remain visible, but do not silently acquire the + // narrower specialist label without an explicit classification decision. + return analyzerCohorts[tool] ?? 'generalist'; +} + /** * One benchmark-controlled `core` population, carrying every analyzer whose * own core tier covers exactly the same case identifiers. From 9387963b92f92ae5b365415344d254f4f24cb117 Mon Sep 17 00:00:00 2001 From: David Baker Effendi <david@brokk.ai> Date: Fri, 4 Sep 2026 17:16:00 +0200 Subject: [PATCH 02/14] Condense evolution chart captions --- docs/src/components/SnapshotEvolution.astro | 155 ++++---------------- 1 file changed, 31 insertions(+), 124 deletions(-) diff --git a/docs/src/components/SnapshotEvolution.astro b/docs/src/components/SnapshotEvolution.astro index 42f18f5f4..a83f4adfb 100644 --- a/docs/src/components/SnapshotEvolution.astro +++ b/docs/src/components/SnapshotEvolution.astro @@ -26,7 +26,7 @@ const cohortLabel = cohort === 'generalist' ? 'Generalists' : 'Specialists'; const cohortDescription = cohort === 'generalist' ? 'analyzers spanning several language ecosystems' - : 'analyzers deliberately focused on one ecosystem or a small related family'; + : 'Analyzers deliberately focused on one ecosystem or a small related family'; const scope = cohort === 'generalist' ? 'generalists' : 'specialists'; interface SeriesPoint { @@ -319,34 +319,6 @@ const rows = columns.flatMap((column) => const first = columns[0]!; const last = columns.at(-1)!; -// Where each analyzer's line begins, derived rather than named, so an -// analyzer added in a later freeze cannot be silently dropped from the -// sentence that explains why a line starts where it does. -const sentenceList = (items: string[]) => - items.length <= 1 - ? (items[0] ?? '') - : items.length === 2 - ? items.join(' and ') - : `${items.slice(0, -1).join(', ')}, and ${items.at(-1)}`; -const debutList = sentenceList( - cohortSeries - .filter((entry) => entry.points[0]!.version !== first.version) - .map((entry) => `${entry.name} first appears in ${entry.points[0]!.version}`), -); -/** Analyzers whose line is a single point: present in exactly one freeze. */ -const singlePoint = cohortSeries.filter((entry) => entry.points.length === 1); -const singlePointList = sentenceList(singlePoint.map((entry) => entry.name)); - -// The snapshot that expanded the population the most: the freeze where a -// falling share is the benchmark moving, not an analyzer regressing. -const biggestJump = columns - .map((column, index) => ({ - column, - previous: columns[index - 1], - growth: column.population - (columns[index - 1]?.population ?? 0), - })) - .filter((entry) => entry.previous !== undefined) - .sort((left, right) => right.growth - left.growth)[0]!; --- {heading && <h2 id="evolution">How the benchmark and its analyzers evolved</h2>} @@ -354,21 +326,20 @@ const biggestJump = columns <h3>{cohortLabel}</h3> <p class="chart-note"> - One series per {cohort === 'generalist' ? 'generalist' : 'specialist'} analyzer, - across every published freeze. This panel contains {cohortDescription}; the - other cohort is drawn separately because breadth and specialization are not - directly comparable. The vertical axis - counts <strong>decisive-correct assertions</strong> — true positives plus - true negatives — on the <code>benchmark-controlled</code> kernel population, - the same no-pooling population the cards at the top of this page read. The - grey step is the benchmark itself: the total core population of that - snapshot, which grew from {first.population} assertions in - {first.version} to {last.population} in {last.version}. Nothing here is - pooled with the modeling matrix or the tool-native probes, and there is no - combined cross-analyzer score: read each line on its own. The second view - divides each analyzer's decisive-correct count by the population it actually - covers, so a narrow language footprint no longer reads as a weak result. The - third view drops the drawing entirely and lists every figure as a table. + {heading ? ( + <> + One line per analyzer across every freeze. The vertical axis counts + decisive-correct assertions; the grey step is the full benchmark core, + which grew from {first.population} assertions in {first.version} to{' '} + {last.population} in {last.version}. Nothing is pooled with other result + profiles or turned into a combined score. + </> + ) : ( + <> + {cohortDescription}, separated because breadth and specialization are not + directly comparable. Uses the same axes as the generalist panel. + </> + )} </p> <div class="evolution"> @@ -692,68 +663,21 @@ const biggestJump = columns </li> </ul> -{/* The view-note caption and whichever view caption is showing are adjacent - and were each carrying their own two columns, so the right column held - the tail of one and then the tail of the other. One wrapper flows both - down the left column first; the hidden view captions cost it nothing. */} -<div class="prose-columns"> - <p class="caption view-note"> - <strong>Absolute counts</strong> plot progress against the whole growing - benchmark: the marker is the analyzer's decisive-correct count and the grey - step is the entire core population of that snapshot, so an analyzer that - supports only part of the benchmark sits far below the step by construction. - <strong>Accuracy on covered kernels</strong> instead divides each analyzer's - decisive-correct count by <em>its own</em> covered population — the assertions - in the kernels it reported on at all — so it answers how well a tool does on - the slice it supports. Coverage is stated beside every number in that view: - the marker grows with the share of the snapshot's kernels covered, each point - carries its <code>k/n</code> kernel count, and the <strong>Data table</strong> - view spells out the denominator, so 78% over 6 kernels is never mistaken for - 78% over 13. Inconclusive and unsupported outcomes <em>inside</em> covered - kernels stay in the denominator — they are non-answers on cases the analyzer - took on. The narrower correct ÷ decided ratio is reported only as a secondary - figure, in the tooltips and the last column of the data table. - <strong>Data table</strong> is the third setting of the same toggle: it - carries the exact figures behind both charts — every count, every denominator - and every percentage — for anyone who would rather read the numbers than the - shape. - </p> - - <p class="caption view-absolute"> - Each marker sits at the analyzer's decisive-correct count. The faint stub - above it reaches that analyzer's <em>own</em> covered population — the - assertions it reported on — so the part of the stub above the marker is - wrong answers plus coverage outcomes, and the remaining gap up to the grey - step is kernels it does not cover at all. A snapshot without a marker is a - snapshot the analyzer was not run in: {debutList}. Absence is never drawn as - a zero and never interpolated across, so an analyzer that appears in exactly - one freeze is one marker and no line — - {singlePoint.length > 0 - ? `${singlePointList} ${ - singlePoint.length === 1 ? 'is' : 'are' - } drawn that way here, and no earlier value is implied for ${ - singlePoint.length === 1 ? 'it' : 'them' - }` - : 'no analyzer is drawn that way here'}. - <code>inconclusive</code>, <code>unsupported</code> and - <code>runner-error</code> are capability coverage: excluded from - correctness, never converted into wrong answers. Every snapshot label links - to that freeze's per-case evidence. - </p> - - <p class="caption view-normalized"> - Each marker sits at the analyzer's decisive-correct share of its own covered - population; the faint stub above it runs to 100%, which is that same covered - population, so the stub is wrong answers plus coverage outcomes inside the - kernels it did report on. A snapshot without a marker is a snapshot the - analyzer was not run in: {debutList}. - Absence is never drawn as a zero and never interpolated across. Percentages - from different snapshots are not the same exam, and percentages from - different analyzers in the same snapshot are only the same exam when their - <code>k/n</code> kernel counts match. Every snapshot label links to that - freeze's per-case evidence. - </p> -</div> +<p class="caption chart-key"> + {heading ? ( + <> + Marker: decisive-correct. Cap or stub: that analyzer's covered population. + Grey step: the full core population. The covered-kernel view keeps + non-answers in its denominator and shows coverage as <code>k/n</code>; + missing runs are absent, never zero. Exact figures are in the table. + </> + ) : ( + <> + Same scales and encoding as the generalists. The covered-kernel view keeps + specialist scope visible as <code>k/n</code>; exact figures are in the table. + </> + )} +</p> <div class="table-scroll view-table"> <table> @@ -821,27 +745,10 @@ const biggestJump = columns </div> <p class="caption view-table"> - Every value the two chart views draw, one row per analyzer per snapshot, with - each denominator spelled out beside the figure it belongs to. This is the - same data the charts plot — nothing here is computed differently, and nothing - is rounded away except the percentages, which are given to one decimal place. - Switch back to <strong>Absolute counts</strong> or <strong>Accuracy on - covered kernels</strong> for the shape of it. + The exact values behind both chart views, one row per analyzer and snapshot. </p> </div> -<p class="chart-note"> - Read the shape, not a ranking. Two different things move a line: an analyzer - getting better, and the benchmark getting harder. The largest expansion so - far is <strong>{biggestJump.column.version}</strong>, which took the core - population from {biggestJump.previous!.population} to - {biggestJump.column.population} assertions — every analyzer's share of the - population fell there without any of them changing. Because the denominator - is drawn alongside the counts, saturation and expansion stay - distinguishable, and no two snapshots' fractions are ever compared as if - they were the same exam. -</p> - <style> /* The view toggle is a plain radio group: no JavaScript, keyboard-operable by default (Tab into the group, arrows between options), and it degrades From dcbea63299e7ea1114d4905c34ee2c27f67f6a90 Mon Sep 17 00:00:00 2001 From: David Baker Effendi <david@brokk.ai> Date: Fri, 4 Sep 2026 17:18:41 +0200 Subject: [PATCH 03/14] Hide empty specialist snapshot columns --- docs/src/components/SnapshotEvolution.astro | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/src/components/SnapshotEvolution.astro b/docs/src/components/SnapshotEvolution.astro index a83f4adfb..84334d729 100644 --- a/docs/src/components/SnapshotEvolution.astro +++ b/docs/src/components/SnapshotEvolution.astro @@ -65,13 +65,13 @@ interface SnapshotColumn { // Oldest first: the registry is newest-first for the sidebar's sake. const timeline = [...snapshots].reverse(); -const columns: SnapshotColumn[] = []; +const allColumns: SnapshotColumn[] = []; const byTool = new Map<string, SeriesPoint[]>(); for (const snapshot of timeline) { const populations = coreKernelPopulations(snapshot.results); const population = populations.reduce((sum, entry) => sum + entry.cases, 0); - columns.push({ + allColumns.push({ version: snapshot.version, slug: snapshot.slug, population, @@ -139,6 +139,14 @@ const series: Series[] = [...byTool.entries()] left.name.localeCompare(right.name), ); const cohortSeries = series.filter((entry) => analyzerCohort(entry.tool) === cohort); +// Do not draw empty historical columns in a cohort facet. Specialists first +// entered the benchmark in v0.6.0, so earlier freezes carry neither a point +// nor a meaningful cohort comparison. Their full benchmark populations still +// contribute to the shared y-axis below, keeping the two facets comparable. +const cohortVersions = new Set( + cohortSeries.flatMap((entry) => entry.points.map((point) => point.version)), +); +const columns = allColumns.filter((column) => cohortVersions.has(column.version)); // ---- Geometry ------------------------------------------------------------- const height = 440; @@ -179,7 +187,7 @@ const plotHeight = height - padTop - padBottom; */ const minWidthRem = Math.round((columns.length * 6 + 5) * 10) / 10; -const ceiling = Math.max(...columns.map((column) => column.population)); +const ceiling = Math.max(...allColumns.map((column) => column.population)); const step = 200; const axisMax = Math.max(step, Math.ceil(ceiling / step) * step); const ticks: number[] = []; From e5d4cbf9fdac7bb3408d30e6402c24868aaf30a6 Mon Sep 17 00:00:00 2001 From: David Baker Effendi <david@brokk.ai> Date: Fri, 4 Sep 2026 17:22:07 +0200 Subject: [PATCH 04/14] Use covered-kernel view for specialists --- docs/src/components/SnapshotEvolution.astro | 28 ++++++++++++++------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/docs/src/components/SnapshotEvolution.astro b/docs/src/components/SnapshotEvolution.astro index 84334d729..399ef3f22 100644 --- a/docs/src/components/SnapshotEvolution.astro +++ b/docs/src/components/SnapshotEvolution.astro @@ -345,24 +345,28 @@ const last = columns.at(-1)!; ) : ( <> {cohortDescription}, separated because breadth and specialization are not - directly comparable. Uses the same axes as the generalist panel. + directly comparable. Uses the same percentage axis as the generalists' + covered-kernel view. </> )} </p> <div class="evolution"> - <input - class="view-radio absolute-radio" - type="radio" - name={`evolution-view-${scope}`} - id={`evolution-view-absolute-${scope}`} - checked - /> + {heading && ( + <input + class="view-radio absolute-radio" + type="radio" + name={`evolution-view-${scope}`} + id={`evolution-view-absolute-${scope}`} + checked + /> + )} <input class="view-radio normalized-radio" type="radio" name={`evolution-view-${scope}`} id={`evolution-view-normalized-${scope}`} + checked={!heading} /> <input class="view-radio table-radio" @@ -371,11 +375,14 @@ const last = columns.at(-1)!; id={`evolution-view-table-${scope}`} /> <div class="view-toggle" role="group" aria-label="Evolution chart view"> - <label class="absolute-label" for={`evolution-view-absolute-${scope}`}>Absolute counts</label> + {heading && ( + <label class="absolute-label" for={`evolution-view-absolute-${scope}`}>Absolute counts</label> + )} <label class="normalized-label" for={`evolution-view-normalized-${scope}`}>Accuracy on covered kernels</label> <label class="table-label" for={`evolution-view-table-${scope}`}>Data table</label> </div> +{heading && ( <div class="chart-frame view-absolute" style={`--evolution-min-width: ${minWidthRem}rem`}> <svg class="evolution-chart" @@ -503,6 +510,7 @@ const last = columns.at(-1)!; </text> </svg> </div> +)} <div class="chart-frame view-normalized" style={`--evolution-min-width: ${minWidthRem}rem`}> <svg @@ -640,6 +648,7 @@ const last = columns.at(-1)!; </svg> </div> +{heading && ( <ul class="chart-legend view-absolute"> {cohortSeries.map((entry) => ( <li class:list={[entry.colorClass]}> @@ -656,6 +665,7 @@ const last = columns.at(-1)!; remainder of that analyzer's own covered population </li> </ul> +)} <ul class="chart-legend view-normalized"> {cohortSeries.map((entry) => ( From 38b16f993afbf5f6a729fdf4e58c80b952b38022 Mon Sep 17 00:00:00 2001 From: David Baker Effendi <david@brokk.ai> Date: Mon, 7 Sep 2026 09:18:22 +0200 Subject: [PATCH 05/14] Center overview on accuracy and coverage --- docs/src/components/AnalyzerLandscape.astro | 176 ++++++++++++++++++++ docs/src/components/LandingResults.astro | 12 +- 2 files changed, 185 insertions(+), 3 deletions(-) create mode 100644 docs/src/components/AnalyzerLandscape.astro diff --git a/docs/src/components/AnalyzerLandscape.astro b/docs/src/components/AnalyzerLandscape.astro new file mode 100644 index 000000000..aa666af90 --- /dev/null +++ b/docs/src/components/AnalyzerLandscape.astro @@ -0,0 +1,176 @@ +--- +// Current-snapshot tradeoff view. Unlike the historical evolution charts, +// this deliberately puts every analyzer in one field: breadth is one axis, +// correctness on the populations actually covered is the other. The two +// dimensions remain separate; no composite score or ranking is manufactured. +import { + analyzerCohort, + coreKernelPopulations, + currentSnapshot, + vendorColorClass, + vendorName, + vendorOrder, +} from '../data/snapshots'; + +interface AnalyzerPoint { + tool: string; + name: string; + colorClass: string; + cohort: 'generalist' | 'specialist'; + kernels: number; + covered: number; + correct: number; + wrong: number; + incomplete: number; +} + +const populations = coreKernelPopulations(currentSnapshot.results); +const totals = new Map<string, AnalyzerPoint>(); +for (const population of populations) { + for (const [tool, entry] of population.entries) { + const point = totals.get(tool) ?? { + tool, + name: vendorName(tool), + colorClass: vendorColorClass(tool), + cohort: analyzerCohort(tool), + kernels: 0, + covered: 0, + correct: 0, + wrong: 0, + incomplete: 0, + }; + point.kernels += 1; + point.covered += population.cases; + for (const result of entry.tier.cases) { + if (result.classification === 'true-positive' || result.classification === 'true-negative') { + point.correct += 1; + } else if (result.classification === 'false-positive' || result.classification === 'false-negative') { + point.wrong += 1; + } else { + point.incomplete += 1; + } + } + totals.set(tool, point); + } +} + +const points = [...totals.values()].sort( + (left, right) => vendorOrder(left.tool) - vendorOrder(right.tool) || left.name.localeCompare(right.name), +); +const kernelCount = populations.length; +const percent = (part: number, whole: number) => whole === 0 ? 0 : (100 * part) / whole; +const formatPercent = (value: number) => `${value.toFixed(1)}%`; + +const width = 820; +const height = 470; +const padLeft = 66; +const padRight = 30; +const padTop = 34; +const padBottom = 62; +const plotWidth = width - padLeft - padRight; +const plotHeight = height - padTop - padBottom; +const x = (kernels: number) => padLeft + percent(kernels, kernelCount) * plotWidth / 100; +const y = (correct: number, covered: number) => padTop + (100 - percent(correct, covered)) * plotHeight / 100; +const xTicks = [0, 25, 50, 75, 100]; +const yTicks = [50, 60, 70, 80, 90, 100]; +// Small, stable offsets keep labels readable without changing point geometry. +const labelOffsets: Record<string, { x: number; y: number; anchor?: 'start' | 'end' }> = { + bifrost: { x: -9, y: -11, anchor: 'end' }, + codeql: { x: -9, y: 17, anchor: 'end' }, + joern: { x: -9, y: -11, anchor: 'end' }, + semgrep: { x: -9, y: 17, anchor: 'end' }, + opentaint: { x: -10, y: -11, anchor: 'end' }, + infer: { x: 9, y: -11 }, + flowdroid: { x: 9, y: 17 }, + pysa: { x: 9, y: 17 }, +}; +--- + +<h2 id="landscape">Accuracy and language coverage — current snapshot</h2> + +<p> + Every analyzer is shown in the same field. Farther right means coverage of + more of the benchmark's {kernelCount} language kernels; higher means more + correct assertions within the kernels the analyzer covers. A specialist can + therefore show its accuracy without hiding the cost of its narrower language + support. These are two independent dimensions, not a combined score. +</p> + +<div class="landscape-frame"> + <svg class="landscape-chart" viewBox={`0 0 ${width} ${height}`} role="img" aria-labelledby="landscape-title landscape-desc"> + <title id="landscape-title">Current analyzer accuracy by language-kernel coverage + Scatter plot of all analyzers in DataFlowBench {currentSnapshot.version}. The horizontal axis is the share of language kernels covered. The vertical axis is decisive-correct assertions divided by every assertion in those covered kernels, including non-answers in the denominator. + {yTicks.map((tick) => { + const tickY = padTop + (100 - tick) * plotHeight / 100; + return <> + + {tick}% + ; + })} + {xTicks.map((tick) => { + const tickX = padLeft + tick * plotWidth / 100; + return <> + + {tick}% + ; + })} + + + language-kernel coverage + accuracy within covered kernels + {points.map((point) => { + const pointX = x(point.kernels); + const pointY = y(point.correct, point.covered); + const offset = labelOffsets[point.tool] ?? { x: 9, y: -9 }; + const accuracy = percent(point.correct, point.covered); + return + {point.name}: {formatPercent(accuracy)} accuracy across {point.kernels}/{kernelCount} language kernels; {point.correct} correct, {point.wrong} wrong, {point.incomplete} incomplete of {point.covered} + + {point.name} + ; + })} + +
+ +
    +
  • generalist
    • +
  • specialist
    • +
+ +
+ Exact figures +
+ + + {points.map((point) => + + + + + + )} +
AnalyzerScopeKernel coverageAccuracyCorrectWrongIncomplete
{point.name}{point.cohort}{point.kernels}/{kernelCount} ({formatPercent(percent(point.kernels, kernelCount))}){point.correct}/{point.covered} ({formatPercent(percent(point.correct, point.covered))}){point.correct}{point.wrong}{point.incomplete}
+
+
+ + diff --git a/docs/src/components/LandingResults.astro b/docs/src/components/LandingResults.astro index 7804d0090..b1cec20ef 100644 --- a/docs/src/components/LandingResults.astro +++ b/docs/src/components/LandingResults.astro @@ -16,6 +16,7 @@ import { type ScoreTier, } from '../data/snapshots'; import SnapshotEvolution from './SnapshotEvolution.astro'; +import AnalyzerLandscape from './AnalyzerLandscape.astro'; import LatencyRanking from './LatencyRanking.astro'; const snapshot = currentSnapshot; @@ -501,7 +502,7 @@ const snapshotPage = `/snapshots/${snapshot.slug}`;
- - +

Modeling matrix — is the model surface load-bearing?

@@ -1121,6 +1121,12 @@ const snapshotPage = `/snapshots/${snapshot.slug}`; +
+ Progress over time + + +
+

Kernels at a glance

From b5e17ea4eeac2c9d4aeaf56c40fed1b4f65aecfc Mon Sep 17 00:00:00 2001 From: David Baker Effendi Date: Mon, 7 Sep 2026 09:27:51 +0200 Subject: [PATCH 06/14] Use supported languages for landscape breadth --- docs/src/components/AnalyzerLandscape.astro | 35 +++++++----- docs/src/data/analyzer-language-support.ts | 63 +++++++++++++++++++++ 2 files changed, 85 insertions(+), 13 deletions(-) create mode 100644 docs/src/data/analyzer-language-support.ts diff --git a/docs/src/components/AnalyzerLandscape.astro b/docs/src/components/AnalyzerLandscape.astro index aa666af90..ea19d5d67 100644 --- a/docs/src/components/AnalyzerLandscape.astro +++ b/docs/src/components/AnalyzerLandscape.astro @@ -11,12 +11,14 @@ import { vendorName, vendorOrder, } from '../data/snapshots'; +import { analyzerLanguageSupport, type AnalyzerLanguageSupport } from '../data/analyzer-language-support'; interface AnalyzerPoint { tool: string; name: string; colorClass: string; cohort: 'generalist' | 'specialist'; + languageSupport: AnalyzerLanguageSupport; kernels: number; covered: number; correct: number; @@ -33,6 +35,7 @@ for (const population of populations) { name: vendorName(tool), colorClass: vendorColorClass(tool), cohort: analyzerCohort(tool), + languageSupport: analyzerLanguageSupport(tool), kernels: 0, covered: 0, correct: 0, @@ -69,9 +72,10 @@ const padTop = 34; const padBottom = 62; const plotWidth = width - padLeft - padRight; const plotHeight = height - padTop - padBottom; -const x = (kernels: number) => padLeft + percent(kernels, kernelCount) * plotWidth / 100; +const languageAxisMax = 14; +const x = (languages: number) => padLeft + languages * plotWidth / languageAxisMax; const y = (correct: number, covered: number) => padTop + (100 - percent(correct, covered)) * plotHeight / 100; -const xTicks = [0, 25, 50, 75, 100]; +const xTicks = [0, 2, 4, 6, 8, 10, 12, 14]; const yTicks = [50, 60, 70, 80, 90, 100]; // Small, stable offsets keep labels readable without changing point geometry. const labelOffsets: Record = { @@ -89,17 +93,20 @@ const labelOffsets: RecordAccuracy and language coverage — current snapshot

- Every analyzer is shown in the same field. Farther right means coverage of - more of the benchmark's {kernelCount} language kernels; higher means more + Every analyzer is shown in the same field. Farther right means documented + data-flow support for more languages; higher means more correct assertions within the kernels the analyzer covers. A specialist can therefore show its accuracy without hiding the cost of its narrower language - support. These are two independent dimensions, not a combined score. + support. The horizontal axis is an analyzer capability, not a count of the + adapters DataFlowBench happens to implement. Benchmark kernel participation + remains visible in the exact figures below. These are two independent + dimensions, not a combined score.

- Current analyzer accuracy by language-kernel coverage - Scatter plot of all analyzers in DataFlowBench {currentSnapshot.version}. The horizontal axis is the share of language kernels covered. The vertical axis is decisive-correct assertions divided by every assertion in those covered kernels, including non-answers in the denominator. + Current analyzer accuracy by supported data-flow languages + Scatter plot of all analyzers in DataFlowBench {currentSnapshot.version}. The horizontal axis is the documented number of languages in which each analyzer performs data-flow analysis. The vertical axis is decisive-correct assertions divided by every assertion in the benchmark kernels that analyzer covers, including non-answers in the denominator. {yTicks.map((tick) => { const tickY = padTop + (100 - tick) * plotHeight / 100; return <> @@ -108,23 +115,23 @@ const labelOffsets: Record; })} {xTicks.map((tick) => { - const tickX = padLeft + tick * plotWidth / 100; + const tickX = x(tick); return <> - {tick}% + {tick} ; })} - language-kernel coverage + languages with documented data-flow support accuracy within covered kernels {points.map((point) => { - const pointX = x(point.kernels); + const pointX = x(point.languageSupport.languages.length); const pointY = y(point.correct, point.covered); const offset = labelOffsets[point.tool] ?? { x: 9, y: -9 }; const accuracy = percent(point.correct, point.covered); return - {point.name}: {formatPercent(accuracy)} accuracy across {point.kernels}/{kernelCount} language kernels; {point.correct} correct, {point.wrong} wrong, {point.incomplete} incomplete of {point.covered} + {point.name}: {point.languageSupport.languages.length} supported languages; {formatPercent(accuracy)} accuracy across {point.kernels}/{kernelCount} benchmark kernels; {point.correct} correct, {point.wrong} wrong, {point.incomplete} incomplete of {point.covered} {point.name} ; @@ -141,10 +148,11 @@ const labelOffsets: RecordExact figures
- + {points.map((point) => + @@ -170,6 +178,7 @@ const labelOffsets: Record = { + bifrost: { + languages: ['C', 'C++', 'C#', 'Go', 'Java', 'JavaScript', 'Kotlin', 'PHP', 'Python', 'Ruby', 'Rust', 'Scala', 'TypeScript'], + source: 'pinned Bifrost adapter inventory', + sourceUrl: 'https://github.com/BrokkAi/dataflowbench/blob/main/adapters/bifrost/README.md', + }, + codeql: { + languages: ['C', 'C++', 'C#', 'Go', 'Java', 'JavaScript', 'Kotlin', 'Python', 'Ruby', 'Rust', 'Swift', 'TypeScript'], + source: 'CodeQL data-flow guides and supported-language inventory', + sourceUrl: 'https://codeql.github.com/docs/codeql-overview/supported-languages-and-frameworks/', + }, + joern: { + languages: ['C', 'C++', 'C#', 'Go', 'Java', 'JavaScript', 'Kotlin', 'PHP', 'Python', 'Ruby', 'Swift'], + source: 'Joern source front ends and data-flow engine documentation', + sourceUrl: 'https://docs.joern.io/frontends/', + }, + semgrep: { + languages: ['C', 'C++', 'Go', 'Java', 'JavaScript', 'Kotlin', 'PHP', 'Python', 'Ruby', 'Rust', 'TypeScript'], + source: 'pinned Semgrep CE taint-language inventory', + sourceUrl: 'https://github.com/BrokkAi/dataflowbench/blob/main/adapters/semgrep/README.md#front-end-maturity', + }, + infer: { + languages: ['C', 'C++', 'Java'], + source: 'verified Pulse taint-language inventory', + sourceUrl: 'https://github.com/BrokkAi/dataflowbench/blob/main/adapters/infer/README.md', + }, + opentaint: { + languages: ['Java', 'Kotlin'], + source: 'verified OpenTaint analyzer inventory', + sourceUrl: 'https://github.com/BrokkAi/dataflowbench/blob/main/adapters/opentaint/README.md', + }, + flowdroid: { + languages: ['Java', 'Kotlin'], + source: 'verified FlowDroid analyzer inventory', + sourceUrl: 'https://github.com/BrokkAi/dataflowbench/blob/main/adapters/flowdroid/README.md', + }, + pysa: { + languages: ['Python'], + source: 'Pysa Python taint-analysis documentation', + sourceUrl: 'https://pyre-check.org/docs/pysa-basics/', + }, +}; + +export function analyzerLanguageSupport(tool: string): AnalyzerLanguageSupport { + const support = supportByTool[tool]; + if (!support) throw new Error(`No data-flow language inventory for ${tool}`); + return support; +} From 505817dd909b4ace937be0fd3c39c2ac118d7c7b Mon Sep 17 00:00:00 2001 From: David Baker Effendi Date: Mon, 7 Sep 2026 09:33:08 +0200 Subject: [PATCH 07/14] Combine overview latency charts --- docs/src/components/LandingResults.astro | 3 +-- docs/src/components/LatencyRanking.astro | 21 ++------------------- 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/docs/src/components/LandingResults.astro b/docs/src/components/LandingResults.astro index b1cec20ef..6c36c3bdc 100644 --- a/docs/src/components/LandingResults.astro +++ b/docs/src/components/LandingResults.astro @@ -1118,8 +1118,7 @@ const snapshotPage = `/snapshots/${snapshot.slug}`; against the correctness sections above, not through them.

- - +
Progress over time diff --git a/docs/src/components/LatencyRanking.astro b/docs/src/components/LatencyRanking.astro index 8f062e668..e93fe6ca8 100644 --- a/docs/src/components/LatencyRanking.astro +++ b/docs/src/components/LatencyRanking.astro @@ -42,11 +42,9 @@ import { } from '../data/invocation-overhead'; import { currentSnapshot, - analyzerCohort, snapshotByVersion, vendorColorClass, vendorName, - type AnalyzerCohort, } from '../data/snapshots'; // Self-references to the latency page follow whichever snapshot is current, @@ -70,14 +68,11 @@ interface Props { * does not, and the chart may not appear there without them. */ stamp?: boolean; - /** Optional overview facet; detailed latency pages keep the full field. */ - cohort?: AnalyzerCohort; } const { kernelViews = true, stamp = false, - cohort, snapshotVersion = currentSnapshot.version, } = Astro.props; const snapshot = snapshotByVersion(snapshotVersion); @@ -88,15 +83,7 @@ const ranking = latencyRanking(model); // The axis is computed across *every* view, including the kernel views this // render may not draw, so the same duration sits at the same place on the // landing page and on the tier page. -const panelsToRender = (kernelViews ? ranking.views : ranking.views.slice(0, 1)).map( - (view) => ({ - ...view, - entries: - cohort === undefined - ? view.entries - : view.entries.filter((entry) => analyzerCohort(entry.tool) === cohort), - }), -); +const panelsToRender = kernelViews ? ranking.views : ranking.views.slice(0, 1); // ---- Geometry ------------------------------------------------------------- const width = 820; @@ -268,7 +255,7 @@ function layout(view: (typeof ranking.views)[number]) { // A suffix keeps element identifiers unique when both renders of this // component end up in one document, and keeps the radio group of one render // from capturing the labels of another. -const scope = kernelViews ? 'k' : cohort ?? 'g'; +const scope = kernelViews ? 'k' : 'all'; const panels = panelsToRender.map((view) => ({ view, ...layout(view), @@ -336,10 +323,6 @@ const belowThreshold = overheadPublished.filter( ); --- -{cohort && ( -

{cohort === 'generalist' ? 'Generalists' : 'Specialists'}

-)} -
{kernelViews && panels.map((panel, index) => ( From 63a997ff9fe19a7f411f4cbcf875224cf303a7c0 Mon Sep 17 00:00:00 2001 From: David Baker Effendi Date: Mon, 7 Sep 2026 10:18:36 +0200 Subject: [PATCH 08/14] Replace kernel cards with decision landscape --- docs/src/components/DecisionLandscape.astro | 196 ++++++++++++++++++++ docs/src/components/LandingResults.astro | 72 +------ 2 files changed, 198 insertions(+), 70 deletions(-) create mode 100644 docs/src/components/DecisionLandscape.astro diff --git a/docs/src/components/DecisionLandscape.astro b/docs/src/components/DecisionLandscape.astro new file mode 100644 index 000000000..023130a14 --- /dev/null +++ b/docs/src/components/DecisionLandscape.astro @@ -0,0 +1,196 @@ +--- +// One shared view of the two quantities the old kernel cards put into prose: +// how often an analyzer decides, and how often those decisions are correct. +// The overall view uses the entire kernel corpus as every analyzer's X-axis +// denominator, so narrow language coverage remains visible rather than being +// erased by aggregation. Language views use that kernel's own population. +import { + coreKernelPopulations, + currentSnapshot, + vendorColorClass, + vendorName, + vendorOrder, +} from '../data/snapshots'; + +interface Point { + tool: string; + name: string; + colorClass: string; + correct: number; + wrong: number; + incomplete: number; +} + +interface Panel { + id: string; + label: string; + denominator: number; + kernelId: string | null; + points: Point[]; +} + +const populations = coreKernelPopulations(currentSnapshot.results); +const totalAssertions = populations.reduce((sum, population) => sum + population.cases, 0); + +function pointOf(tool: string, cases: { classification: string }[]): Point { + let correct = 0; + let wrong = 0; + let incomplete = 0; + for (const result of cases) { + if (result.classification === 'true-positive' || result.classification === 'true-negative') correct += 1; + else if (result.classification === 'false-positive' || result.classification === 'false-negative') wrong += 1; + else incomplete += 1; + } + return { tool, name: vendorName(tool), colorClass: vendorColorClass(tool), correct, wrong, incomplete }; +} + +const overall = new Map(); +for (const population of populations) { + for (const [tool, entry] of population.entries) { + const slice = pointOf(tool, entry.tier.cases); + const point = overall.get(tool) ?? { ...slice, correct: 0, wrong: 0, incomplete: 0 }; + point.correct += slice.correct; + point.wrong += slice.wrong; + point.incomplete += slice.incomplete; + overall.set(tool, point); + } +} + +const sortPoints = (points: Point[]) => points.sort( + (left, right) => vendorOrder(left.tool) - vendorOrder(right.tool) || left.name.localeCompare(right.name), +); +const panels: Panel[] = [ + { id: 'overall', label: 'Overall', denominator: totalAssertions, kernelId: null, points: sortPoints([...overall.values()]) }, + ...populations.map((population, index) => ({ + id: population.language, + label: population.language, + denominator: population.cases, + kernelId: `kernel-${index}`, + points: sortPoints([...population.entries].map(([tool, entry]) => pointOf(tool, entry.tier.cases))), + })), +]; + +const width = 820; +const height = 470; +const padLeft = 66; +const padRight = 30; +const padTop = 34; +const padBottom = 62; +const plotWidth = width - padLeft - padRight; +const plotHeight = height - padTop - padBottom; +const percent = (part: number, whole: number) => whole === 0 ? 0 : (100 * part) / whole; +const formatPercent = (value: number) => `${value.toFixed(1)}%`; +const x = (decided: number, denominator: number) => padLeft + percent(decided, denominator) * plotWidth / 100; +const y = (correct: number, decided: number) => padTop + (100 - percent(correct, decided)) * plotHeight / 100; +const xTicks = [0, 25, 50, 75, 100]; +const yTicks = [50, 60, 70, 80, 90, 100]; +const labelOffsets: Record = { + bifrost: { x: -9, y: -11, anchor: 'end' }, + codeql: { x: -9, y: 17, anchor: 'end' }, + joern: { x: -9, y: -11, anchor: 'end' }, + semgrep: { x: 9, y: 17 }, + opentaint: { x: -10, y: -11, anchor: 'end' }, + infer: { x: 9, y: -11 }, + flowdroid: { x: 9, y: 17 }, + pysa: { x: -9, y: 17, anchor: 'end' }, +}; + +const toggleCss = panels.map((panel) => ` +#decision-view-${panel.id}:checked ~ .decision-ribbon label[for='decision-view-${panel.id}'] { + background: var(--sl-color-accent-low); color: var(--sl-color-white); + box-shadow: inset 0 0 0 1px var(--sl-color-accent); +} +#decision-view-${panel.id}:focus-visible ~ .decision-ribbon label[for='decision-view-${panel.id}'] { + outline: 2px solid var(--sl-color-accent-high); outline-offset: 2px; +} +#decision-view-${panel.id}:checked ~ .decision-panels > #decision-panel-${panel.id} { display: block; } +`).join('\n'); +--- + +
+ {panels.map((panel, index) => )} +
+ {panels.map((panel) => )} +
+
+ {panels.map((panel) =>
+
+ + {panel.label} analyzer decisiveness and correctness + Scatter plot for {panel.label}. The horizontal axis is decisive answers divided by {panel.denominator} assertions. The vertical axis is correct answers divided by decisive answers. + {yTicks.map((tick) => { + const tickY = padTop + (100 - tick) * plotHeight / 100; + return <>{tick}%; + })} + {xTicks.map((tick) => { + const tickX = padLeft + tick * plotWidth / 100; + return <>{tick}%; + })} + + + decisiveness — share of {panel.denominator} assertions answered + correctness among decisive answers + {panel.points.map((point) => { + const decided = point.correct + point.wrong; + const pointX = x(decided, panel.denominator); + const pointY = y(point.correct, decided); + const offset = labelOffsets[point.tool] ?? { x: 9, y: -9 }; + return + {point.name}: {formatPercent(percent(decided, panel.denominator))} decisive ({decided}/{panel.denominator}); {formatPercent(percent(point.correct, decided))} correct among decisions ({point.correct}/{decided}); {panel.denominator - decided} unanswered + + {point.name} + ; + })} + +
+

+ {panel.id === 'overall' + ? `Overall decisiveness uses all ${panel.denominator} assertions in the 13 kernel corpus as every analyzer's denominator; a language with no analyzer entry therefore remains visible as unanswered coverage.` + : `${panel.label} is its own ${panel.denominator}-assertion population. Analyzers without a ${panel.label} kernel entry are absent, never plotted as zero.`} +

+
+ Exact figures +
AnalyzerScopeKernel coverageAccuracyCorrectWrongIncomplete
AnalyzerScopeSupported languagesBenchmark kernelsAccuracyCorrectWrongIncomplete
{point.name} {point.cohort}{point.languageSupport.languages.length}{point.languageSupport.languages.join(', ')} {point.kernels}/{kernelCount} ({formatPercent(percent(point.kernels, kernelCount))}) {point.correct}/{point.covered} ({formatPercent(percent(point.correct, point.covered))}) {point.correct}{point.wrong}{point.incomplete}
+ {panel.kernelId && } + {panel.points.map((point) => { + const decided = point.correct + point.wrong; + return + + + + + {panel.kernelId && } + ; + })} +
AnalyzerDecisivenessCorrectness when decisiveUnansweredEvidence
{point.name}{decided}/{panel.denominator} ({formatPercent(percent(decided, panel.denominator))}){point.correct}/{decided} ({formatPercent(percent(point.correct, decided))}){panel.denominator - decided}
+ + )} +
+ + + + diff --git a/docs/src/components/LandingResults.astro b/docs/src/components/LandingResults.astro index 6c36c3bdc..772776c04 100644 --- a/docs/src/components/LandingResults.astro +++ b/docs/src/components/LandingResults.astro @@ -17,6 +17,7 @@ import { } from '../data/snapshots'; import SnapshotEvolution from './SnapshotEvolution.astro'; import AnalyzerLandscape from './AnalyzerLandscape.astro'; +import DecisionLandscape from './DecisionLandscape.astro'; import LatencyRanking from './LatencyRanking.astro'; const snapshot = currentSnapshot; @@ -228,19 +229,6 @@ breadth.sort((left, right) => left.language.localeCompare(right.language)); const breadthCorrect = breadth.filter((row) => row.correct).length; const breadthIncomplete = breadth.filter((row) => row.incomplete); -// A card's headline fraction is correct-of-population. For an analyzer that -// declines part of the population, that fraction alone reads as a low score -// rather than as a bounded profile, so the card also states the decided -// fraction and names *why* the rest is absent — declared-capability declines -// (`unsupported`) are a design position, not a miss. -function coverageNote(vendor: VendorStats): string { - const parts: string[] = []; - if (vendor.unsupported > 0) parts.push(`${vendor.unsupported} declined`); - if (vendor.inconclusive > 0) parts.push(`${vendor.inconclusive} incomplete`); - if (vendor.runnerErrors > 0) parts.push(`${vendor.runnerErrors} runner-error`); - return `${vendor.name}: ${parts.join(', ')}`; -} - const outcomeClass = (polarity: 'positive' | 'negative', outcome: string) => { if (polarity === 'positive' && outcome === 'reached') return 'good'; if (polarity === 'negative' && outcome === 'not-reached') return 'good'; @@ -615,63 +603,7 @@ const snapshotPage = `/snapshots/${snapshot.slug}`; -
- {kernels.map((kernel) => ( -
2, dense: kernel.vendors.length > 3 }, - ]} - > -

{kernel.language} kernel · correct decisions

-
- {kernel.vendors.map((vendor, index) => ( -
-

{vendor.name}

-

{vendor.correct}/{vendor.total}

- {vendor.incomplete > 0 && ( -

- {vendor.correct} of {vendor.correct + vendor.wrong} decided -

- )} -
- ))} -
-

- {kernel.vendors.some((vendor) => vendor.incomplete > 0) - ? `${kernel.templates} templates · unanswered outcomes (${kernel.vendors - .filter((vendor) => vendor.incomplete > 0) - .map(coverageNote) - .join('; ')}) are coverage, excluded from correctness and never - counted against it` - : `${kernel.templates} templates · all outcomes definitive`} -

-

- -

-
- ))} -
-

Direct-flow breadth · languages fully correct

-
-
-

Bifrost

-

{breadthCorrect}/{breadth.length}

-
-
-

- one positive and one negative direct-propagation assertion per language -

-
-
+
From f2bdeb56f52d3be424945aaeb6e361a909039311 Mon Sep 17 00:00:00 2001 From: David Baker Effendi Date: Mon, 7 Sep 2026 10:25:09 +0200 Subject: [PATCH 09/14] Normalize decision chart spacing --- docs/src/components/DecisionLandscape.astro | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/src/components/DecisionLandscape.astro b/docs/src/components/DecisionLandscape.astro index 023130a14..545644ce5 100644 --- a/docs/src/components/DecisionLandscape.astro +++ b/docs/src/components/DecisionLandscape.astro @@ -170,11 +170,12 @@ const toggleCss = panels.map((panel) => `