diff --git a/CITATION.cff b/CITATION.cff index 455b567..2dd95ac 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -12,5 +12,5 @@ abstract: >- case and fixture digests, analyzer identities, normalized reports, and retained raw evidence. license: MIT -version: v0.3.0 -date-released: "2026-08-24" +version: v0.4.0 +date-released: "2026-08-26" diff --git a/adapters/bifrost/README.md b/adapters/bifrost/README.md index 562a372..4ce5b48 100644 --- a/adapters/bifrost/README.md +++ b/adapters/bifrost/README.md @@ -31,9 +31,9 @@ than reading it from each case. The TypeScript parity slice repeats those 16 templates against `.ts` fixtures through `core-typescript-kernel.rqlp`, and its thirteen challenge templates have now rolled out too, taking its core population to 29 templates and 58 assertions; see the [TypeScript adaptation -matrix](../../docs/typescript-kernel.md). The retained TypeScript report is the -classic 32 — it is freeze-bound by v0.3.0, so its expanded run is deferred to -the v0.4.0 freeze-prep re-run rather than written here. TypeScript is a +matrix](../../docs/typescript-kernel.md). The retained TypeScript report is now +that expanded 58 — `run-bifrost-typescript-kernel` was re-run whole for the +v0.4.0 freeze. TypeScript is a separate population from JavaScript and the two are never mixed. The C# parity slice uses `core-csharp-kernel.rqlp`; its direct-propagation pair is frozen in the v0.2.0 evidence with the breadth `core-direct.rqlp` policy, so the C# @@ -44,9 +44,8 @@ arrangement, and its thirteen challenge templates have now rolled out too, taking its core population to 29 templates and 58 assertions; see [the Go kernel contract](../../docs/go-kernel.md) for its struct, pointer-alias, array, and `panic`/`recover` adaptations, and for the three `reflect`-based challenge -adaptations. The retained Go report is the classic 32 — it is freeze-bound by -v0.3.0, so its expanded run is deferred to the v0.4.0 freeze-prep re-run rather -than written here. The C and C++ +adaptations. The retained Go report is now that expanded 58 — +`run-bifrost-go-kernel` was re-run whole for the v0.4.0 freeze. The C and C++ parity slices use `core-c-kernel.rqlp` and `core-cpp-kernel.rqlp` and are two separate populations with two different denominators: C++ covers all 16 classic templates (32 core assertions), while C's classic core is 15 (30 core @@ -55,9 +54,8 @@ two `language-extension` cases run in the same slice on their own scorecard. C's challenge row has since rolled out — nine of the thirteen preregistered challenge templates are applicable to C — taking its core population to **24 templates and 48 assertions**, and C++'s row has rolled out likewise, to **28 templates and 56 assertions**. The -retained C report is the classic 30: it is freeze-bound by v0.3.0, so **the -expanded Bifrost C evidence is deferred to the v0.4.0 freeze-prep re-run** -rather than written here. See [the C kernel +retained C and C++ reports are those expanded populations: both kernels were +re-run whole for the v0.4.0 freeze. See [the C kernel contract](../../docs/c-kernel.md) and [the C++ kernel contract](../../docs/cpp-kernel.md). The Rust parity slice uses `core-rust-kernel.rqlp` under the same frozen-direct-pair arrangement as C# and @@ -71,8 +69,8 @@ reflection. The `Result`/`?` construct Rust uses instead of exception catch is carried by a `language-extension` pair that the run also evaluates but never counts in the core denominator; see [the Rust kernel contract](../../docs/rust-kernel.md). The retained Rust report -is the classic 30 — it is freeze-bound by v0.3.0, so its expanded run is -deferred to the v0.4.0 freeze-prep re-run rather than written here. The PHP parity slice uses +is now that expanded 54 — `run-bifrost-rust-kernel` was re-run whole for the +v0.4.0 freeze. The PHP parity slice uses `core-php-kernel.rqlp` under the same frozen-direct-pair arrangement as C#, Go, and Rust, and covers PHP's **expanded 29 templates (58 core assertions)** now that its challenge-tier row is rolled out; see [the PHP kernel @@ -97,8 +95,8 @@ templates (58 core assertions)**, and Scala is the only kernel with pinned Joern has no Scala source frontend, so Bifrost is the only tool that produces Scala results at all. See [the Scala kernel contract](../../docs/scala-kernel.md). Every kernel command -selects only its own language's core assertions — 32 for the 16-template -kernels, 48 for the expanded 24-template C kernel, 54 for the expanded +selects only its own language's core assertions — every row is now expanded: +48 for the 24-template C kernel, 54 for the expanded 27-template Rust kernel, 56 for the expanded 28-template C++ kernel, and 58 for the expanded 29-template kernels — and writes a dedicated report. The Java calibration slice also @@ -129,16 +127,18 @@ cargo run -- run-bifrost-ruby-kernel --bifrost /path/to/bifrost cargo run -- run-bifrost-php-kernel --bifrost /path/to/bifrost ``` -`run-bifrost-java-kernel` and `run-bifrost-javascript-kernel` have both now been -run, each over its language's expanded 58-assertion core; their evidence is -described below. Each +All fourteen commands above have now been run for the v0.4.0 freeze, each over +its language's whole current core — 58 assertions for the ten 29-template +kernels, 56 for C++, 54 for Rust, 48 for C, and the pinned 118 cases for the +smoke slice; their evidence is described below. Each selects its language's whole core population and pins the language-qualified policy for the run, accepting the frozen direct-propagation pair's historical policy references (`direct-positive.rqlp` and `explicit-negative.rqlp` for Java, the cross-language breadth policy for JavaScript) rather than rewriting evidence a freeze manifest binds. Their populations follow the challenge rollout table described in [the adapter contract](../../docs/adapters.md): 32 -assertions before a language's row is flipped, the expanded denominator after. +assertions before a language's row is flipped, the expanded denominator after, +and every row is flipped in this freeze. The smoke command selects only cases with an explicit Bifrost policy or unsupported declaration, and never a challenge-tier case: any `template_id` @@ -149,11 +149,14 @@ and evaluated through Bifrost's path-based policy CLI (`--root` plus `--policy-file`). A report with incomplete runs is normalized as `inconclusive` even when it contains no findings; it is never interpreted as a negative. -## Retained v0.10.5 snapshot +## Retained v0.10.6 snapshot -The checked-in `reports/bifrost-smoke.json` was produced with Bifrost v0.10.5, -build identity `728ac69ab93224151c6c951b23d2f5bc681d8558`, and is frozen in -the published v0.3.0 manifest. +The checked-in `reports/bifrost-smoke.json` was produced with Bifrost v0.10.6, +build identity `18d09c57d1e5044dec49acac7635d3255ea8e89c`, and is frozen in +the published v0.4.0 manifest. Every Bifrost report on this tree — the smoke +population and all thirteen kernels — was produced by that one build at the one +fixture revision +`sha256:13a11ff48f26dba889f76aeb9ef60213a129abe5ebcfcb966da3a2418c12807e`. The smoke report contains 118 normalized results: 58 `reached`, 57 `not-reached`, 2 `inconclusive`, and 1 `unsupported`. Every result's `raw_output` points to its retained per-case Bifrost JSON under @@ -166,27 +169,44 @@ The 32-case Java kernel *within the frozen smoke population* has 16 `reached` and 16 `not-reached` outcomes, with 32/32 assertions matching expected polarity and no incomplete outcomes (under v0.10.2 it was 17/32). That population is frozen and does not grow; Java's expanded core is a separate slice, described -under [the Java kernel](#java-kernel-expanded-core) below. The 32-case Python kernel likewise has 16 `reached`, -16 `not-reached`, and 32/32 matching (v0.10.2: 16/32); its dedicated report is -`reports/bifrost-python-kernel.json` and raw evidence is under +under [the Java kernel](#java-kernel-expanded-core) below. + +**An observed instability, published as observed.** +`dfb-taint-java-direct-positive` is `reached` here and `not-reached` in +`reports/bifrost-java-kernel.json`, at the same build and the same fixture +revision. The two are separate populations with separate scorecards, both raw +artifacts are retained and digest-bound, and neither run was repeated to force +agreement: the freeze publishes what the runs produced. The 32/32 above is the +smoke population's own result and the Java kernel's 31/32 is its own; neither +supersedes the other. + +The classic 32 Python assertions likewise have 16 `reached`, 16 `not-reached`, +and 32/32 matching (v0.10.2: 16/32), both inside the smoke population and +inside the dedicated 58-assertion report +`reports/bifrost-python-kernel.json`, whose raw evidence is under `reports/raw/bifrost-python-kernel/`. -**Deferred: the expanded Python population.** Python's challenge-tier row is -rolled out and its core denominator is now 29 templates / 58 assertions, but -`reports/bifrost-python-kernel.json` is one of the nineteen reports -`reports/freeze.json` digest-binds for v0.3.0, so the Python challenge wave did -not re-run it. Those 32 results are the frozen 16-template v0.3.0 evidence and -say nothing either way about the thirteen challenge templates; Bifrost's -evidence for the expanded Python core arrives with the v0.4.0 freeze-prep -re-run. Deferral is not absence of coverage, and the v0.3.0 and v0.4.0 -populations are never compared number-to-number. The Python challenge cases are -also excluded from the smoke selection, which stays pinned at its frozen 118 -cases. - -The 32-case -JavaScript kernel has 16 `reached`, 16 `not-reached`, and 32/32 matching -(v0.10.2: 19/32). Unlike the v0.10.2 snapshot, this v0.10.5 evidence decides -all three of these kernels completely. +**The expanded Python population.** Python's challenge-tier row is rolled out +and its core denominator is 29 templates / 58 assertions; +`reports/bifrost-python-kernel.json` was re-run whole for the v0.4.0 freeze and +now carries all 58. It scores **36/58**: the classic 32 reproduce the smoke +evidence case for case at 32/32, and on the challenge 26 the engine decides +only the depth-6 relay pair and the two-level context pair, both ways, and +**all four decisions are correct** — no false positive and no false negative in +the tier. Twenty are `inconclusive` (12 `capability_incomplete` "no analysis +root contains both a selected source and sink" across the reflective, +dispatch-table, closure, function-field, callback and anonymous-implementation +pairs; 8 `partial_discovery` across the computed-property, map-iteration, +nested-access-path and recursive-carry pairs) and 2 are `runner-error` on the +`element-object` pair, the shared `internal_invariant` "oracle relation" defect +tracked upstream as bifrost-dev #2639. The Python challenge cases stay excluded +from the smoke selection, which remains pinned at its frozen 118 cases, and a +32-assertion score is never compared with a 58-assertion one. + +The 32 classic +JavaScript assertions have 16 `reached`, 16 `not-reached`, and 32/32 matching +(v0.10.2: 19/32). Unlike the v0.10.2 snapshot, this v0.10.6 evidence decides +all three of these classic slices completely. ### JavaScript expanded core — the dedicated kernel run @@ -195,76 +215,95 @@ JavaScript's thirteen preregistered challenge templates templates / **58 assertions**. Those assertions are carried by a new dedicated report, `reports/bifrost-javascript-kernel.json`, written by `run-bifrost-javascript-kernel` and evidenced under -`reports/raw/bifrost-javascript-kernel/`. **The frozen smoke report is -untouched**: its JavaScript slice stays the classic 32 assertions and the smoke -population stays pinned at 118 cases, because the smoke selector excludes +`reports/raw/bifrost-javascript-kernel/`. **The smoke population is +unchanged**: its JavaScript slice stays the classic 32 assertions and the smoke +selection stays pinned at 118 cases, because the smoke selector excludes challenge templates outright. | Stratum | Assertions | Polarity match | Outcomes | | --- | --- | --- | --- | | Classic (16 templates) | 32 | 32/32 | 16 `reached`, 16 `not-reached` | -| Challenge (13 templates) | 26 | 3/26 | 1 `reached`, 2 `not-reached`, 21 `inconclusive`, 2 `runner-error` | +| Challenge (13 templates) | 26 | 4/26 | 2 `reached`, 2 `not-reached`, 20 `inconclusive`, 2 `runner-error` | The classic half reproduces the frozen smoke evidence case for case. On the challenge half the engine decided only two of the thirteen pairs — the -two-level context pair, both ways, and the depth-6 relay negative — and **every +two-level context pair and the depth-6 relay, both ways each — and **every decision it made was correct**: the stratum contains no false positive and no -false negative. Twenty-one assertions are `inconclusive`, each retaining -`partial_discovery` evidence of the form "taint discovery is incomplete: -procedure value-flow snapshot for … is unknown", across the reflective, -computed-property, dispatch-table, closure, function-field, callback, -anonymous-implementation, map-iteration, nested-path and recursive pairs plus -the depth-6 relay positive. The `element-object` pair is `runner-error`: on an -array of object literals the engine reports `internal_invariant` with "invalid -value-flow snapshot: oracle relation does not belong to the required query -arena and role". That is retained exactly as observed — an engine defect worth -reporting upstream, and never a negative result. +false negative. Twenty assertions are `inconclusive`: twelve retain +`capability_incomplete` "taint semantic binding is unavailable: no analysis +root contains both a selected source and sink" (the reflective, dispatch-table, +closure, function-field, callback and anonymous-implementation pairs), two +retain `capability_incomplete` "unsupported (normal_control_flow)" (the +map-iteration pair), and six retain `partial_discovery` (the computed-property, +recursive-carry and nested-access-path pairs). The `element-object` pair is +`runner-error`: on an array of object literals the engine reports +`internal_invariant` with "invalid value-flow snapshot: oracle relation does +not belong to the required query arena and role". That is retained exactly as +observed — an engine defect, tracked upstream as bifrost-dev #2639, and never a +negative result. This run is at fixture revision -`sha256:64ef139f452fd296bb26463bc552e5e5998ca4bb4584d45565d858424814bde9`, -which no earlier Bifrost report carries; the two are not pooled, and a -32-assertion score is never compared with a 58-assertion one. - -Every Bifrost slice in the v0.3.0 freeze — the smoke population and the -Kotlin, TypeScript, C#, Go, C, C++, Python, and Rust kernels — was run on this -one build. - -The Kotlin kernel snapshot covers the **classic 32-case population**: 12 -`reached`, 10 `not-reached`, and 10 `inconclusive`, with 19/32 assertions -matching expected polarity (19 of 22 decisive outcomes; under v0.10.2 it was -17/32). Its dedicated report is `reports/bifrost-kotlin-kernel.json` and raw -evidence is under `reports/raw/bifrost-kotlin-kernel/`. The ten `inconclusive` +`sha256:13a11ff48f26dba889f76aeb9ef60213a129abe5ebcfcb966da3a2418c12807e`, the +single revision the v0.4.0 freeze binds; a 32-assertion score is never compared +with a 58-assertion one. + +Every Bifrost slice in the v0.4.0 freeze — the smoke population and the Java, +JavaScript, Kotlin, TypeScript, C#, Go, C, C++, Python, Rust, Ruby, PHP, and +Scala kernels — was run on this one build. + +The Kotlin kernel's **classic 32 assertions** are 12 `reached`, 10 +`not-reached`, and 10 `inconclusive`, with 19/32 matching expected polarity (19 +of 22 decisive outcomes; under v0.10.2 it was 17/32). Its dedicated report is +`reports/bifrost-kotlin-kernel.json` and raw evidence is under +`reports/raw/bifrost-kotlin-kernel/`. The ten `inconclusive` results are the complete heap/separation stratum and the exception-catch pair, both polarities, each retaining `partial_discovery` evidence; they mirror the -Java kernel profile and are never counted as negatives. +Java kernel profile and are never counted as negatives. The three decisive +mismatches are the `expression` positive (a false negative) and the +`infeasible-branch` and `loop-carried` negatives (false positives). Kotlin's core population is now **58 assertions**, its challenge-tier row -having been rolled out, but `reports/bifrost-kotlin-kernel.json` is one of the -nineteen reports `reports/freeze.json` digest-binds for v0.3.0, so -`run-bifrost-kotlin-kernel` was **not** re-run for that expansion: **expanded -Bifrost evidence for Kotlin is pending the v0.4.0 freeze-prep re-run**, and the -snapshot above remains a valid classic-population result that says nothing -about the 26 challenge assertions. Deferral is not absence of coverage; see the +having been rolled out, and `run-bifrost-kotlin-kernel` was re-run whole for +the v0.4.0 freeze, so the report carries all 58 and scores **25/58**. The +classic 32 above reproduce the pre-expansion snapshot case for case. On the +challenge 26 the engine decides three pairs — recursive carry, the two-level +context pair, and the depth-6 relay — **all six decisions correct**, with no +false positive and no false negative in the tier; 18 are `inconclusive` (12 +`capability_incomplete` "no analysis root contains both a selected source and +sink" across the reflective, dispatch-table, closure, function-field, callback +and anonymous-implementation pairs, 6 `partial_discovery` across the +computed-property, map-iteration and nested-access-path pairs) and 2 are +`runner-error` on the `element-object` pair, the `internal_invariant` "oracle +relation" defect tracked upstream as bifrost-dev #2639. Twenty undecided +challenge assertions are not twenty misses; see the [Kotlin kernel contract](../../docs/kotlin-kernel.md). -The 32-case TypeScript kernel, in its own report +The TypeScript kernel's classic 32 assertions, in its own report `reports/bifrost-typescript-kernel.json` with raw evidence under -`reports/raw/bifrost-typescript-kernel/`, has 15 `reached`, 15 `not-reached`, -and 2 `inconclusive`, with 30/32 assertions matching expected polarity — all +`reports/raw/bifrost-typescript-kernel/`, are 15 `reached`, 15 `not-reached`, +and 2 `inconclusive`, with 30/32 matching expected polarity — all 30 decisive outcomes are correct. The two `inconclusive` results are the -exception-catch pair, retaining `capability_incomplete` evidence. Under +exception-catch pair, retaining `capability_incomplete` "unsupported +(exceptional_control_flow)" evidence. Under v0.10.2 this kernel matched 19/32; the alias-propagation and array-element -pairs are now decisive and correct. That report is the classic 32-assertion -population only: TypeScript's core is now 58 assertions, but the report is -freeze-bound by v0.3.0, so **the expanded Bifrost TypeScript evidence is -pending the v0.4.0 freeze-prep re-run** and this snapshot is not an -expanded-core number. - -The C# kernel's frozen 32-case run, in its own report +pairs are now decisive and correct. TypeScript's core is now 58 assertions and +`run-bifrost-typescript-kernel` was re-run whole for the v0.4.0 freeze, so the +report carries all 58 and scores **34/58**. On the challenge 26 it decides the +depth-6 relay pair and the two-level context pair, both ways, and **all four +decisions are correct**; 20 are `inconclusive` (12 `capability_incomplete` "no +analysis root contains both a selected source and sink", 2 +`capability_incomplete` "unsupported (normal_control_flow)" on the +map-iteration pair, 6 `partial_discovery` on the computed-property, +recursive-carry and nested-access-path pairs) and 2 are `runner-error` on the +`element-object` pair — the `internal_invariant` "oracle relation" defect +tracked upstream as bifrost-dev #2639. That is the same challenge profile +JavaScript shows, on a population that is nonetheless never pooled with it. + +The C# kernel's classic 32 assertions, in its own report `reports/bifrost-csharp-kernel.json` -with raw evidence under `reports/raw/bifrost-csharp-kernel/`, contains 1 +with raw evidence under `reports/raw/bifrost-csharp-kernel/`, are 1 `reached`, 1 -`not-reached`, and 30 `inconclusive` results: only the direct-propagation pair +`not-reached`, and 30 `inconclusive`: only the direct-propagation pair is decisive, and both of its outcomes match the expected polarity. The 30 inconclusive results retain `partial_discovery` (20) or `capability_incomplete` (10) evidence with per-case diagnostics showing that Bifrost's procedure @@ -273,21 +312,26 @@ This is capability coverage, never a negative result. The same incompleteness reproduces under the language-agnostic `core-direct.rqlp` policy, so it is not an artifact of the language-qualified policy. -**Deferred: the expanded C# population.** C#'s challenge-tier row is now -rolled out and its core denominator is 29 templates / 58 assertions, but -`reports/bifrost-csharp-kernel.json` is one of the nineteen reports -`reports/freeze.json` digest-binds for v0.3.0, so the C# challenge wave did not -re-run it — the same treatment the Python wave gave its freeze-bound report. -Those 32 results are the frozen 16-template v0.3.0 evidence and say nothing -either way about the thirteen challenge templates; Bifrost's evidence for the -expanded C# core arrives with the v0.4.0 freeze-prep re-run. Deferral is not -absence of coverage, and the v0.3.0 and v0.4.0 populations are never compared -number-to-number. The C# challenge cases are also excluded from the smoke -selection, which stays pinned at its frozen 118 cases. - -The 32-case Go kernel, in its own report `reports/bifrost-go-kernel.json` with -raw evidence under `reports/raw/bifrost-go-kernel/`, contains 5 `reached`, 5 -`not-reached`, and 22 `inconclusive` results. Five template pairs are decisive — +**The expanded C# population.** C#'s challenge-tier row is rolled out and its +core denominator is 29 templates / 58 assertions; +`reports/bifrost-csharp-kernel.json` was re-run whole for the v0.4.0 freeze and +carries all 58, scoring **3/58**. The classic 32 above reproduce the +pre-expansion snapshot case for case. On the challenge 26 exactly one assertion +is decisive — the depth-6 relay positive, `reached` and correct — and the other +25 are `inconclusive`: 12 retain `capability_incomplete` "no analysis root +contains both a selected source and sink" (the reflective, dispatch-table, +closure, function-field, callback and anonymous-implementation pairs), 6 retain +`capability_incomplete` "unsupported (assignments)" (the element-object, +map-iteration and nested-access-path pairs), and 7 retain `partial_discovery` +(the computed-property and recursive-carry pairs, the two-level context pair, +and the depth-6 relay negative). One decisive cell beside 25 declines is +capability coverage, never 25 negatives. The C# challenge cases are also +excluded from the smoke selection, which stays pinned at its frozen 118 cases. + +The Go kernel's classic 32 assertions, in its own report +`reports/bifrost-go-kernel.json` with +raw evidence under `reports/raw/bifrost-go-kernel/`, are 5 `reached`, 5 +`not-reached`, and 22 `inconclusive`. Five template pairs are decisive — direct propagation, the local multi-step chain, call-context separation, and the one-hop and two-hop return relays — and all ten of those outcomes match the expected polarity. The 22 inconclusive results retain `partial_discovery` (12) @@ -295,30 +339,53 @@ or `capability_incomplete` (10) evidence; the ten are the four heap pairs ("procedure value-flow snapshot ... is unsupported (assignments)") and the `panic`/`recover` exception pair, where Bifrost cannot bind the sink operand supplied by `recover()`. All of this is capability coverage, never a negative -result; see [the Go kernel contract](../../docs/go-kernel.md). That report is -the classic 32-assertion population only: Go's core is now 58 assertions, but -the report is freeze-bound by v0.3.0, so **the expanded Bifrost Go evidence is -pending the v0.4.0 freeze-prep re-run** and this snapshot is not an -expanded-core number. - -**Deferred: the expanded C population.** C's challenge-tier row is now rolled -out and its core denominator is 24 templates / 48 assertions — nine of the -thirteen preregistered challenge templates are applicable to C — but -`reports/bifrost-c-kernel.json` is one of the nineteen reports -`reports/freeze.json` digest-binds for v0.3.0, so the C challenge wave did not -re-run it, exactly as the C# and TypeScript waves treated their freeze-bound -reports. The retained C snapshot (32 results: 1 `reached`, 1 `not-reached`, 28 -`inconclusive` core outcomes plus 2 `inconclusive` `language-extension` cases, -detailed in [the C kernel contract](../../docs/c-kernel.md)) is the classic -30-assertion population and is not an expanded-core number. **Expanded Bifrost -C evidence is pending the v0.4.0 freeze-prep re-run.** - -The 30-assertion Rust kernel, in its own report +result; see [the Go kernel contract](../../docs/go-kernel.md). Go's core is now +58 assertions and `run-bifrost-go-kernel` was re-run whole for the v0.4.0 +freeze, so the report carries all 58 and scores **14/58**. The classic 32 +reproduce the pre-expansion snapshot case for case, and on the challenge 26 the +engine decides the depth-6 relay pair and the two-level context pair, both +ways, **all four correctly**. The remaining 22 are `inconclusive`: 12 +`capability_incomplete` "no analysis root contains both a selected source and +sink", 6 `capability_incomplete` "unsupported (assignments)" on the +element-object, map-iteration and nested-access-path pairs, and 4 +`partial_discovery` on the computed-property and recursive-carry pairs. Go +produces no `runner-error` anywhere in its population. + +**The expanded C population.** C's challenge-tier row is rolled out and its +core denominator is 24 templates / 48 assertions — nine of the thirteen +preregistered challenge templates are applicable to C — and +`reports/bifrost-c-kernel.json` was re-run whole for the v0.4.0 freeze. It +contains 50 results and scores **2/48**: 1 `reached` and 1 `not-reached` on the +direct-propagation pair, 46 `inconclusive` core outcomes (28 classic and 18 +challenge), plus 2 `inconclusive` `language-extension` cases reported on their +own scorecard, detailed in [the C kernel contract](../../docs/c-kernel.md). No +challenge assertion is decisive: 6 retain `capability_incomplete` "no analysis +root contains both a selected source and sink" (the dispatch-table, +function-field and callback-registration pairs) and 12 retain +`partial_discovery` "procedure value-flow snapshot … is unknown". Eighteen +undecided challenge assertions are capability coverage, never eighteen +negatives. + +**The expanded C++ population.** C++'s core denominator is 28 templates / 56 +assertions — twelve of the thirteen challenge templates apply, the +reflective-invocation cell being inapplicable — and +`reports/bifrost-cpp-kernel.json`, with raw evidence under +`reports/raw/bifrost-cpp-kernel/`, was re-run whole for the v0.4.0 freeze. It +scores **2/56**: the direct-propagation pair is the only decisive one, both +ways correct, and the other 54 assertions are `inconclusive` — 10 retaining +`capability_incomplete` "no analysis root contains both a selected source and +sink" (the dispatch-table, closure, function-field, callback and +anonymous-implementation pairs) and 44 retaining `partial_discovery`. C++ +produces no `runner-error` anywhere in its population; see [the C++ kernel +contract](../../docs/cpp-kernel.md). + +The **54-assertion** Rust kernel, in its own report `reports/bifrost-rust-kernel.json` with raw evidence under -`reports/raw/bifrost-rust-kernel/`, produces 1 `reached`, 1 `not-reached`, and -20 `inconclusive` and 8 `runner-error` core results: only the +`reports/raw/bifrost-rust-kernel/`, was re-run whole for the v0.4.0 freeze and +scores **2/54**. On its classic 30 it produces 1 `reached`, 1 `not-reached`, +20 `inconclusive` and 8 `runner-error` results: only the direct-propagation pair is decisive, and both of its outcomes match the -expected polarity (2 of 2 decisive outcomes, 2 of 30 assertions). The two +expected polarity (2 of 2 decisive outcomes, 2 of 30 classic assertions). The two `language-extension` assertions are both `inconclusive` and are reported separately, never in the core denominator. The inconclusive core results retain `partial_discovery` evidence. The eight `runner-error` results — the @@ -327,10 +394,21 @@ same-object field separation, alias propagation, and array element — retain raw runs that complete as `failed` with `internal_invariant` ("semantic IR gap_contract error in procedure 2: gap 8 duplicates the same scoped fact"): a failed evaluation is an execution error, normalized as `runner-error`, and is -never counted as a negative. That report is the classic 30-assertion population -only: Rust's core is now 54 assertions, but the report is freeze-bound by -v0.3.0, so **the expanded Bifrost Rust evidence is pending the v0.4.0 -freeze-prep re-run** and this snapshot is not an expanded-core number. +never counted as a negative. That signature is tracked upstream as bifrost-dev +#2638. + +On Rust's twelve applicable challenge templates — 24 assertions, the +reflective-invocation cell being inapplicable to a language with no run-time +reflection — **no assertion is decisive**: 20 are `inconclusive` (10 retaining +`capability_incomplete` "no analysis root contains both a selected source and +sink" across the dispatch-table, closure, function-field, callback and +anonymous-implementation pairs, and 10 retaining `partial_discovery` across the +computed-property, map-iteration, recursive-carry, depth-6 relay and two-level +context pairs) and 4 are `runner-error` on the `element-object` and +`nested-access-path` pairs, failing with the same `gap_contract` signature as +the classic heap stratum — bifrost-dev #2638, not the separate "oracle +relation" defect the other frontends hit. Twenty-four undecided challenge +assertions are execution and capability coverage, never twenty-four negatives. The 58-case Ruby kernel, in its own report `reports/bifrost-ruby-kernel.json` with raw evidence under `reports/raw/bifrost-ruby-kernel/`, contains **58 @@ -339,10 +417,10 @@ polarity match is 0 of 0 decisive outcomes. This is not a regression and not a negative result — it is the analyzer-coverage gate `docs/applicability-matrix.md` records for Ruby, now measured over the whole expanded 29-template population instead of the 16-template core or the two -breadth assertions. `reports/bifrost-ruby-kernel.json` is **not** freeze-bound — -the Ruby kernel landed after v0.3.0, so no Ruby report appears in -`reports/freeze.json` — so this is a whole-population replacement rather than a -deferral. Twenty-eight results retain `partial_discovery` evidence ("procedure +breadth assertions, and it is tracked upstream as bifrost-dev #2637. +`reports/bifrost-ruby-kernel.json` was not bound by the v0.3.0 freeze — the +Ruby kernel landed after it — and is one of the 42 reports +`reports/freeze.json` digest-binds for v0.4.0. Twenty-eight results retain `partial_discovery` evidence ("procedure value-flow snapshot for `` is unknown"); fourteen retain `capability_incomplete` "unsupported (assignments)" (the four classic heap/separation pairs, the exception-catch pair, and the challenge @@ -377,15 +455,15 @@ and anonymous-implementation pairs; 8 `capability_incomplete` "unsupported nested-path pairs; 2 `partial_discovery` on the recursive-carry pair) and the 4 it decides — the deep-relay and depth-2 context pairs — are all correct. No inconclusive result is ever counted as a negative. This report was produced -after the v0.3.0 freeze and is not bound by it; see [the PHP kernel +after the v0.3.0 freeze and is bound by the v0.4.0 one; see [the PHP kernel contract](../../docs/php-kernel.md). The Scala kernel, in its own report `reports/bifrost-scala-kernel.json` with raw evidence under `reports/raw/bifrost-scala-kernel/`, was re-run whole on the same -v0.10.5 build over its **expanded 58-assertion core** — 32 classic plus the 26 +v0.10.6 build over its **expanded 58-assertion core** — 32 classic plus the 26 challenge assertions the [challenge-tier -preregistration](../../docs/challenge-tier.md) adds. The report is post-freeze -and not bound by v0.3.0, so replacing it invalidated nothing. It produces 5 +preregistration](../../docs/challenge-tier.md) adds. The report post-dates the +v0.3.0 freeze and is bound by the v0.4.0 one. It produces 5 `reached`, 5 `not-reached`, and 48 `inconclusive` results, **10 of 58** matching the expected polarity with **no decisive mismatch anywhere**. The classic thirty-two are identical case-for-case to the pre-expansion report — the same @@ -411,35 +489,43 @@ the smoke population that also covers Java is frozen at 118 cases and must not grow. The challenge tier is excluded from the smoke selection outright for that reason. -The first run of this slice, on the same v0.10.5 build, produces 18 `reached`, -19 `not-reached`, 19 `inconclusive`, and 2 `runner-error` results. Its classic -32 assertions reproduce the frozen smoke slice's Java outcomes case for case — -16 `reached`, 16 `not-reached`, 32/32 matching — which is the control that says -the expansion did not disturb the population it was added to. - -Of the 26 challenge assertions, five are decisive and all five are correct -(both `recursive-carry` cells, both `context-pair-depth2` cells, and -`deep-relay-chain-negative`); there is no false positive and no false negative -anywhere in the tier. The other 21 are capability or execution coverage: 19 +The v0.4.0 run of this slice, on the v0.10.6 build, produces 18 `reached`, +20 `not-reached`, 18 `inconclusive`, and 2 `runner-error` results, scoring +**37/58**. Its classic 32 assertions match **31/32**: they reproduce the +smoke slice's Java outcomes case for case except for +`dfb-taint-java-direct-positive`, which is `not-reached` here — a false +negative — and `reached` in `reports/bifrost-smoke.json` at the same build and +the same fixture revision. That disagreement is published as observed rather +than re-run to agreement; the two populations keep separate scorecards and both +raw artifacts are retained and digest-bound. + +Of the 26 challenge assertions, six are decisive and all six are correct (both +`recursive-carry` cells, both `context-pair-depth2` cells, and both +`deep-relay-chain` cells — Java and Kotlin are the only Bifrost kernels that +decide all three stratum-D pairs); there is no false positive and no false negative +anywhere in the tier. The other 20 are capability or execution coverage: 18 `inconclusive` — 10 retaining `capability_incomplete` "no analysis root contains both a selected source and sink" (the reflective-invocation, dispatch-table, closure-capture, function-field, and callback-registration -pairs) and 9 retaining `partial_discovery` "procedure value-flow snapshot ... +pairs) and 8 retaining `partial_discovery` "procedure value-flow snapshot ... is unknown" (the computed-property, anonymous-implementation, map-iteration, -and nested-access-path pairs, plus the six-hop relay positive) — and 2 +and nested-access-path pairs) — and 2 `runner-error` on the `element-object` pair, where the run fails with `internal_invariant` and "invalid value-flow snapshot: oracle relation does not belong to the required query arena and role". That failure is retained -verbatim and published as an engine defect; it is not a negative result. Per -the preregistration's own reading rule, correct stratum-D negatives beside an -undecided six-hop positive describe a bound, not precision. See [the Java +verbatim and published as an engine defect, tracked upstream as bifrost-dev +#2639; it is not a negative result. The undecided stratum-A and stratum-B +pairs are the nested-callable-root gap tracked as bifrost-dev #2640: the sink +call sits inside a lambda, an anonymous class body, or a registered callback, +and no analysis root contains both endpoints. See [the Java kernel contract](../../docs/java-kernel.md). -The JavaScript alias-propagation and array-element pairs retain -`partial_discovery` evidence, while the exception-catch pair retains -`capability_incomplete` evidence; all six remain `inconclusive`. The Java -heap/separation and exception pairs and the Python heap/control-flow pairs -likewise remain `inconclusive`, never negative results. The Ruby direct-flow -pair retains `partial_discovery` evidence, and the modeled-external Java -calibration case is the single explicit `unsupported` result. [Bifrost #1951](https://github.com/BrokkAi/bifrost/issues/1951) -tracks the final cross-language production-taint acceptance work. +Within the pinned smoke population the Java, JavaScript, and Python slices are +now decided completely: the only incomplete outcomes left are the Ruby +direct-flow pair, which retains `partial_discovery` evidence, and the +modeled-external Java calibration case, the single explicit `unsupported` +result. Neither is ever counted as a negative. [Bifrost #1951](https://github.com/BrokkAi/bifrost/issues/1951) +tracks the final cross-language production-taint acceptance work; the defects +and coverage gaps this population exposes are tracked as bifrost-dev #2637 +(Ruby), #2638 (Rust `gap_contract`), #2639 (`element-object`), and #2640 +(nested-callable roots). diff --git a/adapters/codeql/README.md b/adapters/codeql/README.md index e2b943f..484bfc1 100644 --- a/adapters/codeql/README.md +++ b/adapters/codeql/README.md @@ -21,8 +21,9 @@ challenge-tier row, plus two Rust `language-extension` cases that never enter the Rust core denominator. Ruby has its own production extractor and its own expanded 29-template population; it is the primary decisive analyzer for the Ruby tranche, whose Bifrost coverage gate is recorded in [the Ruby kernel -contract](../../docs/ruby-kernel.md), and it is the only CodeQL kernel whose -report is not freeze-bound. +contract](../../docs/ruby-kernel.md). All eleven kernel reports are bound by +the v0.4.0 freeze, each re-run over its own expanded population at one fixture +revision. Scala is deliberately absent. CodeQL CLI 2.26.3 has no Scala extractor and no Scala library pack in any build mode, so there is no `scala/` pack, no query, @@ -31,8 +32,8 @@ recorded in [the Scala kernel contract](../../docs/scala-kernel.md) — and never a negative result for any Scala assertion. It is restated unchanged for Scala's **expanded 29-template / 58-assertion core**: the 26 challenge assertions are covered by the same absence as the 32 classic ones, and because -the extractor does not exist, this is coverage rather than evidence deferred to -the v0.4.0 re-run. +the extractor does not exist, this is coverage rather than a missing run: the +v0.4.0 freeze re-ran every CodeQL kernel that exists, and Scala is not one. The checked-in query packs contain the Java, JavaScript, TypeScript, Python, Kotlin, C#, Go, C, C++, Rust, and Ruby kernel queries. Each query uses that @@ -145,10 +146,11 @@ them — the direct-propagation pair frozen in v0.2.0 as part of the cross-language breadth slice — declare no CodeQL reference at all; see the [Kotlin kernel contract](../../docs/kotlin-kernel.md). -`reports/codeql-kotlin-kernel.json` is freeze-bound, so the runner was **not** -executed over the expanded population: the retained Kotlin snapshot below is a -classic 32-assertion result, and **expanded CodeQL evidence for Kotlin is -pending the v0.4.0 freeze-prep re-run**. +`reports/codeql-kotlin-kernel.json` was re-run whole over that expanded +population for the v0.4.0 freeze and now carries all 58 assertions, scoring +**46/58** — 27/32 on the classic sixteen templates, unchanged case for case, +and 19/26 on the challenge thirteen. The retained Kotlin snapshot below reports +both strata. CodeQL CLI 2.26.3 cannot extract Kotlin under `--build-mode=none`, so the runner traces a real `kotlinc` compile per case: @@ -198,31 +200,38 @@ model and must not be used as a proxy for this kernel. ## Retained JavaScript snapshot -**Expanded evidence is deferred.** `reports/codeql-javascript-kernel.json` is -one of the nineteen reports `reports/freeze.json` digest-binds for v0.3.0, so -the JavaScript challenge expansion did not overwrite it: the expanded -58-assertion CodeQL evidence is pending the v0.4.0 freeze-prep re-run, on this -repository's established re-run-at-freeze pattern, and the deferral is recorded -in `docs/javascript-kernel.md`. What follows is the valid classic -32-assertion snapshot, and it describes a different population from the -expanded one. +`reports/codeql-javascript-kernel.json` was re-run whole over the expanded +population for the v0.4.0 freeze, on this repository's established +re-run-at-freeze pattern, and now contains **58 results**: 23 `reached` and 35 +`not-reached`, with zero `inconclusive`, `unsupported`, or `runner-error` +outcomes. **48 of 58** match the expected polarity — 29/32 on the classic +sixteen templates, identical case for case to the pre-expansion snapshot, and +19/26 on the challenge thirteen. A 32-assertion score and a 58-assertion score +are different populations and are never compared. -The checked-in `reports/codeql-javascript-kernel.json` contains 32 results: -15 `reached` and 17 `not-reached`, with zero `inconclusive`, `unsupported`, or -`runner-error` outcomes. Twenty-nine of 32 outcomes match the expected -polarity. The three mismatches are: +The three classic mismatches are: - `dfb-taint-javascript-alias-propagation-positive`: false negative. - `dfb-taint-javascript-expression-positive`: false negative. - `dfb-taint-javascript-loop-carried-negative`: false positive. -All 32 raw case outputs are SARIF files under +The seven challenge mismatches — A **3/6**, B **6/8**, C **4/6**, D **6/6** — +are six false negatives on positives (`reflective-invocation`, +`dispatch-table`, `function-field`, `callback-registration`, `map-iteration`, +`nested-access-path`) and one false positive, the `computed-property` negative. +The `reflective-invocation` and `dispatch-table` positives are missed while +their negatives are correct — an under-approximating refusal to follow a callee +named at run time — and the `computed-property` pair inverts that, reaching the +positive and also flagging the negative. The preregistration reads stratum A as +approximation character rather than as a ranking. + +All 58 raw case outputs are SARIF files under `reports/raw/codeql-javascript/`; there are zero retained error files. The normalized report has empty `witness_checkpoints` for every case because the current adapter records anchor-backed flow outcomes while retaining path evidence in SARIF rather than fabricating normalized witness markers. Its configuration hash is -`a038e39eb93d6fc674ab59cf2e4de5b3608f1d7b294c19da75ce1bd041c75ac5`. +`cb54d749e915208a1fa7fceaa1e5e5302c18960aebf724573040fda66c7a7ba8`. ## TypeScript kernel @@ -262,21 +271,24 @@ the freeze rationale. ### Retained TypeScript snapshot -**Expanded evidence is deferred.** `reports/codeql-typescript-kernel.json` is -one of the nineteen reports `reports/freeze.json` digest-binds for v0.3.0, so -the TypeScript challenge expansion did not overwrite it: the expanded -58-assertion CodeQL evidence is pending the v0.4.0 freeze-prep re-run, on this -repository's established re-run-at-freeze pattern, and the deferral is recorded -in `docs/typescript-kernel.md`. What follows is the valid classic 32-assertion -snapshot, and it describes a different population from the expanded one. - -The checked-in `reports/codeql-typescript-kernel.json` contains 32 results: 15 -`reached` and 17 `not-reached`, with zero `inconclusive`, `unsupported`, or -`runner-error` outcomes. Twenty-nine of 32 match expected polarity; the -alias-propagation and arithmetic-expression positives are false negatives and -the loop-carried negative is a false positive. That is case-for-case identical -to the JavaScript snapshot above, as expected for fixtures that differ only by -type annotations. All 32 raw outputs are SARIF files with zero error files. +`reports/codeql-typescript-kernel.json` was re-run whole over the expanded +population for the v0.4.0 freeze, on this repository's established +re-run-at-freeze pattern, and now contains **58 results**: 23 `reached` and 35 +`not-reached`, with zero `inconclusive`, `unsupported`, or `runner-error` +outcomes. **48 of 58** match expected polarity — 29/32 on the classic sixteen, +identical case for case to the pre-expansion snapshot, and 19/26 on the +challenge thirteen (A **3/6**, B **6/8**, C **4/6**, D **6/6**). + +The classic mismatches are the alias-propagation and arithmetic-expression +positives, both false negatives, and the loop-carried negative, a false +positive; the challenge mismatches are false negatives on the +`reflective-invocation`, `dispatch-table`, `function-field`, +`callback-registration`, `map-iteration` and `nested-access-path` positives and +one false positive on the `computed-property` negative. That is case-for-case +identical to the JavaScript snapshot above, across both strata, as expected for +fixtures that differ only by +type annotations — and the two remain separate populations that are never +pooled. All 58 raw outputs are SARIF files with zero error files. The run used CodeQL CLI 2.26.3, build SHA `7d097a43199effe04ecd9c6bd3ad9bb02a45b3d7`, with `codeql/javascript-all@2.9.0` resolved by `codeql pack install` from the registry — no `--codeql-packs` @@ -352,16 +364,19 @@ populates the compilation cache before the runner starts, so per-case analysis does not repeat query compilation. Every case still uses an isolated cold database; no database or compiled fixture is reused across the pair. -**Deferred: the expanded Python population.** Python's challenge-tier row is -now rolled out and its core denominator is 29 templates / 58 assertions, but -`reports/codeql-python-kernel.json` is one of the nineteen reports -`reports/freeze.json` digest-binds for v0.3.0. Overwriting it would invalidate -a published freeze, so the Python challenge wave left it untouched. Its 32 -results remain the frozen 16-template v0.3.0 evidence, and CodeQL's evidence -for the expanded Python core arrives with the v0.4.0 freeze-prep re-run. This -is deferral, not absence of coverage, and the two populations are never -compared number-to-number. The selection code already expects 58; the runner is -simply not invoked until the freeze is re-cut. +**The expanded Python population.** Python's challenge-tier row is rolled out +and its core denominator is 29 templates / 58 assertions; +`reports/codeql-python-kernel.json` was re-run whole for the v0.4.0 freeze and +now contains **58 results**: 23 `reached` and 35 `not-reached`, with zero +`inconclusive`, `unsupported`, or `runner-error` outcomes. **48 of 58** match +the expected polarity — 28/32 on the classic sixteen, identical case for case +to the frozen v0.3.0 evidence, and 20/26 on the challenge thirteen (A **3/6**, +B **6/8**, C **5/6**, D **6/6**). The six challenge mismatches are false +negatives on the `reflective-invocation`, `computed-property`, +`dispatch-table`, `closure-capture` and `callback-registration` positives and +one false positive on the `element-object` negative. The v0.3.0 and v0.4.0 +populations are never compared number-to-number. Its configuration hash is +`f97f0198f19f2d1d8630b48ff5d30d947e9f83b940de38af425076cf73e82230`. ## C# kernel @@ -401,25 +416,26 @@ diagnostics) under `reports/raw/codeql-csharp-kernel/`. SARIF locations are reconciled with the case's `DFB-SINK:` anchor by resolving the declared sink method name and accepting a finding on a line that calls it in the same file. -The checked-in `reports/codeql-csharp-kernel.json` contains 32 results: 15 -`reached` and 17 `not-reached`, with zero `inconclusive`, `unsupported`, or -`runner-error` outcomes. 27 of 32 match the expected polarity; the false +The checked-in `reports/codeql-csharp-kernel.json` contains **58 results**: 24 +`reached` and 34 `not-reached`, with zero `inconclusive`, `unsupported`, or +`runner-error` outcomes. **47 of 58** match the expected polarity — 27/32 on +the classic sixteen and 20/26 on the challenge thirteen. On the classic stratum +the false negatives are the alias-propagation, exception-catch, and expression positives, and the false positives are the array-element and loop-carried negatives — the -same mismatch set the Java kernel shows on those templates. Its configuration +same mismatch set the Java kernel shows on those templates; that half is +identical case for case to the frozen v0.3.0 evidence. Its configuration hash is `cd5f68b8ccb2e4de27cf1606b0c9f2ee8981ce5dfdf8ee2fea08fe977a0c56c9`. -**Deferred: the expanded C# population.** C#'s challenge-tier row is now -rolled out and its core denominator is 29 templates / 58 assertions, but -`reports/codeql-csharp-kernel.json` is one of the nineteen reports -`reports/freeze.json` digest-binds for v0.3.0. Overwriting it would invalidate -a published freeze, so the C# challenge wave left it untouched, exactly as the -Python and JavaScript waves left theirs. Its 32 results remain the frozen -16-template v0.3.0 evidence, and CodeQL's evidence for the expanded C# core -arrives with the v0.4.0 freeze-prep re-run. This is deferral, not absence of -coverage, and the two populations are never compared number-to-number. The -selection code already expects 58; the runner is simply not invoked until the -freeze is re-cut. +**The expanded C# population.** C#'s challenge-tier row is rolled out and its +core denominator is 29 templates / 58 assertions; +`reports/codeql-csharp-kernel.json` was re-run whole for the v0.4.0 freeze, +exactly as the Python and JavaScript kernels were. Split by stratum the +challenge 20/26 is A **3/6**, B **6/8**, C **5/6**, D **6/6**: five false +negatives on the `reflective-invocation`, `computed-property`, +`dispatch-table`, `function-field` and `callback-registration` positives, and +one false positive on the `element-object` negative. The v0.3.0 and v0.4.0 +populations are never compared number-to-number. ## Go kernel @@ -464,35 +480,33 @@ diagnostics) under `reports/raw/codeql-go-kernel/`. SARIF locations are reconciled with the case's `DFB-SINK:` anchor by resolving the declared sink function name and accepting a finding on a line that calls it in the same file. -**Expanded evidence is deferred.** `reports/codeql-go-kernel.json` is one of the -nineteen reports `reports/freeze.json` digest-binds for v0.3.0, so the Go -challenge expansion did not overwrite it: the expanded 58-assertion CodeQL -evidence is pending the v0.4.0 freeze-prep re-run, on this repository's -established re-run-at-freeze pattern, and the deferral is recorded in -`docs/go-kernel.md`. What follows is the valid classic 32-assertion snapshot, -and it describes a different population from the expanded one. - -The checked-in `reports/codeql-go-kernel.json` contains 32 results: 16 `reached` -and 16 `not-reached`, with zero `inconclusive`, `unsupported`, or `runner-error` -outcomes, extracted through go1.26.0. 26 of 32 match the expected polarity; the +`reports/codeql-go-kernel.json` was re-run whole over the expanded population +for the v0.4.0 freeze, on this repository's established re-run-at-freeze +pattern. + +It contains **58 results**: 24 `reached` +and 34 `not-reached`, with zero `inconclusive`, `unsupported`, or `runner-error` +outcomes, extracted through go1.26.0. **45 of 58** match the expected polarity — +26/32 on the classic sixteen, identical case for case to the frozen v0.3.0 +evidence, and 19/26 on the challenge thirteen (A **3/6**, B **5/8**, C **5/6**, +D **6/6**). On the classic stratum the false negatives are the alias-propagation, exception-catch, and expression positives, and the false positives are the array-element, loop-carried, and infeasible-branch negatives. The first five are the same mismatch set the Java and C# kernels show on those templates; the infeasible-branch false positive is Go-specific, and the exception-catch false negative is the capability evidence -the `panic`/`recover` adaptation anticipates. Its configuration hash is +the `panic`/`recover` adaptation anticipates. The seven challenge mismatches are +six false negatives on the `reflective-invocation`, `computed-property`, +`dispatch-table`, `function-field`, `callback-registration` and +`anonymous-implementation` positives, plus one false positive on the +`element-object` negative. Its configuration hash is `56f44b3d983f7ea1dc2fa77a796ac547b01d12535a124f0c9975d3d0b7989161`. ## C and C++ kernels The C++ runner selects exactly the 56 `taint`/`core` cases whose `language` is `cpp` — 32 classic assertions plus the 24 the challenge-tier expansion added; -the C runner selects the 30 `taint`/`core` cases whose `language` is `c` -plus its 2 `language-extension` cases, which are scored on their own scorecard -and never counted in the core denominator. Each analyzes its own query: - -The C++ runner selects exactly the 32 `taint`/`core` cases whose `language` is -`cpp`; the C runner selects the whole C `taint`/`core` population — 30 +the C runner selects the whole C `taint`/`core` population — 30 assertions classically, and **48** now that C's nine applicable challenge templates have rolled out (`docs/challenge-tier.md`) — plus its 2 `language-extension` cases, which are scored on their own scorecard and never @@ -532,37 +546,43 @@ accepting a finding on a line that calls it in the same file; a `.`, `->`, or contract](../../docs/c-kernel.md) and [the C++ kernel contract](../../docs/cpp-kernel.md). -The checked-in `reports/codeql-cpp-kernel.json` contains 32 results: 16 -`reached` and 16 `not-reached`, with zero `inconclusive`, `unsupported`, or -`runner-error` outcomes, and 28 of 32 matching the expected polarity — false -negatives on the alias-propagation and exception-catch positives, false +The checked-in `reports/codeql-cpp-kernel.json` contains **56 results**: 18 +`reached` and 38 `not-reached`, with zero `inconclusive`, `unsupported`, or +`runner-error` outcomes, and **42 of 56** matching the expected polarity — +28/32 on the classic sixteen, identical case for case to the frozen v0.3.0 +evidence, and 14/24 on the challenge twelve (A **2/4**, B **4/8**, C **3/6**, +D **5/6**). The classic mismatches are false +negatives on the alias-propagation and exception-catch positives and false positives on the array-element and loop-carried negatives. Its configuration hash is `8873a63a5898c8b6b10dc24a9fbf2fae3ed5a088faf024524b0bae50f0fc4cc0`. -That snapshot is the **classic 32-assertion population only**. It is one of -the nineteen reports `reports/freeze.json` digest-binds for v0.3.0, so the C++ -challenge wave did not re-run it: **the expanded CodeQL C++ evidence is pending -the v0.4.0 freeze-prep re-run**, on the repository's established -re-run-at-freeze pattern. The selector already expects the full 56; deferral is -not absence of coverage, and a 32-assertion score is never compared with a -56-assertion one. - -**Expanded C evidence is deferred.** `reports/codeql-c-kernel.json` is one of -the nineteen reports `reports/freeze.json` digest-binds for v0.3.0, so the C -challenge expansion did not overwrite it: the expanded 48-assertion CodeQL -evidence for C is pending the v0.4.0 freeze-prep re-run, on this repository's -established re-run-at-freeze pattern, and the deferral is recorded in -`docs/c-kernel.md`. What follows is the valid classic 30-assertion snapshot, -and it describes a different population from the expanded one. - -The checked-in `reports/codeql-c-kernel.json` contains 32 results with the same -clean execution profile. Of the 30 core assertions, 16 are `reached` and 14 are -`not-reached`, with 27 of 30 matching the expected polarity — the same +Every one of the ten challenge mismatches is a **false negative on a positive** +— `computed-property`, `dispatch-table`, `closure-capture`, `function-field`, +`callback-registration`, `anonymous-implementation`, `map-iteration`, +`nested-access-path`, `element-object`, and the two-level context positive — with +no false positive anywhere in the tier. That is a uniformly under-approximating +character on this population, not half a score, and a 32-assertion score is +never compared with a 56-assertion one. `reflective-invocation` is inapplicable +to C++ and reduces only its denominator. + +`reports/codeql-c-kernel.json` was likewise re-run whole over C's expanded +48-assertion population for the v0.4.0 freeze, on this repository's established +re-run-at-freeze pattern. + +It contains **50 results** with the same +clean execution profile. Of the 48 core assertions, 23 are `reached` and 25 are +`not-reached`, with **41 of 48** matching the expected polarity — 27/30 on the +classic fifteen, identical case for case to the frozen v0.3.0 evidence, and +14/18 on C's nine applicable challenge templates (A **1/2**, B **2/4**, C +**5/6**, D **6/6**). The classic mismatches are the same alias-propagation false negative and array-element and loop-carried false -positives, with no exception-catch cell in the C population. Both +positives, with no exception-catch cell in the C population; the challenge +mismatches are false negatives on the `dispatch-table`, `function-field` and +`callback-registration` positives plus one false positive on the +`element-object` negative. Both `language-extension` cases are `reached`, matching their positive polarity, and -are scored on their own scorecard rather than in the 30-assertion denominator. +are scored on their own scorecard rather than in the 48-assertion denominator. Its configuration hash is `719415b9134dfd43390ffdb76eef45f7ed022f907f22913226c22f93277b62f8`. @@ -620,31 +640,39 @@ gets its own callsite rule because Rust reaches a member through `.` and a path through `::`, and neither is a call of the free sink function the anchor declares. -**Expanded evidence is deferred.** `reports/codeql-rust-kernel.json` is one of -the nineteen reports `reports/freeze.json` digest-binds for v0.3.0, so the Rust -challenge expansion did not overwrite it: the expanded 54-assertion CodeQL -evidence is pending the v0.4.0 freeze-prep re-run, on this repository's -established re-run-at-freeze pattern, and the deferral is recorded in -`docs/rust-kernel.md`. What follows is the valid classic 30-assertion snapshot, -and it describes a different population from the expanded one. - -The checked-in `reports/codeql-rust-kernel.json` contains 32 results. Its 30 -core assertions are 17 `reached` and 13 `not-reached`, with zero `inconclusive`, -`unsupported`, or `runner-error` outcomes; 28 of 30 match the expected polarity. -All 15 positives are `reached`, so there are no false negatives; the two -mismatches are the array-element and loop-carried negatives, which are false +`reports/codeql-rust-kernel.json` was re-run whole over the expanded +54-assertion population for the v0.4.0 freeze, on this repository's established +re-run-at-freeze pattern. + +It contains **56 results**. Its 54 +core assertions are 21 `reached` and 33 `not-reached`, with zero `inconclusive`, +`unsupported`, or `runner-error` outcomes; **44 of 54** match the expected +polarity — 28/30 on the classic fifteen, identical case for case to the frozen +v0.3.0 evidence, and 16/24 on Rust's twelve applicable challenge templates +(A **2/4**, B **4/8**, C **4/6**, D **6/6**). +All 15 classic positives are `reached`, so there is no classic false negative; +the two classic mismatches are the array-element and loop-carried negatives, +which are false positives here as they are for the Java, Kotlin, and C# kernels against this build. The alias-propagation and expression positives that are false negatives in every other CodeQL kernel are `reached` here. +On the challenge tier the character inverts: all eight mismatches are **false +negatives on positives** — `computed-property`, `dispatch-table`, +`closure-capture`, `function-field`, `callback-registration`, +`anonymous-implementation`, `map-iteration`, and `element-object` — with no +false positive anywhere in the tier, and every stratum-D cell correct. +`reflective-invocation` is inapplicable to a language with no run-time +reflection and reduces only Rust's denominator. + The two `language-extension` assertions are reported separately and never enter that denominator: both are `not-reached`, so `dfb-taint-rust-result-error-propagation-positive` is a false negative — the pinned preview analyzer does not carry the value through the `Result` error variant and `?` across the call boundary — and the negative is correct. -All 32 raw outputs are SARIF files under `reports/raw/codeql-rust-kernel/` with -zero error files. Per-case wall clock ran 50.8 s to 98.4 s, about 40 minutes for +All 56 raw outputs are SARIF files under `reports/raw/codeql-rust-kernel/` with +zero error files. Per-case wall clock ran 40.9 s to 43.2 s, about 39 minutes for the population, because every case re-extracts the Cargo workspace's library sources. Its configuration hash is `cc2c728b66e0c273545e3531a672c0987473f3830f5df80b0839f5d04c33600b`. @@ -693,15 +721,15 @@ prefix is not a call of the free sink method the anchor declares. The checked-in `reports/codeql-ruby-kernel.json` contains 58 results: 22 `reached` and 36 `not-reached`, with zero `inconclusive`, `unsupported`, or -`runner-error` outcomes, over 468 s wall clock and 5.7 s to 14.5 s per case. +`runner-error` outcomes, over 243 s wall clock and 4.1 s to 4.4 s per case. **49 of 58** match the expected polarity — 29/32 on the classic sixteen templates and 20/26 on the challenge thirteen. -Ruby is the one CodeQL kernel whose report is **not** freeze-bound: the Ruby -kernel landed after v0.3.0, so `reports/freeze.json` binds the other ten CodeQL -kernel reports and not this one. It could therefore be re-run whole over the -expanded population in the Ruby challenge wave, where the other ten languages' -expanded CodeQL evidence is deferred to the v0.4.0 freeze-prep re-run. +Ruby was the one CodeQL kernel the v0.3.0 freeze did not bind: the Ruby kernel +landed after it, so it could be re-run whole over the expanded population in +the Ruby challenge wave while the other ten languages waited. The v0.4.0 freeze +binds all eleven CodeQL kernel reports, every one of them re-run over its +expanded population at one fixture revision, so Ruby is no longer an exception. The classic mismatch set is unchanged case for case: the false negatives are the alias-propagation and exception-catch positives and the false positive is the @@ -720,34 +748,53 @@ neither the query nor the pack moved, only the population. ## Retained v2.26.3 snapshot -The checked-in report uses CodeQL CLI v2.26.3 build -`7d097a43199effe04ecd9c6bd3ad9bb02a45b3d7` with -`codeql/java-all@9.2.3`. Of 32 assertions, 15 are `reached` and 17 are -`not-reached`; 27 match their expected polarity. The expression, alias, and +Every CodeQL report on this tree uses CodeQL CLI v2.26.3 build +`7d097a43199effe04ecd9c6bd3ad9bb02a45b3d7`, at the one fixture revision +`sha256:13a11ff48f26dba889f76aeb9ef60213a129abe5ebcfcb966da3a2418c12807e` the +v0.4.0 freeze binds, and every kernel was re-run whole over its expanded +population. + +The Java kernel uses `codeql/java-all@9.2.3`. Of its **58** assertions, 29 are +`reached` and 29 are `not-reached`, with **48/58** matching their expected +polarity: 27/32 on the classic sixteen and 21/26 on the challenge thirteen +(A **3/6**, B **7/8**, C **5/6**, D **6/6**). On the classic stratum the +expression, alias, and exception positives are false negatives, while the array-element and loop-kill -negatives are false positives. Each case uses an isolated cold database; no +negatives are false positives; on the challenge stratum the +`reflective-invocation` and `computed-property` positives are false negatives +and the `dispatch-table`, `function-field` and `element-object` negatives are +false positives. Its configuration hash is +`eedf28b140e6aaf2c27cac6369ee552803cbc7b7674abd70583e3e962e1ef8b6`. Each case +uses an isolated cold database; no database or compiled fixture is reused across the pair. The adapter removes temporary databases and workspaces after retaining SARIF. -The retained Kotlin snapshot uses the same CodeQL CLI v2.26.3 build -`7d097a43199effe04ecd9c6bd3ad9bb02a45b3d7` with `codeql/java-all@9.2.3`, and -traced Kotlin extraction through kotlinc-jvm 2.4.10. All 32 Kotlin assertions -executed with ordinary reached/not-reached outcomes: 15 are `reached` and 17 -are `not-reached`, with 27/32 matching the expected polarity and zero special -or error outcomes. The false negatives are the expression, alias-propagation, +The Kotlin snapshot uses the same build with `codeql/java-all@9.2.3`, and +traced Kotlin extraction through kotlinc-jvm 2.4.10. All **58** Kotlin +assertions +executed with ordinary reached/not-reached outcomes: 25 are `reached` and 33 +are `not-reached`, with **46/58** matching the expected polarity and zero +special +or error outcomes — 27/32 on the classic sixteen, identical case for case to +the frozen v0.3.0 evidence, and 19/26 on the challenge thirteen (A **3/6**, +B **5/8**, C **5/6**, D **6/6**). The classic false negatives are the +expression, alias-propagation, and exception-catch positives; the array-element and loop-carried negatives are false positives — the same five mismatches the Java snapshot shows against this -build. Its configuration hash is -`25b92ad6190d65fd76c67da51c3ec0d638cea7699e976941c027a48700b9096e`. It covers -Kotlin's classic 32-assertion population only; the 26 challenge assertions -added since are not in it, and their CodeQL evidence is deferred to the v0.4.0 -re-run. - -The retained Python snapshot uses the same CodeQL CLI v2.26.3 build -`7d097a43199effe04ecd9c6bd3ad9bb02a45b3d7` with `codeql/python-all@7.2.3`. -All 32 Python assertions executed with ordinary reached/not-reached outcomes: -14 are `reached` and 18 are `not-reached`, with 28/32 matching the expected -polarity. The false negatives are the alias-propagation positive, +build. The challenge mismatches are false negatives on the +`reflective-invocation`, `computed-property`, `dispatch-table`, +`callback-registration` and `anonymous-implementation` positives and false +positives on the `function-field` and `element-object` negatives. Its +configuration hash is +`25b92ad6190d65fd76c67da51c3ec0d638cea7699e976941c027a48700b9096e`. + +The Python snapshot uses the same build with `codeql/python-all@7.2.3`. +All **58** Python assertions executed with ordinary reached/not-reached +outcomes: +23 are `reached` and 35 are `not-reached`, with **48/58** matching the expected +polarity — 28/32 classic and 20/26 challenge, detailed under [the Python +kernel](#python-kernel) above. The classic false negatives are the +alias-propagation positive, array-element positive, and exception-catch positive; the loop-carried negative is a false positive. No special or error outcomes occurred. Every Python case uses an isolated cold database, with no database or compiled fixture reused diff --git a/adapters/joern/README.md b/adapters/joern/README.md index 214f20e..15aa943 100644 --- a/adapters/joern/README.md +++ b/adapters/joern/README.md @@ -242,25 +242,15 @@ zero error documents, zero `inconclusive`, `unsupported`, or `runner-error` outcomes. None of the six still stands at the pre-expansion fixture revision -`sha256:aee59a14f96633cf5798df6d211525ea0d10748800ba9c9ac0a3787406bd19ea`. -The Python, JavaScript, Java, Rust, PHP, and Ruby kernels were each re-run whole -after that -language's challenge-tier row was rolled out, and each carries the expanded -corpus revision current when it ran — -`sha256:3e7a8de5e1eefb18e8166af0ccdf309bccf1d5c26026893a4513f1943926ab1f` for -Python, -`sha256:64ef139f452fd296bb26463bc552e5e5998ca4bb4584d45565d858424814bde9` for -JavaScript, -`sha256:f476894a41d283e3bcaaf5188ee08abe7886ce8e3919257403b0aa853ef718e2` for -Java, -`sha256:88ad35289ae465278b95fd436532132118a6b6aa681adb3d266d67766c8770c5` for -Rust, -`sha256:f74647fe824ca9f6900c48aa9d403f0e9f59230e4193e0b02bd65e29a9e4e660` for -PHP, and -`sha256:020d0d8f79360af6e74064a692e2d65ffa31cd97f9971f9dad8bec065d862043` for -Ruby, the last wave to land. `fixture_revision` digests the whole case corpus, -so each wave's fixtures -moved it for every run after it. Reports at different fixture revisions are not +`sha256:aee59a14f96633cf5798df6d211525ea0d10748800ba9c9ac0a3787406bd19ea`, and +none stands at an intermediate wave revision either. The Python, JavaScript, +Java, Rust, PHP, and Ruby kernels were all re-run whole for the v0.4.0 freeze +after every challenge-tier row had rolled out, so all six carry the single +revision +`sha256:13a11ff48f26dba889f76aeb9ef60213a129abe5ebcfcb966da3a2418c12807e`. +`fixture_revision` digests the whole case corpus, so each wave's fixtures +moved it for every run after it; re-running the whole adapter at one revision +is what removes that skew. Reports at different fixture revisions are not pooled, and each language's expanded assertions are a different population from the 32 (30 for Rust) it reported in v0.3.0, not a movement within one. @@ -286,7 +276,7 @@ expansion introduced no drift — and 16/24 on its challenge twelve**. templates: the sixteen v0.3.0 templates plus the thirteen preregistered challenge templates ([the challenge tier](../../docs/challenge-tier.md)). Each report was re-run whole — a whole-population replacement, not an append — and -each carries the expanded corpus revision current when it ran; no Joern kernel +all six carry the one v0.4.0 fixture revision; no Joern kernel is left at `sha256:aee59a14f96633cf5798df6d211525ea0d10748800ba9c9ac0a3787406bd19ea`. Split by stratum, JavaScript is **26/32 on the classic sixteen — identical case for case to its v0.3.0 snapshot, so the expansion introduced no drift — and diff --git a/adapters/semgrep/README.md b/adapters/semgrep/README.md index 9c986dc..fdfae95 100644 --- a/adapters/semgrep/README.md +++ b/adapters/semgrep/README.md @@ -390,29 +390,19 @@ does not require the marker's own line. ## Observed results Semgrep CE 1.174.0. No kernel is left at the pre-expansion fixture revision -`sha256:aee59a14f96633cf5798df6d211525ea0d10748800ba9c9ac0a3787406bd19ea`. All +`sha256:aee59a14f96633cf5798df6d211525ea0d10748800ba9c9ac0a3787406bd19ea`, and +none is left at an intermediate wave revision either. All eleven — Python, JavaScript, Java, TypeScript, Kotlin, Go, C++, C, Rust, PHP, -and Ruby — were each re-run whole after that language's challenge-tier row was -rolled out and carry the expanded corpus revision current when each ran — - -| Kernel | `fixture_revision` | -| --- | --- | -| Python | `sha256:3e7a8de5e1eefb18e8166af0ccdf309bccf1d5c26026893a4513f1943926ab1f` | -| JavaScript | `sha256:61c06a78b95b86764d3c220cfefd7af37373db64b15ae0b76c6ebf924217ab2e` | -| Java | `sha256:cf571f29e434030019d5e8f8361319b0bb3b4d6c4c752bd65860e07bfcf26bbc` | -| TypeScript | `sha256:2c906faeb98b48d1aba7da7bc80a78c4084051b84efac6ac3a1b74f54c843fd2` | -| Kotlin | `sha256:7ac23321e5d0974ed9087b9642ee3c88b3f3af014ba507330131da30fbb9b4d7` | -| Go | `sha256:7f37b99ddab7764a8536112c09ff7c8d77e0b02f7786abde65dfbaf3654d9949` | -| C++ | `sha256:a1570fc74526f0088488e3fba0941a7da47244635d7ceecf6787f1f76200b4ee` | -| C | `sha256:75f631ca05df2609055972622faaf3946331f7537140b08ba7ec6648bd0e077c` | -| Rust | `sha256:88ad35289ae465278b95fd436532132118a6b6aa681adb3d266d67766c8770c5` | -| PHP | `sha256:f74647fe824ca9f6900c48aa9d403f0e9f59230e4193e0b02bd65e29a9e4e660` | -| Ruby | `sha256:020d0d8f79360af6e74064a692e2d65ffa31cd97f9971f9dad8bec065d862043` | +and Ruby — were re-run whole for the v0.4.0 freeze, after every +challenge-tier row had rolled out, so all eleven carry the single revision +`sha256:13a11ff48f26dba889f76aeb9ef60213a129abe5ebcfcb966da3a2418c12807e`. `fixture_revision` digests the whole case corpus, so each wave's fixtures moved it for every run after it, and reports at different fixture revisions are not -pooled. The configuration hash is unchanged across all eleven: no rule file was -touched. +pooled; re-running the whole adapter at one revision is what removes that skew. +The configuration hash +`865d0bd2989f9ddd0b90f2d6675584e86706b109a033d4a1ac00bd21a617b100` is the same +across all eleven: no rule file was touched. All eleven kernels ran. 622 assertions: 154 executed against Semgrep, 468 excluded by declared capability. Zero `inconclusive` and zero `runner-error` diff --git a/docs/astro.config.mjs b/docs/astro.config.mjs index f672d9a..5bab1a8 100644 --- a/docs/astro.config.mjs +++ b/docs/astro.config.mjs @@ -8,7 +8,7 @@ export default defineConfig({ base: '/dataflowbench', redirects: { // Explicit current-snapshot pointer alongside versioned snapshot URLs. - '/current': '/dataflowbench/snapshots/v0-3-0/', + '/current': '/dataflowbench/snapshots/v0-4-0/', }, integrations: [ starlight({ @@ -34,7 +34,18 @@ export default defineConfig({ items: [ { label: 'All snapshots', slug: 'snapshots' }, { - label: 'v0.3.0 (current)', + label: 'v0.4.0 (current)', + items: [ + { label: 'Snapshot overview', slug: 'snapshots/v0-4-0' }, + { label: 'Analyzers', slug: 'snapshots/v0-4-0/analyzers' }, + { label: 'Languages', slug: 'snapshots/v0-4-0/languages' }, + { label: 'Semantic templates', slug: 'snapshots/v0-4-0/templates' }, + { label: 'Case evidence', slug: 'snapshots/v0-4-0/evidence' }, + ], + }, + { + label: 'v0.3.0 (archived)', + collapsed: true, items: [ { label: 'Snapshot overview', slug: 'snapshots/v0-3-0' }, { label: 'Analyzers', slug: 'snapshots/v0-3-0/analyzers' }, diff --git a/docs/milestones.md b/docs/milestones.md index cce23d5..831358a 100644 --- a/docs/milestones.md +++ b/docs/milestones.md @@ -162,6 +162,30 @@ v0.4.0 cores are different populations and are never compared number-to-number; the frozen v0.3.0 evidence is unaffected because freeze validation is manifest-scoped. +### Current expanded-breadth status + +Release v0.4.0 is that first expanded-breadth freeze: 744 cases and 42 bound +reports at one fixture revision, with all thirteen kernels carrying their +expanded core denominators and four analyzer populations — Bifrost v0.10.6 +(build `18d09c57`) over the pinned 118-case breadth slice plus thirteen +kernels, CodeQL 2.26.3 over eleven, Joern 4.0.610 over six, and Semgrep CE +1.174.0 over eleven. Coverage differs per analyzer; a language with no report +for an analyzer is coverage, not a score. + +The tier did what it was preregistered to do: no analyzer answers a whole +expanded core correctly in any language, so the core no longer saturates. +Bifrost decides 115 of the 118 breadth-slice cases and 227 of its 738 core +assertions, 222 of those decisions correct, declining the rest as +`inconclusive`; its `element-object` `internal_invariant` failures and its +fully inconclusive Ruby kernel are published as retained execution and +capability coverage and tracked upstream. CodeQL produces a definitive answer +for all 626 of its bound assertions, 509 correct, with dynamic dispatch its +systematic miss. Joern answers all 344 of its assertions, 270 correct, and its +depth-6 behavior tracks the call-depth bound the preregistration verified in +advance. Semgrep CE scores the 154-assertion intraprocedural partition of its +populations and declines the other 468 by declared capability. See +[`releases/v0.4.0.md`](releases/v0.4.0.md) for the bound evidence. + ## M3: taint modeling Add balanced categories for sources and sinks, propagators, sanitizers, opaque diff --git a/docs/releases/v0.4.0.md b/docs/releases/v0.4.0.md new file mode 100644 index 0000000..aaab280 --- /dev/null +++ b/docs/releases/v0.4.0.md @@ -0,0 +1,288 @@ +# DataFlowBench v0.4.0 + +Fourth immutable release snapshot, and the first **expanded-breadth** release: +all thirteen core kernels now carry the preregistered challenge-tier templates +of [the challenge-tier document](../challenge-tier.md), and four analyzers — +Bifrost v0.10.6, CodeQL 2.26.3, Joern 4.0.610, and Semgrep CE 1.174.0 — are +bound at one fixture revision under the `freeze/v1` contract with claim scope +`release`. + +## Freeze identity + +- Freeze ID (manifest SHA-256): + `91b0008a546e6b782c1b790f174a71ce44e60039239797674eb49ebc6ac6c366` +- Manifest: `reports/freeze.json` +- Benchmark revision: `306211a` (tag `v0.4.0`) +- Fixture revision: + `sha256:13a11ff48f26dba889f76aeb9ef60213a129abe5ebcfcb966da3a2418c12807e` +- Case schema v2, normalized result schema v1 +- 744 frozen cases, 42 bound reports, 2452 scored case results + +## What "expanded breadth" means here + +The thirteen challenge templates were preregistered before any challenge +fixture existed, and the preregistration fixed the population decision in +advance: they carry `score_tier: "core"` and **fold into each language's core +kernel**, with no new score tier. Each language's core denominator is therefore +its sixteen-template core (fifteen for C and Rust) plus its applicable +challenge templates: 29 templates / 58 assertions for ten languages, 28 / 56 +for C++, 27 / 54 for Rust, and 24 / 48 for C. + +The canonical construction of `dfb-template-chal-context-pair-depth2` follows +**Amendment A1** (2026-08-25) of the preregistration: `helper` returns its +argument and the caller sinks the result of the selected two-deep path. The +amendment was recorded before any analyzer ran against either implementing +fixture and invalidates no published freeze. + +The v0.3.0 sixteen-template core and this expanded core are **different +populations of the same name**. No number in this document is compared with a +v0.3.0-era number, and the frozen v0.3.0 evidence remains valid and unamended. + +## Bound evidence + +| Normalized report | SHA-256 | Analyzer | Cases | +| --- | --- | --- | --- | +| `reports/bifrost-c-kernel.json` | `e334de9b8752daf1b0ed67bc8403232449c10805c4817be3975b2ca62643ba03` | Bifrost 0.10.6, build `18d09c57d1e5044dec49acac7635d3255ea8e89c` | 50 | +| `reports/bifrost-cpp-kernel.json` | `12353998cacf0bf7c3e74574961d0eaec9204da633a6cdf87d2c36526bd71e27` | Bifrost 0.10.6, build `18d09c57d1e5044dec49acac7635d3255ea8e89c` | 56 | +| `reports/bifrost-csharp-kernel.json` | `a004277d218cf4793afed987394c0d6285ecd190622e23558368f2ba0ec2eeb2` | Bifrost 0.10.6, build `18d09c57d1e5044dec49acac7635d3255ea8e89c` | 58 | +| `reports/bifrost-go-kernel.json` | `a3597db4e3ccdb2489b720eca08173100cb3220ee5db31743bf0d7a64708c46d` | Bifrost 0.10.6, build `18d09c57d1e5044dec49acac7635d3255ea8e89c` | 58 | +| `reports/bifrost-java-kernel.json` | `d8ad10a3cdca6d0be1e207dcbdf2b276dd46caeeb2f98d43352f003082113b0d` | Bifrost 0.10.6, build `18d09c57d1e5044dec49acac7635d3255ea8e89c` | 58 | +| `reports/bifrost-javascript-kernel.json` | `9a4f3c8c5a319dff7e5d5dd5f159329c2580997d742ae8f75c86176a99a1096f` | Bifrost 0.10.6, build `18d09c57d1e5044dec49acac7635d3255ea8e89c` | 58 | +| `reports/bifrost-kotlin-kernel.json` | `dad5d772414d41644b3d8892fd3495f86974054b90a8c678d4e63364b17678f0` | Bifrost 0.10.6, build `18d09c57d1e5044dec49acac7635d3255ea8e89c` | 58 | +| `reports/bifrost-php-kernel.json` | `5881b8e17f332d344f1539b79d6f1833abc1dde245302fd9ebdacb84cc14169a` | Bifrost 0.10.6, build `18d09c57d1e5044dec49acac7635d3255ea8e89c` | 58 | +| `reports/bifrost-python-kernel.json` | `87f9a195df465554734796412867f5b32721200ee74863fc9d1ff3b13d6ca3fe` | Bifrost 0.10.6, build `18d09c57d1e5044dec49acac7635d3255ea8e89c` | 58 | +| `reports/bifrost-ruby-kernel.json` | `7af673389c1146294684f2154fb8ec13c39a587a21e8b381d46119f51eb49aa1` | Bifrost 0.10.6, build `18d09c57d1e5044dec49acac7635d3255ea8e89c` | 58 | +| `reports/bifrost-rust-kernel.json` | `2ad1e552b00c7c6370b26d948e2e5e850f38adeff27a87b6b1f72aefa15f4119` | Bifrost 0.10.6, build `18d09c57d1e5044dec49acac7635d3255ea8e89c` | 56 | +| `reports/bifrost-scala-kernel.json` | `11c7c6462f6d7cae9d9042540014294d66909c9566d02d68c7cc2513ab5298dc` | Bifrost 0.10.6, build `18d09c57d1e5044dec49acac7635d3255ea8e89c` | 58 | +| `reports/bifrost-smoke.json` | `bfbd71c8ea921f71eacae6983ac45361edb0264be0ef4ded17cb17449dc880f9` | Bifrost 0.10.6, build `18d09c57d1e5044dec49acac7635d3255ea8e89c` | 118 | +| `reports/bifrost-typescript-kernel.json` | `6d1ec2fce985b22def1de1d32ae49049ea5be0fb6b52a8ae024adbdac9110e1c` | Bifrost 0.10.6, build `18d09c57d1e5044dec49acac7635d3255ea8e89c` | 58 | +| `reports/codeql-c-kernel.json` | `0b6c59ac6e4435e049a45972d297d665b55eba07fefae98535930a01543b7f0c` | CodeQL 2.26.3, build `codeql-cli:7d097a43199effe04ecd9c6bd3ad9bb02a45b3d7` | 50 | +| `reports/codeql-cpp-kernel.json` | `6e772fe6740133ee4a0b20a9683d145b832f3ede088c535936ecdaf2f6a802ff` | CodeQL 2.26.3, build `codeql-cli:7d097a43199effe04ecd9c6bd3ad9bb02a45b3d7` | 56 | +| `reports/codeql-csharp-kernel.json` | `3436137a9aa293bbf0efb62e263a90b4c16e47ede83cb45602cf7c229fb31fed` | CodeQL 2.26.3, build `codeql-cli:7d097a43199effe04ecd9c6bd3ad9bb02a45b3d7` | 58 | +| `reports/codeql-go-kernel.json` | `7812a935ee53a26fab3a7f3b1d74c169e5ae98a4c3d1e0c6b02a09496612aa93` | CodeQL 2.26.3, build `codeql-cli:7d097a43199effe04ecd9c6bd3ad9bb02a45b3d7` | 58 | +| `reports/codeql-java-kernel.json` | `bad0469701d0f3c45825cc4ee8d0448bdbec40e9006cf78935112e09e77e08db` | CodeQL 2.26.3, build `codeql-cli:7d097a43199effe04ecd9c6bd3ad9bb02a45b3d7` | 58 | +| `reports/codeql-javascript-kernel.json` | `98d8064493ddfbbee98e63f855cba6c6dc3eb7ae85ee6405ad0e0ab0167fc045` | CodeQL 2.26.3, build `codeql-cli:7d097a43199effe04ecd9c6bd3ad9bb02a45b3d7` | 58 | +| `reports/codeql-kotlin-kernel.json` | `bada2d8ddf781ff12b569fd240f4b94014f647d71f5fb1d653308d5a5995858f` | CodeQL 2.26.3, build `codeql-cli:7d097a43199effe04ecd9c6bd3ad9bb02a45b3d7` | 58 | +| `reports/codeql-python-kernel.json` | `8cad62ca0ae9206f172ab9cdbdbe95a8dc72d5af62d9624a97e7afb491eafea7` | CodeQL 2.26.3, build `codeql-cli:7d097a43199effe04ecd9c6bd3ad9bb02a45b3d7` | 58 | +| `reports/codeql-ruby-kernel.json` | `09358d58fb97df1bc2024545ec56438d23989ad4c3a2c13a4a0e7c407a450858` | CodeQL 2.26.3, build `codeql-cli:7d097a43199effe04ecd9c6bd3ad9bb02a45b3d7` | 58 | +| `reports/codeql-rust-kernel.json` | `30882a1c1aab9919ff484a89166f10ded6050e9b3ffef37f1568c6cce605195f` | CodeQL 2.26.3, build `codeql-cli:7d097a43199effe04ecd9c6bd3ad9bb02a45b3d7` | 56 | +| `reports/codeql-typescript-kernel.json` | `ffe51480b9c3ab103e67484e5a86e4ba1c35c15d2ffaca6ff11831d7694a501e` | CodeQL 2.26.3, build `codeql-cli:7d097a43199effe04ecd9c6bd3ad9bb02a45b3d7` | 58 | +| `reports/joern-java-kernel.json` | `7e3c4cb6adbe7325bb5f9f12a62f0d0176faacd5825ce5c9f32c7f67a03c1e85` | Joern 4.0.610, build `joern-cli:4.0.610` | 58 | +| `reports/joern-javascript-kernel.json` | `d794fbb8d72cb5d836ffe45d8caf421113931b9bc376424c62b57f753b820dfb` | Joern 4.0.610, build `joern-cli:4.0.610` | 58 | +| `reports/joern-php-kernel.json` | `1638b81a764de42658700e1f3deeeebf808055b2582db4e21c65c8ed0b7ddab5` | Joern 4.0.610, build `joern-cli:4.0.610` | 58 | +| `reports/joern-python-kernel.json` | `0facb3aa7e4e2855f04c356c103beb9f1a84b4879c41f7b826747d7d25c637f4` | Joern 4.0.610, build `joern-cli:4.0.610` | 58 | +| `reports/joern-ruby-kernel.json` | `0d1bd231bae8ffec048e0cd7cbe83d128e7cf3c0cb49a3a1faafea3a10960b3c` | Joern 4.0.610, build `joern-cli:4.0.610` | 58 | +| `reports/joern-rust-kernel.json` | `cf346a6c0bda2dc3b6cd6af1f24bc2c4b895fdea0cce7c85f4b2c7998ff54351` | Joern 4.0.610, build `joern-cli:4.0.610` | 54 | +| `reports/semgrep-c-kernel.json` | `a75b5f37004166d20de264ee95ba7c6f4905ab0cc9e82c47aeed69ee35f4e4c5` | Semgrep CE 1.174.0, build `semgrep-oss:1.174.0` | 48 | +| `reports/semgrep-cpp-kernel.json` | `ef9b23c9a1fa0764ffbe0db62e80c9bd5986374a24ecf7db702b221661d5445d` | Semgrep CE 1.174.0, build `semgrep-oss:1.174.0` | 56 | +| `reports/semgrep-go-kernel.json` | `bab1bcdeef91fc8fbaece22857db61efac141793573046d13a06ee9921889426` | Semgrep CE 1.174.0, build `semgrep-oss:1.174.0` | 58 | +| `reports/semgrep-java-kernel.json` | `60ccdc90635606d8eaa150d0823c626ea5b3cf69963f6575f324fb020287e2bb` | Semgrep CE 1.174.0, build `semgrep-oss:1.174.0` | 58 | +| `reports/semgrep-javascript-kernel.json` | `df957a71497cec253003f0c0a1941d9bc13a8424d0d9bb9058e079f11259d0c9` | Semgrep CE 1.174.0, build `semgrep-oss:1.174.0` | 58 | +| `reports/semgrep-kotlin-kernel.json` | `e1e23bfcf109af1e7345649be232de413e7f1e655513947ecb0224d9bbaefba6` | Semgrep CE 1.174.0, build `semgrep-oss:1.174.0` | 58 | +| `reports/semgrep-php-kernel.json` | `22bc4cf3c0d3192edf1391eef4e2c9e0e4b3e69fbe9769737df397dd8c58e695` | Semgrep CE 1.174.0, build `semgrep-oss:1.174.0` | 58 | +| `reports/semgrep-python-kernel.json` | `9eb2778656f6f46c95181fca43832e0c1f8ed03ce7e85b3e7ded51a80bc45137` | Semgrep CE 1.174.0, build `semgrep-oss:1.174.0` | 58 | +| `reports/semgrep-ruby-kernel.json` | `6f3b58ae53d16816b90076d449361e75728b20af8f9866c0e1f37ce12874b279` | Semgrep CE 1.174.0, build `semgrep-oss:1.174.0` | 58 | +| `reports/semgrep-rust-kernel.json` | `1739e7212692082488ac9ef6f3661a46dce3f9536f708b41113cb0b7574a3e89` | Semgrep CE 1.174.0, build `semgrep-oss:1.174.0` | 54 | +| `reports/semgrep-typescript-kernel.json` | `50a0635d83a7d24c6ae71b12b1e0d066ff1262b35d1934d5bc3827d8af97fba3` | Semgrep CE 1.174.0, build `semgrep-oss:1.174.0` | 58 | + +Every report uses the `benchmark-controlled` model profile on the taint track +and remains its own scorecard. The Bifrost smoke population is the pinned +118-case breadth slice — challenge templates are excluded from the smoke +selection outright, so the smoke population did not grow — and every other +report is a single-language kernel population. The C and Rust kernel reports +each carry two `language-extension` cases in addition to their core tier; +those stay on their own tier and are never folded into a core denominator. +Raw evidence for every result is retained under `reports/raw/` and +digest-bound in the manifest. + +## Results + +Generated pages live in `results/` and derive exclusively from the frozen +manifest; see `results/index.md`. Incomplete outcomes (`inconclusive`, +`unsupported`, `runner-error`) are capability and execution coverage and are +never counted as clean negatives. + +Correct decisions (true positives plus true negatives) on each kernel's `core` +tier, with incomplete outcomes shown beside them. Each language is its own +population with its own denominator, and each analyzer column is read +independently: DataFlowBench publishes no combined leaderboard, and cores of +different sizes are never pooled. `n/a` means the analyzer has no report for +that kernel in this freeze — no extractor, no frontend, or no adapter — which +is coverage, not a score. + +| Kernel core | Bifrost 0.10.6 | CodeQL 2.26.3 | Joern 4.0.610 | Semgrep CE 1.174.0 | +| --- | --- | --- | --- | --- | +| Java (29 templates, 58 assertions) | 37/58 (18 inc, 2 err) | 48/58 | 47/58 | 12/58 (44 uns) | +| JavaScript (29 templates, 58 assertions) | 36/58 (20 inc, 2 err) | 48/58 | 44/58 | 12/58 (44 uns) | +| TypeScript (29 templates, 58 assertions) | 34/58 (22 inc, 2 err) | 48/58 | n/a | 12/58 (44 uns) | +| Python (29 templates, 58 assertions) | 36/58 (20 inc, 2 err) | 48/58 | 48/58 | 12/58 (44 uns) | +| Kotlin (29 templates, 58 assertions) | 25/58 (28 inc, 2 err) | 46/58 | n/a | 12/58 (44 uns) | +| Scala (29 templates, 58 assertions) | 10/58 (48 inc) | n/a | n/a | n/a | +| C# (29 templates, 58 assertions) | 3/58 (55 inc) | 47/58 | n/a | n/a | +| Go (29 templates, 58 assertions) | 14/58 (44 inc) | 45/58 | n/a | 12/58 (44 uns) | +| PHP (29 templates, 58 assertions) | 21/58 (36 inc) | n/a | 48/58 | 12/58 (44 uns) | +| Ruby (29 templates, 58 assertions) | 0/58 (58 inc) | 49/58 | 40/58 | 12/58 (44 uns) | +| C++ (28 templates, 56 assertions) | 2/56 (54 inc) | 42/56 | n/a | 12/56 (42 uns) | +| C (24 templates, 48 assertions) | 2/48 (46 inc) | 41/48 | n/a | 12/48 (34 uns) | +| Rust (27 templates, 54 assertions) | 2/54 (40 inc, 12 err) | 44/54 | 43/54 | 12/54 (40 uns) | + +Nobody is perfect on the expanded kernels. The saturation the preregistration +set out to end — a top scorer answering every question correctly — is gone: no +analyzer answers a whole expanded core correctly in any of the thirteen +languages, and no column above reaches its own denominator. + +The challenge templates fold into those core denominators, but they stay +individually visible as a stratum, exactly as the preregistration requires. +The same populations, restricted to the challenge templates only (13 templates +/ 26 assertions, 12 / 24 for C++ and Rust, 9 / 18 for C): + +| Kernel core | Bifrost 0.10.6 | CodeQL 2.26.3 | Joern 4.0.610 | Semgrep CE 1.174.0 | +| --- | --- | --- | --- | --- | +| Java (13 templates, 26 assertions) | 6/26 (18 inc, 2 err) | 21/26 | 19/26 | 0/26 (26 uns) | +| JavaScript (13 templates, 26 assertions) | 4/26 (20 inc, 2 err) | 19/26 | 18/26 | 0/26 (26 uns) | +| TypeScript (13 templates, 26 assertions) | 4/26 (20 inc, 2 err) | 19/26 | n/a | 0/26 (26 uns) | +| Python (13 templates, 26 assertions) | 4/26 (20 inc, 2 err) | 20/26 | 20/26 | 0/26 (26 uns) | +| Kotlin (13 templates, 26 assertions) | 6/26 (18 inc, 2 err) | 19/26 | n/a | 0/26 (26 uns) | +| Scala (13 templates, 26 assertions) | 0/26 (26 inc) | n/a | n/a | n/a | +| C# (13 templates, 26 assertions) | 1/26 (25 inc) | 20/26 | n/a | n/a | +| Go (13 templates, 26 assertions) | 4/26 (22 inc) | 19/26 | n/a | 0/26 (26 uns) | +| PHP (13 templates, 26 assertions) | 4/26 (22 inc) | n/a | 20/26 | 0/26 (26 uns) | +| Ruby (13 templates, 26 assertions) | 0/26 (26 inc) | 20/26 | 14/26 | 0/26 (26 uns) | +| C++ (12 templates, 24 assertions) | 0/24 (24 inc) | 14/24 | n/a | 0/24 (24 uns) | +| C (9 templates, 18 assertions) | 0/18 (18 inc) | 14/18 | n/a | 0/18 (18 uns) | +| Rust (12 templates, 24 assertions) | 0/24 (20 inc, 4 err) | 16/24 | 16/24 | 0/24 (24 uns) | + +Reading the four populations, each on its own terms: + +- **Bifrost 0.10.6** decides **115 of the 118** cases in the pinned breadth + smoke population correctly; the three non-decisive results are Ruby's + direct-propagation pair (`inconclusive`) and the modeled-external Java + calibration case (`unsupported`, and not scored). Across the thirteen + expanded kernels it produces 227 decisive outcomes, of which **222 are + correct**: the five decisive mismatches are one false negative on Java's + `direct-propagation` positive, three on Kotlin (an `expression` false + negative plus `infeasible-branch` and `loop-carried` false positives), and + one PHP `infeasible-branch` false positive. Everything else it does not + answer it declines: 489 `inconclusive` core results retaining + `partial_discovery` or `capability_incomplete` diagnostics. On the challenge + strata specifically it decides 33 of 326 assertions — the depth-6 relay, the + two-level context pair, and Java's and Kotlin's recursive carry — and **all + 33 are correct**. Per the preregistration's own reading rule, correct + stratum-D negatives beside undecided positives describe a bound, not + precision. +- **Bifrost's two published defects.** 22 core results are `runner-error` and + are retained verbatim as execution coverage. Ten of them are the + `element-object` pair in Java, JavaScript, Kotlin, Python and TypeScript, + where the run fails with `internal_invariant` and "invalid value-flow + snapshot: oracle relation does not belong to the required query arena and + role". The other twelve are Rust's heap and access-path cases, failing with + a different signature — `internal_invariant`, "semantic IR gap_contract + error … duplicates the same scoped fact". They are tracked upstream as + bifrost-dev #2639 (element-object) and #2638 (Rust gap_contract). Bifrost's Ruby kernel is **58/58 `inconclusive`**: no + assertion is decisive, which is the analyzer-coverage gate + `docs/applicability-matrix.md` already records for Ruby, now measured over + the whole expanded population; it is tracked upstream as bifrost-dev #2637 + and is never counted as 58 misses. +- **An observed instability, published as observed.** The case + `dfb-taint-java-direct-positive` is `reached` (true positive) in + `reports/bifrost-smoke.json` and `not-reached` (false negative) in + `reports/bifrost-java-kernel.json`, at the same fixture revision and the + same build. The two are separate populations with separate scorecards, and + both raw artifacts are retained and digest-bound. Neither result was + re-run to agreement: the freeze publishes what the runs produced. +- **CodeQL 2.26.3** brings the first challenge-tier evidence across **eleven** + languages, with **zero** incomplete outcomes anywhere: every one of its 626 + bound assertions gets a definitive answer, 509 of them correct (506 of the + 622 core assertions, plus 3 of the 4 `language-extension` assertions). That + freeze-wide tally is a count of bound evidence, not a score — the eleven + populations behind it have different denominators and are never pooled. + Per kernel the results run from 42/56 on C++ and 41/48 on C to 49/58 on + Ruby, and on the challenge strata from 14/24 (C++) and 14/18 (C) to 21/26 + (Java) — read one denominator at a time, never as a sequence. Its systematic miss is **dynamic dispatch**: across all + eleven languages the `reflective-invocation` pairs score 8/16 — every + negative correct, every one of the eight positives missed — and the + `dispatch-table` pairs 11/22, where ten of the eleven positives are missed. + That is an under-approximating refusal to follow a callee named at run time, + which stratum A was written to make visible as approximation character + rather than as a ranking. At the other end, `deep-relay-chain` and + `recursive-carry` are 22/22 each. +- **Joern 4.0.610** covers six kernels and answers every assertion + definitively, 270 of 344 correctly. Its per-language character is the story + rather than any single number: Python and PHP at 48/58, Java at 47/58, + JavaScript at 44/58 (6 false positives, tied with Ruby), Rust at 43/54, + and Ruby at 40/58, where 12 false negatives and 6 false positives make it + the frontend with the widest approximation spread. Stratum D behaves as the + preregistration predicted from the verified `maxCallDepth = 4` default: + five of the six kernels miss the depth-6 relay positive while answering its + negative correctly. **Ruby departs from that pattern** — it is the one + frontend that reaches the depth-6 positive, and it is also the one that + false-positives the `recursive-carry` negative. Both readings belong + together: a bound that does not bind here comes with a widening that does + not kill. +- **Semgrep CE 1.174.0** is a bounded-profile population by construction. Of + its 622 bound core assertions, **468 are `unsupported` by declared + capability**, decided from case metadata before Semgrep is invoked, and + **154 are scored** — the intraprocedural partition, identical in every one + of its eleven languages (14 scored per kernel, 12 correct, and the same two + false positives everywhere: the `infeasible-branch` and `loop-carried-kill` + negatives). All 274 challenge assertions in its populations are declined. The preregistration said in + advance that this would happen and why: an engine that documents a construct + as out of scope takes `unsupported`, and that is correct behavior, not a + gap. "12/58" is not a low score on a 58-assertion population; it is 12 + correct of 14 decided, beside 44 declines. +- **`language-extension` tiers** stay outside every core denominator. On C's + two cases CodeQL is 2/2 and Bifrost is `inconclusive` on both; on Rust's + `Result`/`?` pair CodeQL is 1/2 (one false negative) and Bifrost is + `inconclusive` on both. + +No analyzer is declared a winner. The populations, model profiles, coverage, +and denominators differ per language and per analyzer; incomplete outcomes are +coverage evidence rather than incorrect answers; and stratum A results are +approximation character, not a ranking. + +## Reproduction + +```bash +git checkout v0.4.0 # frozen benchmark revision (evidence commit) +# then check out the merge commit carrying reports/freeze.json for this release +cargo run -- validate-freeze reports/freeze.json +cargo run -- generate-results --manifest reports/freeze.json --output-directory results --check +``` + +Adapter re-execution (produces new evidence, therefore a new freeze): + +```bash +cargo run -- run-bifrost-smoke --bifrost +for kernel in java javascript typescript python kotlin scala csharp go php ruby c cpp rust; do + cargo run -- run-bifrost-$kernel-kernel --bifrost +done + +codeql pack install adapters/codeql +for pack in javascript typescript python kotlin csharp go cpp rust ruby; do + codeql pack install adapters/codeql/$pack +done +for kernel in java javascript typescript python kotlin csharp go ruby c cpp rust; do + cargo run -- run-codeql-$kernel-kernel --codeql +done + +for kernel in java javascript python php ruby rust; do + cargo run -- run-joern-$kernel-kernel --joern +done + +for kernel in java javascript typescript python kotlin go php ruby c cpp rust; do + cargo run -- run-semgrep-$kernel-kernel --semgrep +done +``` + +The Kotlin and Go CodeQL runners trace a real compile, so `kotlinc` and the Go +toolchain must be available; the Rust CodeQL runner uses the CLI's public +preview Rust extractor. Joern's `php2cpg` shells out to its bundled +PHP-Parser, so a host `php` interpreter must be on PATH, and its `rust2cpg` +frontend materializes each case as a minimal Cargo crate. + +## Immutability + +This snapshot is immutable. Corrected evidence creates a new freeze with a new +release name and digests; the v0.1.0, v0.2.0, and v0.3.0 manifests and evidence +remain available at their release commits for audit. This release has no DOI +and no external review claim. diff --git a/docs/src/components/LandingResults.astro b/docs/src/components/LandingResults.astro index ebc3447..c4ef840 100644 --- a/docs/src/components/LandingResults.astro +++ b/docs/src/components/LandingResults.astro @@ -31,6 +31,7 @@ const vendorNames: Record = { bifrost: 'Bifrost', codeql: 'CodeQL', joern: 'Joern', + semgrep: 'Semgrep CE', }; function vendorName(tool: string): string { return vendorNames[tool] ?? tool.charAt(0).toUpperCase() + tool.slice(1); @@ -209,9 +210,14 @@ kernels.forEach((kernel, index) => { kernel.id = `kernel-${index}`; }); -const templateSizes = [...new Set(kernels.map((kernel) => kernel.templates))] - .sort((left, right) => left - right) - .join('–'); +// Core sizes differ per language. Two sizes read naturally as a range; more +// than two have to be enumerated, or "24–29" would imply sizes nobody has. +const templateSizeList = [...new Set(kernels.map((kernel) => kernel.templates))] + .sort((left, right) => left - right); +const templateSizes = + templateSizeList.length <= 2 + ? templateSizeList.join('–') + : `${templateSizeList.slice(0, -1).join(', ')} or ${templateSizeList.at(-1)}`; // Payload for the details dialog: embedded at build time, so the page stays // fully static and the dialog needs no network access. @@ -308,6 +314,19 @@ function kernelTable(kernel: Kernel): KernelRow[] { ); } +// A card's headline fraction is correct-of-population. For an analyzer that +// declines part of the population, that fraction alone reads as a low score +// rather than as a bounded profile, so the card also states the decided +// fraction and names *why* the rest is absent — declared-capability declines +// (`unsupported`) are a design position, not a miss. +function coverageNote(vendor: VendorStats): string { + const parts: string[] = []; + if (vendor.unsupported > 0) parts.push(`${vendor.unsupported} declined`); + if (vendor.inconclusive > 0) parts.push(`${vendor.inconclusive} incomplete`); + if (vendor.runnerErrors > 0) parts.push(`${vendor.runnerErrors} runner-error`); + return `${vendor.name}: ${parts.join(', ')}`; +} + const outcomeClass = (polarity: 'positive' | 'negative', outcome: string) => { if (polarity === 'positive' && outcome === 'reached') return 'good'; if (polarity === 'negative' && outcome === 'not-reached') return 'good'; @@ -328,13 +347,14 @@ const caseCount = snapshot.results.scorecards