From 7e51b376df0e50f38de5cf4e86a2442fe15b0cb9 Mon Sep 17 00:00:00 2001 From: "Vitaly D." Date: Sat, 16 May 2026 20:16:03 +0300 Subject: [PATCH] test(policy-scanner): expand generated path eval corpus Why: - Generated manifest paths are ambiguous because they may be disposable output or runtime/supply-chain inputs. - The scanner should keep that policy boundary explicit in the frozen eval corpus before any future behavior change. What changed: - Add generated-path policy scanner fixtures that preserve top-level generated manifest files as dependency review signals. - Add negative fixtures for generated dependency examples under existing non-production prefixes and for generated non-manifest text. - Update the policy scanner baseline and README metrics for the 82-fixture corpus. Testing: - bash tests/test-policy-scanner-evals.sh - bash tests/test-policy-scanner-eval-compare.sh - bash tests/test-signum-evolve-v1.sh - PATH=/opt/homebrew/bin:/Users/vi/.codex/tmp/arg0/codex-arg0IyheAr:/opt/homebrew/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home/bin:/Users/vi/Library/Android/sdk/platform-tools:/Users/vi/Library/Android/sdk/emulator:/Users/vi/.antigravity/antigravity/bin:/Users/vi/.agents/bin:/Users/vi/.opencode/bin:/Users/vi/.local/bin:/Users/vi/go/bin:/opt/homebrew/opt/libpq/bin:/Users/vi/.local/bin:/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/System/Cryptexes/App/usr/bin:/usr/bin:/bin:/usr/sbin:/sbin:/var/run/com.apple.security.cryptexd/codex.system/bootstrap/usr/local/bin:/var/run/com.apple.security.cryptexd/codex.system/bootstrap/usr/bin:/var/run/com.apple.security.cryptexd/codex.system/bootstrap/usr/appleinternal/bin:/opt/pkg/env/active/bin:/opt/pmk/env/global/bin:/Library/Apple/usr/bin:/usr/local/go/bin:/Users/vi/.local/bin:/Users/vi/.cargo/bin:/Users/vi/Library/Application Support/JetBrains/Toolbox/scripts:/Applications/Codex.app/Contents/Resources bash scripts/run-deterministic-tests.sh Risk: - narrow - This changes eval coverage and baseline fixture count only; scanner behavior and policy rule catalog are unchanged. Constraint: Do not change scanner behavior, policy rules, Codex prompt, Claude overlay runtime, CI wiring, or generated experiment output. --- evals/policy_scanner/README.md | 15 +++++++------ evals/policy_scanner/baselines/current.json | 14 ++++++------ ...-client-package-json-current-behavior.json | 22 +++++++++++++++++++ .../generated-go-mod-current-behavior.json | 22 +++++++++++++++++++ ...nerated-requirements-current-behavior.json | 22 +++++++++++++++++++ ...ocs-generated-package-json-no-trigger.json | 14 ++++++++++++ ...les-generated-requirements-no-trigger.json | 14 ++++++++++++ ...manifest-dependency-string-no-trigger.json | 14 ++++++++++++ .../tests-generated-go-mod-no-trigger.json | 14 ++++++++++++ 9 files changed, 137 insertions(+), 14 deletions(-) create mode 100644 evals/policy_scanner/fixtures/adversarial/generated-client-package-json-current-behavior.json create mode 100644 evals/policy_scanner/fixtures/adversarial/generated-go-mod-current-behavior.json create mode 100644 evals/policy_scanner/fixtures/adversarial/generated-requirements-current-behavior.json create mode 100644 evals/policy_scanner/fixtures/negative/docs-generated-package-json-no-trigger.json create mode 100644 evals/policy_scanner/fixtures/negative/examples-generated-requirements-no-trigger.json create mode 100644 evals/policy_scanner/fixtures/negative/generated-nonmanifest-dependency-string-no-trigger.json create mode 100644 evals/policy_scanner/fixtures/negative/tests-generated-go-mod-no-trigger.json diff --git a/evals/policy_scanner/README.md b/evals/policy_scanner/README.md index 6e90e37..065926c 100644 --- a/evals/policy_scanner/README.md +++ b/evals/policy_scanner/README.md @@ -16,12 +16,12 @@ The scanner is intentionally unchanged by this harness. The goal is to measure c - `fixtures/suppression/*.json` covers suppression acceptance and rejection behavior. - `fixtures/adversarial/*.json` covers brittle formatting and close-call cases. -The expanded corpus v1 contains 75 fixtures: +The expanded corpus contains 82 fixtures: - `positive`: 34 -- `negative`: 14 +- `negative`: 18 - `suppression`: 15 -- `adversarial`: 12 +- `adversarial`: 15 The harness is read-only with respect to scanner behavior. It creates temporary directories, writes each fixture patch to `combined.patch`, invokes: @@ -201,7 +201,7 @@ Do not update the baseline only to hide a regression. Baseline changes should be ## Current baseline sample -Local expanded-corpus baseline run on 2026-05-09: +Local expanded-corpus baseline run with generated-path audit cases: ```json { @@ -212,14 +212,14 @@ Local expanded-corpus baseline run on 2026-05-09: "failed": 0, "falseNegatives": 0, "falsePositives": 0, - "fixtureCount": 75, + "fixtureCount": 82, "hardGatePassed": true, "knownBaselineFailures": 0, - "passed": 75, + "passed": 82, "precision": 1.0, "recall": 1.0, "severityMismatches": 0, - "truePositives": 55, + "truePositives": 58, "unexpectedCriticalFindings": 0 } ``` @@ -233,3 +233,4 @@ Local expanded-corpus baseline run on 2026-05-09: ## Current behavior notes - `adversarial-generated-package-json-current-behavior`: generated-like `generated/package.json` paths are still treated as manifest dependency findings by current scanner behavior. This corpus records that behavior without changing scanner scope. +- Generated manifest files under top-level `generated/` remain dependency review signals in the current corpus because they may still affect runtime or supply-chain surfaces. Generated dependency examples under already non-production prefixes such as `docs/`, `examples/`, and `tests/` remain excluded by path scope. diff --git a/evals/policy_scanner/baselines/current.json b/evals/policy_scanner/baselines/current.json index ff1436d..f569897 100644 --- a/evals/policy_scanner/baselines/current.json +++ b/evals/policy_scanner/baselines/current.json @@ -1,9 +1,9 @@ { - "fixtureCount": 75, + "fixtureCount": 82, "generatedFrom": { "command": "python3 evals/policy_scanner/run_policy_scanner_eval.py --repo-root . --json-output ", "fixturesDir": "evals/policy_scanner/fixtures", - "note": "Frozen from the expanded policy scanner eval corpus v1; no timestamps or temp paths included.", + "note": "Frozen from the expanded policy scanner eval corpus with generated-path audit cases; no timestamps or temp paths included.", "repeat": 2 }, "harnessName": "policy_scanner", @@ -16,17 +16,17 @@ "failed": 0, "falseNegatives": 0, "falsePositives": 0, - "fixtureCount": 75, + "fixtureCount": 82, "hardGateFailures": [], "hardGatePassed": true, "knownBaselineFailures": 0, - "passed": 75, + "passed": 82, "precision": 1.0, "recall": 1.0, - "runtimeMsP50": 160.184458, - "runtimeMsP95": 280.426723, + "runtimeMsP50": 248.451375, + "runtimeMsP95": 455.602164, "severityMismatches": 0, - "truePositives": 55, + "truePositives": 58, "unexpectedCriticalFindings": 0 }, "schemaVersion": "1.0" diff --git a/evals/policy_scanner/fixtures/adversarial/generated-client-package-json-current-behavior.json b/evals/policy_scanner/fixtures/adversarial/generated-client-package-json-current-behavior.json new file mode 100644 index 0000000..537aec8 --- /dev/null +++ b/evals/policy_scanner/fixtures/adversarial/generated-client-package-json-current-behavior.json @@ -0,0 +1,22 @@ +{ + "allowedExtraFindings": [], + "caseId": "adversarial-generated-client-package-json-current-behavior", + "description": "Nested generated package.json paths are still treated as manifest dependency findings by current scanner behavior.", + "expectedFindings": [ + { + "file": "generated/client/package.json", + "line": 1, + "mustBlock": false, + "ruleId": "POLICY_NEW_NPM_DEPENDENCY", + "severity": "MAJOR" + } + ], + "kind": "adversarial", + "patch": "diff --git a/generated/client/package.json b/generated/client/package.json\n@@ -0,0 +1,1 @@\n+\"sdk-runtime\": \"^2.0.0\"\n", + "tags": [ + "dependency", + "generated", + "current-behavior", + "supply-chain" + ] +} diff --git a/evals/policy_scanner/fixtures/adversarial/generated-go-mod-current-behavior.json b/evals/policy_scanner/fixtures/adversarial/generated-go-mod-current-behavior.json new file mode 100644 index 0000000..0882b15 --- /dev/null +++ b/evals/policy_scanner/fixtures/adversarial/generated-go-mod-current-behavior.json @@ -0,0 +1,22 @@ +{ + "allowedExtraFindings": [], + "caseId": "adversarial-generated-go-mod-current-behavior", + "description": "Generated go.mod paths are still treated as manifest dependency findings by current scanner behavior.", + "expectedFindings": [ + { + "file": "generated/go.mod", + "line": 1, + "mustBlock": false, + "ruleId": "POLICY_NEW_GO_DEPENDENCY", + "severity": "MAJOR" + } + ], + "kind": "adversarial", + "patch": "diff --git a/generated/go.mod b/generated/go.mod\n@@ -0,0 +1,1 @@\n+github.com/acme/sdk v1.2.3\n", + "tags": [ + "dependency", + "generated", + "current-behavior", + "supply-chain" + ] +} diff --git a/evals/policy_scanner/fixtures/adversarial/generated-requirements-current-behavior.json b/evals/policy_scanner/fixtures/adversarial/generated-requirements-current-behavior.json new file mode 100644 index 0000000..bdca530 --- /dev/null +++ b/evals/policy_scanner/fixtures/adversarial/generated-requirements-current-behavior.json @@ -0,0 +1,22 @@ +{ + "allowedExtraFindings": [], + "caseId": "adversarial-generated-requirements-current-behavior", + "description": "Generated requirements.txt paths are still treated as manifest dependency findings by current scanner behavior.", + "expectedFindings": [ + { + "file": "generated/requirements.txt", + "line": 1, + "mustBlock": false, + "ruleId": "POLICY_NEW_PYTHON_DEPENDENCY", + "severity": "MAJOR" + } + ], + "kind": "adversarial", + "patch": "diff --git a/generated/requirements.txt b/generated/requirements.txt\n@@ -0,0 +1,1 @@\n+requests==2.31.0\n", + "tags": [ + "dependency", + "generated", + "current-behavior", + "supply-chain" + ] +} diff --git a/evals/policy_scanner/fixtures/negative/docs-generated-package-json-no-trigger.json b/evals/policy_scanner/fixtures/negative/docs-generated-package-json-no-trigger.json new file mode 100644 index 0000000..7ba5967 --- /dev/null +++ b/evals/policy_scanner/fixtures/negative/docs-generated-package-json-no-trigger.json @@ -0,0 +1,14 @@ +{ + "allowedExtraFindings": [], + "caseId": "negative-docs-generated-package-json-no-trigger", + "description": "Generated dependency examples under docs should remain excluded by docs/ path scope.", + "expectedFindings": [], + "kind": "negative", + "patch": "diff --git a/docs/generated/package.json b/docs/generated/package.json\n@@ -0,0 +1,1 @@\n+\"left-pad\": \"^1.3.0\"\n", + "tags": [ + "dependency", + "generated", + "docs", + "negative" + ] +} diff --git a/evals/policy_scanner/fixtures/negative/examples-generated-requirements-no-trigger.json b/evals/policy_scanner/fixtures/negative/examples-generated-requirements-no-trigger.json new file mode 100644 index 0000000..851b799 --- /dev/null +++ b/evals/policy_scanner/fixtures/negative/examples-generated-requirements-no-trigger.json @@ -0,0 +1,14 @@ +{ + "allowedExtraFindings": [], + "caseId": "negative-examples-generated-requirements-no-trigger", + "description": "Generated dependency examples under examples should remain excluded by examples/ path scope.", + "expectedFindings": [], + "kind": "negative", + "patch": "diff --git a/examples/generated/requirements.txt b/examples/generated/requirements.txt\n@@ -0,0 +1,1 @@\n+requests==2.31.0\n", + "tags": [ + "dependency", + "generated", + "examples", + "negative" + ] +} diff --git a/evals/policy_scanner/fixtures/negative/generated-nonmanifest-dependency-string-no-trigger.json b/evals/policy_scanner/fixtures/negative/generated-nonmanifest-dependency-string-no-trigger.json new file mode 100644 index 0000000..1c077aa --- /dev/null +++ b/evals/policy_scanner/fixtures/negative/generated-nonmanifest-dependency-string-no-trigger.json @@ -0,0 +1,14 @@ +{ + "allowedExtraFindings": [], + "caseId": "negative-generated-nonmanifest-dependency-string-no-trigger", + "description": "Dependency-looking strings in generated non-manifest files should not trigger dependency findings.", + "expectedFindings": [], + "kind": "negative", + "patch": "diff --git a/generated/metadata.txt b/generated/metadata.txt\n@@ -0,0 +1,1 @@\n+\"left-pad\": \"^1.3.0\"\n", + "tags": [ + "dependency", + "generated", + "nonmanifest", + "negative" + ] +} diff --git a/evals/policy_scanner/fixtures/negative/tests-generated-go-mod-no-trigger.json b/evals/policy_scanner/fixtures/negative/tests-generated-go-mod-no-trigger.json new file mode 100644 index 0000000..087fe2f --- /dev/null +++ b/evals/policy_scanner/fixtures/negative/tests-generated-go-mod-no-trigger.json @@ -0,0 +1,14 @@ +{ + "allowedExtraFindings": [], + "caseId": "negative-tests-generated-go-mod-no-trigger", + "description": "Generated dependency examples under tests should remain excluded by tests/ path scope.", + "expectedFindings": [], + "kind": "negative", + "patch": "diff --git a/tests/generated/go.mod b/tests/generated/go.mod\n@@ -0,0 +1,1 @@\n+github.com/acme/sdk v1.2.3\n", + "tags": [ + "dependency", + "generated", + "tests", + "negative" + ] +}