From c4101d5a5a279926957c6df734eb26a7cc534b47 Mon Sep 17 00:00:00 2001 From: GovKM Date: Thu, 22 Oct 2026 17:14:58 -0700 Subject: [PATCH] Add GovKM experimental authority-resolution generator and public fixtures --- .../CONTRIBUTION_NOTES.md | 3 + experimental/authority_resolution/README.md | 28 ++++ .../authority_resolution/SOURCE_SHA256.json | 9 ++ .../authority_resolution/SUBMISSION.md | 11 ++ .../examples/public_seed_20261022/cases.json | 1 + .../expected_outcomes.json | 1 + .../public_seed_20261022/metadata.json | 1 + .../public_seed_20261022/sha256_manifest.json | 1 + .../generate_authority_cases.py | 138 ++++++++++++++++++ .../tests/test_generator.py | 118 +++++++++++++++ 10 files changed, 311 insertions(+) create mode 100644 experimental/authority_resolution/CONTRIBUTION_NOTES.md create mode 100644 experimental/authority_resolution/README.md create mode 100644 experimental/authority_resolution/SOURCE_SHA256.json create mode 100644 experimental/authority_resolution/SUBMISSION.md create mode 100644 experimental/authority_resolution/examples/public_seed_20261022/cases.json create mode 100644 experimental/authority_resolution/examples/public_seed_20261022/expected_outcomes.json create mode 100644 experimental/authority_resolution/examples/public_seed_20261022/metadata.json create mode 100644 experimental/authority_resolution/examples/public_seed_20261022/sha256_manifest.json create mode 100644 experimental/authority_resolution/generate_authority_cases.py create mode 100644 experimental/authority_resolution/tests/test_generator.py diff --git a/experimental/authority_resolution/CONTRIBUTION_NOTES.md b/experimental/authority_resolution/CONTRIBUTION_NOTES.md new file mode 100644 index 0000000..a95024a --- /dev/null +++ b/experimental/authority_resolution/CONTRIBUTION_NOTES.md @@ -0,0 +1,3 @@ +# Contribution review notes + +Candidate only; not submitted. GovKM is fixture author. Intended upstream licensing is ADEBench repository MIT, subject to separate approval. This archive contains public fictional fixtures only, zero holdout seed/material, no scoring or model-adapter integration. Avoid publishing until maintainer agrees exact directory/schema and oracle visibility rules. diff --git a/experimental/authority_resolution/README.md b/experimental/authority_resolution/README.md new file mode 100644 index 0000000..678445e --- /dev/null +++ b/experimental/authority_resolution/README.md @@ -0,0 +1,28 @@ +# GovKM / ADEBench authority-resolution fixture generator v0.2.0 + +**Scope:** Public, purely synthetic fixture-generation candidate for ADEBench's proposed experimental non-Core authority-resolution track. **Not** an ADEBench scorer, model-memory adapter, or completed benchmark. No scored tests, upstream submission, private holdout, or external communication. + +## Quick start (non-scored fixture self-validation) + +```sh +python -m unittest discover -s tests -v +python generate_authority_cases.py --seed 20261022 --count 2 --set public --output-dir examples/public_seed_20261022 +``` + +`--count N` generates N variants **per** six-pattern family (6N total). The seed is a nonnegative integer <2^63. The output directory must not exist; no overwrite. Holdout export is explicitly rejected. Python's PRNG stream should be replayed using the Python runtime version recorded in metadata. + +## v0.2 protocol corrections + +- `cases.json` includes `source_inventory_text`, which is **only the input record collection** offered for indexing. It must never be counted as delivered-to-model evidence. The only scoring boundary accepted by Simone is ADEBench's independently captured `delivered_text_after_cut`. This implementation does not create, simulate or evaluate that field. `expected_outcomes.json` is a separated private-to-evaluator oracle; never provide it to a model or retrieval system. +- Authorized supersession AR-01 requires source E1, superseding E2, and source grant G1 as **required** evidence rather than assuming supersession alone suffices. Revocation AR-04 likewise requires E1, E2 and G1. +- AR-06 recognizes two different outcomes: `NO_AUTHORITY_IF_COVERAGE_PROVEN` only where grant-register coverage (SOURCE-REGISTRY) is independently supported in model-delivered evidence; otherwise `AUTHORITY_UNRESOLVED`. `source_coverage` is a fictional source assertion, **not proof that the memory delivered it**. External score thresholds and semantic paraphrase matching remain ADEBench decisions. +- All scenarios preserve distinct `valid_at` (effective authority) versus `recorded_at` (ingestion/source chronology). AR-05 historical as-of t2 must not be superseded retroactively by t3. Exact boundary conditions remain subject to ADEBench's scoring contract. +- Three separate evaluator dimensions are declared: operative value, authority outcome and source event delivery. This generator DOES NOT implement any evaluator, cutoff algorithm, or score. + +## Files and provenance + +`cases.json` public fictional events and coverage assumptions; `expected_outcomes.json` candidate oracle (never feed to memory); `metadata.json` seed/version/runtime; `sha256_manifest.json` SHA-256 for output files and generator/test source; `tests/test_generator.py` local non-scored consistency checks. Source and tests are included in manifest rather than embedding self-referential hashes into source. + +**Packaging boundary:** No partner records, GovKM doctrine, internal scoring methods, or proprietary AIL logic may be contributed. ADEBench controls its own scoring and experimental leaderboard; attribution of GovKM as fixture author and licensing under actual repo license must be verified at the separate upstream approval gate. ADEBench, not GovKM, retains any private holdout seeds. + +**Protocol note:** `source_coverage` currently declares a fictional complete registry as a source assertion. It does not prove actual retrieved or delivered coverage. This deliberately prevents false-positive claims that "no grant found" means "no authority." An independent evaluator must define a falsifiable model-visible coverage proof before scoring NO_AUTHORITY. Do not infer a passing result from this metadata. diff --git a/experimental/authority_resolution/SOURCE_SHA256.json b/experimental/authority_resolution/SOURCE_SHA256.json new file mode 100644 index 0000000..7771c66 --- /dev/null +++ b/experimental/authority_resolution/SOURCE_SHA256.json @@ -0,0 +1,9 @@ +{ + "README.md": "9db79f9da2668a144982acb21e1c9f1097c44c05517e2de2ba6f14f45188f4db", + "examples/public_seed_20261022/cases.json": "f6c2515f64ed6ddc2226cfef19c0e9db0426092893ee8d211195cbd119f1e0a5", + "examples/public_seed_20261022/expected_outcomes.json": "b8f64ef4f018958db56a4ed0a828dab006a56cfc09c26093e91cb4410232c192", + "examples/public_seed_20261022/metadata.json": "e169842f562b59cd3bad28e0ac8c710338f29df2712ee797cd761c9a02895908", + "examples/public_seed_20261022/sha256_manifest.json": "361ac71f6d0ccfb8cb5431b3290d0860a77dbbd5deab236874b5b8921c411a5b", + "generate_authority_cases.py": "a2321ffed53aaa7a9c42fb47293c8f4875581effe76bf9760b83e676fbfe4d58", + "tests/test_generator.py": "f9d4570b94002eac7ee51ccf4cf25994554b35ae9d207a73727a2d2e79e77dbc" +} diff --git a/experimental/authority_resolution/SUBMISSION.md b/experimental/authority_resolution/SUBMISSION.md new file mode 100644 index 0000000..4d307d5 --- /dev/null +++ b/experimental/authority_resolution/SUBMISSION.md @@ -0,0 +1,11 @@ +# GovKM experimental authority-resolution contribution + +GovKM is fixture author. The synthetic contribution is offered under this repository's MIT license; see ../../LICENSE. The original v0.2 source, example files and manifests are preserved byte for byte. README.md and CONTRIBUTION_NOTES.md describe the original candidate preparation status; this submission note records the subsequent authorized proposal. + +This directory is a proposed experimental location outside Core, subject to maintainer review under issue #2. It adds no scorer, adapter, package dependency, CI configuration or leaderboard change. ADEBench retains scoring, metrics, experimental leaderboard and private holdout custody. + +Only source_inventory_text is offered as unscored source input. expected_outcomes.json is a public-example evaluation oracle and must never enter memory/model input. Only independently captured delivered_text_after_cut can support future ADEBench scores. No private holdout was generated or included. + +Validation: Python 3.12.14, 17/17 offline generator tests passed. Public seed 20261022, count 2 per family gives 12 cases. Cases and expected outcomes reproduce approved sample bytes. Original metadata records Python 3.13.5; current-runtime replay changes python_version and its manifest hash only. No scored benchmark or native AI-memory/model experiment was run. + +Maintainer review remains needed for directory/schema, public-oracle placement, delivered coverage proof for no-authority cases, temporal boundaries and matching/cut rules. No claim of adoption before merge. diff --git a/experimental/authority_resolution/examples/public_seed_20261022/cases.json b/experimental/authority_resolution/examples/public_seed_20261022/cases.json new file mode 100644 index 0000000..fe5c26b --- /dev/null +++ b/experimental/authority_resolution/examples/public_seed_20261022/cases.json @@ -0,0 +1 @@ +[{"case_id":"AR-01-S20261022-V0001","events":[{"authority_grant_ids":["G1"],"claim":"Juniper approve configuration Moss for Westhaven","event_id":"E1","operation":"approve","principal_id":"Juniper","recorded_at":"2031-04-22T12:00:00Z","source_ref":"SOURCE-E1","target_id":null,"valid_at":"2031-04-08T12:00:00Z","value":"Moss"},{"authority_grant_ids":["G1"],"claim":"Juniper supersede configuration Fern for Westhaven","event_id":"E2","operation":"supersede","principal_id":"Juniper","recorded_at":"2031-04-18T12:00:00Z","source_ref":"SOURCE-E2","target_id":"E1","valid_at":"2031-04-17T12:00:00Z","value":"Fern"}],"evidence_inventory_role":"unscored_source_input; not delivered_text_after_cut","generator_version":"0.2.0","grants":[{"capability":"approve","grant_id":"G1","grantee":"Juniper","issuer":"Synthetic Council","scope":"Westhaven","source_ref":"SOURCE-G1","valid_from":"2031-04-07T12:00:00Z","valid_to":"2031-04-28T12:00:00Z"}],"pattern":"AR-01","query_at":"2031-04-23T12:00:00Z","query_text":"Which configuration governs Westhaven at 2031-04-23T12:00:00Z, and what source evidence establishes it?","schema_version":"2.0","source_coverage":{"coverage_declared_at":"2031-04-23T12:00:00Z","event_log_complete_through":"2031-04-23T12:00:00Z","grant_registry_complete":true,"registry_source_ref":"SOURCE-REGISTRY","scope":"Westhaven"},"source_inventory_text":"Case AR-01-S20261022-V0001 / Scope Westhaven\nAuthority grants:\n[SOURCE-G1] Juniper may approve in Westhaven, 2031-04-07T12:00:00Z through 2031-04-28T12:00:00Z\nEvent records:\n[SOURCE-E1] E1 Juniper approve configuration Moss for Westhaven; effective=2031-04-08T12:00:00Z; recorded=2031-04-22T12:00:00Z; grants=G1; target=NONE\n[SOURCE-E2] E2 Juniper supersede configuration Fern for Westhaven; effective=2031-04-17T12:00:00Z; recorded=2031-04-18T12:00:00Z; grants=G1; target=E1\n","variant_index":1},{"case_id":"AR-01-S20261022-V0002","events":[{"authority_grant_ids":["G1"],"claim":"Lumen approve configuration Dawn for Westhaven","event_id":"E1","operation":"approve","principal_id":"Lumen","recorded_at":"2031-04-21T12:00:00Z","source_ref":"SOURCE-E1","target_id":null,"valid_at":"2031-04-07T12:00:00Z","value":"Dawn"},{"authority_grant_ids":["G1"],"claim":"Lumen supersede configuration Amber for Westhaven","event_id":"E2","operation":"supersede","principal_id":"Lumen","recorded_at":"2031-04-17T12:00:00Z","source_ref":"SOURCE-E2","target_id":"E1","valid_at":"2031-04-16T12:00:00Z","value":"Amber"}],"evidence_inventory_role":"unscored_source_input; not delivered_text_after_cut","generator_version":"0.2.0","grants":[{"capability":"approve","grant_id":"G1","grantee":"Lumen","issuer":"Synthetic Council","scope":"Westhaven","source_ref":"SOURCE-G1","valid_from":"2031-04-06T12:00:00Z","valid_to":"2031-04-27T12:00:00Z"}],"pattern":"AR-01","query_at":"2031-04-22T12:00:00Z","query_text":"Which configuration governs Westhaven at 2031-04-22T12:00:00Z, and what source evidence establishes it?","schema_version":"2.0","source_coverage":{"coverage_declared_at":"2031-04-22T12:00:00Z","event_log_complete_through":"2031-04-22T12:00:00Z","grant_registry_complete":true,"registry_source_ref":"SOURCE-REGISTRY","scope":"Westhaven"},"source_inventory_text":"Case AR-01-S20261022-V0002 / Scope Westhaven\nAuthority grants:\n[SOURCE-G1] Lumen may approve in Westhaven, 2031-04-06T12:00:00Z through 2031-04-27T12:00:00Z\nEvent records:\n[SOURCE-E1] E1 Lumen approve configuration Dawn for Westhaven; effective=2031-04-07T12:00:00Z; recorded=2031-04-21T12:00:00Z; grants=G1; target=NONE\n[SOURCE-E2] E2 Lumen supersede configuration Amber for Westhaven; effective=2031-04-16T12:00:00Z; recorded=2031-04-17T12:00:00Z; grants=G1; target=E1\n","variant_index":2},{"case_id":"AR-02-S20261022-V0001","events":[{"authority_grant_ids":["G1"],"claim":"Lumen approve configuration Ember for Silverbank","event_id":"E1","operation":"approve","principal_id":"Lumen","recorded_at":"2032-04-06T12:00:00Z","source_ref":"SOURCE-E1","target_id":null,"valid_at":"2032-04-05T12:00:00Z","value":"Ember"},{"authority_grant_ids":[],"claim":"Kestrel approve configuration Moss for Silverbank","event_id":"E2","operation":"approve","principal_id":"Kestrel","recorded_at":"2032-04-15T12:00:00Z","source_ref":"SOURCE-E2","target_id":null,"valid_at":"2032-04-14T12:00:00Z","value":"Moss"}],"evidence_inventory_role":"unscored_source_input; not delivered_text_after_cut","generator_version":"0.2.0","grants":[{"capability":"approve","grant_id":"G1","grantee":"Lumen","issuer":"Synthetic Council","scope":"Silverbank","source_ref":"SOURCE-G1","valid_from":"2032-04-04T12:00:00Z","valid_to":"2032-04-25T12:00:00Z"}],"pattern":"AR-02","query_at":"2032-04-20T12:00:00Z","query_text":"Which configuration governs Silverbank at 2032-04-20T12:00:00Z, and what source evidence establishes it?","schema_version":"2.0","source_coverage":{"coverage_declared_at":"2032-04-20T12:00:00Z","event_log_complete_through":"2032-04-20T12:00:00Z","grant_registry_complete":true,"registry_source_ref":"SOURCE-REGISTRY","scope":"Silverbank"},"source_inventory_text":"Case AR-02-S20261022-V0001 / Scope Silverbank\nAuthority grants:\n[SOURCE-G1] Lumen may approve in Silverbank, 2032-04-04T12:00:00Z through 2032-04-25T12:00:00Z\nEvent records:\n[SOURCE-E1] E1 Lumen approve configuration Ember for Silverbank; effective=2032-04-05T12:00:00Z; recorded=2032-04-06T12:00:00Z; grants=G1; target=NONE\n[SOURCE-E2] E2 Kestrel approve configuration Moss for Silverbank; effective=2032-04-14T12:00:00Z; recorded=2032-04-15T12:00:00Z; grants=NONE; target=NONE\n","variant_index":1},{"case_id":"AR-02-S20261022-V0002","events":[{"authority_grant_ids":["G1"],"claim":"Delta approve configuration Jade for Ashport","event_id":"E1","operation":"approve","principal_id":"Delta","recorded_at":"2031-10-05T12:00:00Z","source_ref":"SOURCE-E1","target_id":null,"valid_at":"2031-10-04T12:00:00Z","value":"Jade"},{"authority_grant_ids":[],"claim":"Juniper approve configuration Blue for Ashport","event_id":"E2","operation":"approve","principal_id":"Juniper","recorded_at":"2031-10-14T12:00:00Z","source_ref":"SOURCE-E2","target_id":null,"valid_at":"2031-10-13T12:00:00Z","value":"Blue"}],"evidence_inventory_role":"unscored_source_input; not delivered_text_after_cut","generator_version":"0.2.0","grants":[{"capability":"approve","grant_id":"G1","grantee":"Delta","issuer":"Synthetic Council","scope":"Ashport","source_ref":"SOURCE-G1","valid_from":"2031-10-03T12:00:00Z","valid_to":"2031-10-24T12:00:00Z"}],"pattern":"AR-02","query_at":"2031-10-19T12:00:00Z","query_text":"Which configuration governs Ashport at 2031-10-19T12:00:00Z, and what source evidence establishes it?","schema_version":"2.0","source_coverage":{"coverage_declared_at":"2031-10-19T12:00:00Z","event_log_complete_through":"2031-10-19T12:00:00Z","grant_registry_complete":true,"registry_source_ref":"SOURCE-REGISTRY","scope":"Ashport"},"source_inventory_text":"Case AR-02-S20261022-V0002 / Scope Ashport\nAuthority grants:\n[SOURCE-G1] Delta may approve in Ashport, 2031-10-03T12:00:00Z through 2031-10-24T12:00:00Z\nEvent records:\n[SOURCE-E1] E1 Delta approve configuration Jade for Ashport; effective=2031-10-04T12:00:00Z; recorded=2031-10-05T12:00:00Z; grants=G1; target=NONE\n[SOURCE-E2] E2 Juniper approve configuration Blue for Ashport; effective=2031-10-13T12:00:00Z; recorded=2031-10-14T12:00:00Z; grants=NONE; target=NONE\n","variant_index":2},{"case_id":"AR-03-S20261022-V0001","events":[{"authority_grant_ids":["G1"],"claim":"Juniper approve configuration Fern for Ashport","event_id":"E1","operation":"approve","principal_id":"Juniper","recorded_at":"2031-01-06T12:00:00Z","source_ref":"SOURCE-E1","target_id":null,"valid_at":"2031-01-05T12:00:00Z","value":"Fern"},{"authority_grant_ids":["G2"],"claim":"Fable approve configuration Copper for Ashport","event_id":"E2","operation":"approve","principal_id":"Fable","recorded_at":"2031-01-06T12:00:00Z","source_ref":"SOURCE-E2","target_id":null,"valid_at":"2031-01-05T12:00:00Z","value":"Copper"}],"evidence_inventory_role":"unscored_source_input; not delivered_text_after_cut","generator_version":"0.2.0","grants":[{"capability":"approve","grant_id":"G1","grantee":"Juniper","issuer":"Synthetic Council","scope":"Ashport","source_ref":"SOURCE-G1","valid_from":"2031-01-04T12:00:00Z","valid_to":"2031-01-25T12:00:00Z"},{"capability":"approve","grant_id":"G2","grantee":"Fable","issuer":"Synthetic Council","scope":"Ashport","source_ref":"SOURCE-G2","valid_from":"2031-01-04T12:00:00Z","valid_to":"2031-01-25T12:00:00Z"}],"pattern":"AR-03","query_at":"2031-01-08T12:00:00Z","query_text":"Which configuration governs Ashport at 2031-01-08T12:00:00Z, and what source evidence establishes it?","schema_version":"2.0","source_coverage":{"coverage_declared_at":"2031-01-08T12:00:00Z","event_log_complete_through":"2031-01-08T12:00:00Z","grant_registry_complete":true,"registry_source_ref":"SOURCE-REGISTRY","scope":"Ashport"},"source_inventory_text":"Case AR-03-S20261022-V0001 / Scope Ashport\nAuthority grants:\n[SOURCE-G1] Juniper may approve in Ashport, 2031-01-04T12:00:00Z through 2031-01-25T12:00:00Z\n[SOURCE-G2] Fable may approve in Ashport, 2031-01-04T12:00:00Z through 2031-01-25T12:00:00Z\nEvent records:\n[SOURCE-E1] E1 Juniper approve configuration Fern for Ashport; effective=2031-01-05T12:00:00Z; recorded=2031-01-06T12:00:00Z; grants=G1; target=NONE\n[SOURCE-E2] E2 Fable approve configuration Copper for Ashport; effective=2031-01-05T12:00:00Z; recorded=2031-01-06T12:00:00Z; grants=G2; target=NONE\n","variant_index":1},{"case_id":"AR-03-S20261022-V0002","events":[{"authority_grant_ids":["G1"],"claim":"Birch approve configuration Copper for Silverbank","event_id":"E1","operation":"approve","principal_id":"Birch","recorded_at":"2034-09-18T12:00:00Z","source_ref":"SOURCE-E1","target_id":null,"valid_at":"2034-09-17T12:00:00Z","value":"Copper"},{"authority_grant_ids":["G2"],"claim":"Lumen approve configuration Indigo for Silverbank","event_id":"E2","operation":"approve","principal_id":"Lumen","recorded_at":"2034-09-18T12:00:00Z","source_ref":"SOURCE-E2","target_id":null,"valid_at":"2034-09-17T12:00:00Z","value":"Indigo"}],"evidence_inventory_role":"unscored_source_input; not delivered_text_after_cut","generator_version":"0.2.0","grants":[{"capability":"approve","grant_id":"G1","grantee":"Birch","issuer":"Synthetic Council","scope":"Silverbank","source_ref":"SOURCE-G1","valid_from":"2034-09-16T12:00:00Z","valid_to":"2034-10-07T12:00:00Z"},{"capability":"approve","grant_id":"G2","grantee":"Lumen","issuer":"Synthetic Council","scope":"Silverbank","source_ref":"SOURCE-G2","valid_from":"2034-09-16T12:00:00Z","valid_to":"2034-10-07T12:00:00Z"}],"pattern":"AR-03","query_at":"2034-09-20T12:00:00Z","query_text":"Which configuration governs Silverbank at 2034-09-20T12:00:00Z, and what source evidence establishes it?","schema_version":"2.0","source_coverage":{"coverage_declared_at":"2034-09-20T12:00:00Z","event_log_complete_through":"2034-09-20T12:00:00Z","grant_registry_complete":true,"registry_source_ref":"SOURCE-REGISTRY","scope":"Silverbank"},"source_inventory_text":"Case AR-03-S20261022-V0002 / Scope Silverbank\nAuthority grants:\n[SOURCE-G1] Birch may approve in Silverbank, 2034-09-16T12:00:00Z through 2034-10-07T12:00:00Z\n[SOURCE-G2] Lumen may approve in Silverbank, 2034-09-16T12:00:00Z through 2034-10-07T12:00:00Z\nEvent records:\n[SOURCE-E1] E1 Birch approve configuration Copper for Silverbank; effective=2034-09-17T12:00:00Z; recorded=2034-09-18T12:00:00Z; grants=G1; target=NONE\n[SOURCE-E2] E2 Lumen approve configuration Indigo for Silverbank; effective=2034-09-17T12:00:00Z; recorded=2034-09-18T12:00:00Z; grants=G2; target=NONE\n","variant_index":2},{"case_id":"AR-04-S20261022-V0001","events":[{"authority_grant_ids":["G1"],"claim":"Cobalt approve configuration Fern for Silverbank","event_id":"E1","operation":"approve","principal_id":"Cobalt","recorded_at":"2037-02-15T12:00:00Z","source_ref":"SOURCE-E1","target_id":null,"valid_at":"2037-02-14T12:00:00Z","value":"Fern"},{"authority_grant_ids":["G1"],"claim":"Cobalt revoke configuration Fern for Silverbank","event_id":"E2","operation":"revoke","principal_id":"Cobalt","recorded_at":"2037-02-24T12:00:00Z","source_ref":"SOURCE-E2","target_id":"E1","valid_at":"2037-02-23T12:00:00Z","value":"Fern"}],"evidence_inventory_role":"unscored_source_input; not delivered_text_after_cut","generator_version":"0.2.0","grants":[{"capability":"approve","grant_id":"G1","grantee":"Cobalt","issuer":"Synthetic Council","scope":"Silverbank","source_ref":"SOURCE-G1","valid_from":"2037-02-13T12:00:00Z","valid_to":"2037-03-06T12:00:00Z"}],"pattern":"AR-04","query_at":"2037-03-01T12:00:00Z","query_text":"Which configuration governs Silverbank at 2037-03-01T12:00:00Z, and what source evidence establishes it?","schema_version":"2.0","source_coverage":{"coverage_declared_at":"2037-03-01T12:00:00Z","event_log_complete_through":"2037-03-01T12:00:00Z","grant_registry_complete":true,"registry_source_ref":"SOURCE-REGISTRY","scope":"Silverbank"},"source_inventory_text":"Case AR-04-S20261022-V0001 / Scope Silverbank\nAuthority grants:\n[SOURCE-G1] Cobalt may approve in Silverbank, 2037-02-13T12:00:00Z through 2037-03-06T12:00:00Z\nEvent records:\n[SOURCE-E1] E1 Cobalt approve configuration Fern for Silverbank; effective=2037-02-14T12:00:00Z; recorded=2037-02-15T12:00:00Z; grants=G1; target=NONE\n[SOURCE-E2] E2 Cobalt revoke configuration Fern for Silverbank; effective=2037-02-23T12:00:00Z; recorded=2037-02-24T12:00:00Z; grants=G1; target=E1\n","variant_index":1},{"case_id":"AR-04-S20261022-V0002","events":[{"authority_grant_ids":["G1"],"claim":"Cobalt approve configuration Indigo for Silverbank","event_id":"E1","operation":"approve","principal_id":"Cobalt","recorded_at":"2034-01-04T12:00:00Z","source_ref":"SOURCE-E1","target_id":null,"valid_at":"2034-01-03T12:00:00Z","value":"Indigo"},{"authority_grant_ids":["G1"],"claim":"Cobalt revoke configuration Indigo for Silverbank","event_id":"E2","operation":"revoke","principal_id":"Cobalt","recorded_at":"2034-01-13T12:00:00Z","source_ref":"SOURCE-E2","target_id":"E1","valid_at":"2034-01-12T12:00:00Z","value":"Indigo"}],"evidence_inventory_role":"unscored_source_input; not delivered_text_after_cut","generator_version":"0.2.0","grants":[{"capability":"approve","grant_id":"G1","grantee":"Cobalt","issuer":"Synthetic Council","scope":"Silverbank","source_ref":"SOURCE-G1","valid_from":"2034-01-02T12:00:00Z","valid_to":"2034-01-23T12:00:00Z"}],"pattern":"AR-04","query_at":"2034-01-18T12:00:00Z","query_text":"Which configuration governs Silverbank at 2034-01-18T12:00:00Z, and what source evidence establishes it?","schema_version":"2.0","source_coverage":{"coverage_declared_at":"2034-01-18T12:00:00Z","event_log_complete_through":"2034-01-18T12:00:00Z","grant_registry_complete":true,"registry_source_ref":"SOURCE-REGISTRY","scope":"Silverbank"},"source_inventory_text":"Case AR-04-S20261022-V0002 / Scope Silverbank\nAuthority grants:\n[SOURCE-G1] Cobalt may approve in Silverbank, 2034-01-02T12:00:00Z through 2034-01-23T12:00:00Z\nEvent records:\n[SOURCE-E1] E1 Cobalt approve configuration Indigo for Silverbank; effective=2034-01-03T12:00:00Z; recorded=2034-01-04T12:00:00Z; grants=G1; target=NONE\n[SOURCE-E2] E2 Cobalt revoke configuration Indigo for Silverbank; effective=2034-01-12T12:00:00Z; recorded=2034-01-13T12:00:00Z; grants=G1; target=E1\n","variant_index":2},{"case_id":"AR-05-S20261022-V0001","events":[{"authority_grant_ids":["G1"],"claim":"Elm approve configuration Fern for Westhaven","event_id":"E1","operation":"approve","principal_id":"Elm","recorded_at":"2031-02-04T12:00:00Z","source_ref":"SOURCE-E1","target_id":null,"valid_at":"2031-02-03T12:00:00Z","value":"Fern"},{"authority_grant_ids":["G1"],"claim":"Elm supersede configuration Blue for Westhaven","event_id":"E2","operation":"supersede","principal_id":"Elm","recorded_at":"2031-02-13T12:00:00Z","source_ref":"SOURCE-E2","target_id":"E1","valid_at":"2031-02-12T12:00:00Z","value":"Blue"}],"evidence_inventory_role":"unscored_source_input; not delivered_text_after_cut","generator_version":"0.2.0","grants":[{"capability":"approve","grant_id":"G1","grantee":"Elm","issuer":"Synthetic Council","scope":"Westhaven","source_ref":"SOURCE-G1","valid_from":"2031-02-02T12:00:00Z","valid_to":"2031-02-23T12:00:00Z"}],"pattern":"AR-05","query_at":"2031-02-06T12:00:00Z","query_text":"Which configuration governs Westhaven at 2031-02-06T12:00:00Z, and what source evidence establishes it?","schema_version":"2.0","source_coverage":{"coverage_declared_at":"2031-02-06T12:00:00Z","event_log_complete_through":"2031-02-06T12:00:00Z","grant_registry_complete":true,"registry_source_ref":"SOURCE-REGISTRY","scope":"Westhaven"},"source_inventory_text":"Case AR-05-S20261022-V0001 / Scope Westhaven\nAuthority grants:\n[SOURCE-G1] Elm may approve in Westhaven, 2031-02-02T12:00:00Z through 2031-02-23T12:00:00Z\nEvent records:\n[SOURCE-E1] E1 Elm approve configuration Fern for Westhaven; effective=2031-02-03T12:00:00Z; recorded=2031-02-04T12:00:00Z; grants=G1; target=NONE\n[SOURCE-E2] E2 Elm supersede configuration Blue for Westhaven; effective=2031-02-12T12:00:00Z; recorded=2031-02-13T12:00:00Z; grants=G1; target=E1\n","variant_index":1},{"case_id":"AR-05-S20261022-V0002","events":[{"authority_grant_ids":["G1"],"claim":"Juniper approve configuration Granite for Ashport","event_id":"E1","operation":"approve","principal_id":"Juniper","recorded_at":"2032-09-03T12:00:00Z","source_ref":"SOURCE-E1","target_id":null,"valid_at":"2032-09-02T12:00:00Z","value":"Granite"},{"authority_grant_ids":["G1"],"claim":"Juniper supersede configuration Dawn for Ashport","event_id":"E2","operation":"supersede","principal_id":"Juniper","recorded_at":"2032-09-12T12:00:00Z","source_ref":"SOURCE-E2","target_id":"E1","valid_at":"2032-09-11T12:00:00Z","value":"Dawn"}],"evidence_inventory_role":"unscored_source_input; not delivered_text_after_cut","generator_version":"0.2.0","grants":[{"capability":"approve","grant_id":"G1","grantee":"Juniper","issuer":"Synthetic Council","scope":"Ashport","source_ref":"SOURCE-G1","valid_from":"2032-09-01T12:00:00Z","valid_to":"2032-09-22T12:00:00Z"}],"pattern":"AR-05","query_at":"2032-09-05T12:00:00Z","query_text":"Which configuration governs Ashport at 2032-09-05T12:00:00Z, and what source evidence establishes it?","schema_version":"2.0","source_coverage":{"coverage_declared_at":"2032-09-05T12:00:00Z","event_log_complete_through":"2032-09-05T12:00:00Z","grant_registry_complete":true,"registry_source_ref":"SOURCE-REGISTRY","scope":"Ashport"},"source_inventory_text":"Case AR-05-S20261022-V0002 / Scope Ashport\nAuthority grants:\n[SOURCE-G1] Juniper may approve in Ashport, 2032-09-01T12:00:00Z through 2032-09-22T12:00:00Z\nEvent records:\n[SOURCE-E1] E1 Juniper approve configuration Granite for Ashport; effective=2032-09-02T12:00:00Z; recorded=2032-09-03T12:00:00Z; grants=G1; target=NONE\n[SOURCE-E2] E2 Juniper supersede configuration Dawn for Ashport; effective=2032-09-11T12:00:00Z; recorded=2032-09-12T12:00:00Z; grants=G1; target=E1\n","variant_index":2},{"case_id":"AR-06-S20261022-V0001","events":[{"authority_grant_ids":[],"claim":"Juniper approve configuration Fern for Northgate","event_id":"E1","operation":"approve","principal_id":"Juniper","recorded_at":"2038-01-09T12:00:00Z","source_ref":"SOURCE-E1","target_id":null,"valid_at":"2038-01-08T12:00:00Z","value":"Fern"}],"evidence_inventory_role":"unscored_source_input; not delivered_text_after_cut","generator_version":"0.2.0","grants":[],"pattern":"AR-06","query_at":"2038-01-11T12:00:00Z","query_text":"Which configuration governs Northgate at 2038-01-11T12:00:00Z, and what source evidence establishes it?","schema_version":"2.0","source_coverage":{"coverage_declared_at":"2038-01-11T12:00:00Z","event_log_complete_through":"2038-01-11T12:00:00Z","grant_registry_complete":true,"registry_source_ref":"SOURCE-REGISTRY","scope":"Northgate"},"source_inventory_text":"Case AR-06-S20261022-V0001 / Scope Northgate\nAuthority grants:\nNo authority grants recorded for this scope.\nEvent records:\n[SOURCE-E1] E1 Juniper approve configuration Fern for Northgate; effective=2038-01-08T12:00:00Z; recorded=2038-01-09T12:00:00Z; grants=NONE; target=NONE\n","variant_index":1},{"case_id":"AR-06-S20261022-V0002","events":[{"authority_grant_ids":[],"claim":"Elm approve configuration Amber for Silverbank","event_id":"E1","operation":"approve","principal_id":"Elm","recorded_at":"2036-01-11T12:00:00Z","source_ref":"SOURCE-E1","target_id":null,"valid_at":"2036-01-10T12:00:00Z","value":"Amber"}],"evidence_inventory_role":"unscored_source_input; not delivered_text_after_cut","generator_version":"0.2.0","grants":[],"pattern":"AR-06","query_at":"2036-01-13T12:00:00Z","query_text":"Which configuration governs Silverbank at 2036-01-13T12:00:00Z, and what source evidence establishes it?","schema_version":"2.0","source_coverage":{"coverage_declared_at":"2036-01-13T12:00:00Z","event_log_complete_through":"2036-01-13T12:00:00Z","grant_registry_complete":true,"registry_source_ref":"SOURCE-REGISTRY","scope":"Silverbank"},"source_inventory_text":"Case AR-06-S20261022-V0002 / Scope Silverbank\nAuthority grants:\nNo authority grants recorded for this scope.\nEvent records:\n[SOURCE-E1] E1 Elm approve configuration Amber for Silverbank; effective=2036-01-10T12:00:00Z; recorded=2036-01-11T12:00:00Z; grants=NONE; target=NONE\n","variant_index":2}] diff --git a/experimental/authority_resolution/examples/public_seed_20261022/expected_outcomes.json b/experimental/authority_resolution/examples/public_seed_20261022/expected_outcomes.json new file mode 100644 index 0000000..f1d3304 --- /dev/null +++ b/experimental/authority_resolution/examples/public_seed_20261022/expected_outcomes.json @@ -0,0 +1 @@ +[{"case_id":"AR-01-S20261022-V0001","conflicting_event_ids":[],"delivery_boundary":"ADEBench measured delivered_text_after_cut only; source_inventory_text never scores","expected_resolution":"RESOLVED","historical_event_ids":["E1"],"independent_dimensions":["operative_value","authority_outcome","source_event_delivery"],"interpretation_note":"Expected event and authority evidence pertains to delivered text after cut, not retrieval pool or final model answer.","operative_values":["Fern"],"required_event_ids":["E1","E2"],"required_grant_ids":["G1"],"supersession_event_ids":["E2"],"temporal_basis":"valid_at; recorded_at provenance only"},{"case_id":"AR-01-S20261022-V0002","conflicting_event_ids":[],"delivery_boundary":"ADEBench measured delivered_text_after_cut only; source_inventory_text never scores","expected_resolution":"RESOLVED","historical_event_ids":["E1"],"independent_dimensions":["operative_value","authority_outcome","source_event_delivery"],"interpretation_note":"Expected event and authority evidence pertains to delivered text after cut, not retrieval pool or final model answer.","operative_values":["Amber"],"required_event_ids":["E1","E2"],"required_grant_ids":["G1"],"supersession_event_ids":["E2"],"temporal_basis":"valid_at; recorded_at provenance only"},{"case_id":"AR-02-S20261022-V0001","conflicting_event_ids":[],"delivery_boundary":"ADEBench measured delivered_text_after_cut only; source_inventory_text never scores","disallowed_event_ids":["E2"],"expected_resolution":"RESOLVED","historical_event_ids":[],"independent_dimensions":["operative_value","authority_outcome","source_event_delivery"],"interpretation_note":"Expected event and authority evidence pertains to delivered text after cut, not retrieval pool or final model answer.","operative_values":["Ember"],"required_event_ids":["E1"],"required_grant_ids":["G1"],"supersession_event_ids":[],"temporal_basis":"valid_at; recorded_at provenance only"},{"case_id":"AR-02-S20261022-V0002","conflicting_event_ids":[],"delivery_boundary":"ADEBench measured delivered_text_after_cut only; source_inventory_text never scores","disallowed_event_ids":["E2"],"expected_resolution":"RESOLVED","historical_event_ids":[],"independent_dimensions":["operative_value","authority_outcome","source_event_delivery"],"interpretation_note":"Expected event and authority evidence pertains to delivered text after cut, not retrieval pool or final model answer.","operative_values":["Jade"],"required_event_ids":["E1"],"required_grant_ids":["G1"],"supersession_event_ids":[],"temporal_basis":"valid_at; recorded_at provenance only"},{"case_id":"AR-03-S20261022-V0001","conflicting_event_ids":["E1","E2"],"delivery_boundary":"ADEBench measured delivered_text_after_cut only; source_inventory_text never scores","expected_resolution":"CONFLICT_UNRESOLVED","historical_event_ids":[],"independent_dimensions":["operative_value","authority_outcome","source_event_delivery"],"interpretation_note":"Expected event and authority evidence pertains to delivered text after cut, not retrieval pool or final model answer.","operative_values":[],"required_event_ids":["E1","E2"],"required_grant_ids":["G1","G2"],"supersession_event_ids":[],"temporal_basis":"valid_at; recorded_at provenance only"},{"case_id":"AR-03-S20261022-V0002","conflicting_event_ids":["E1","E2"],"delivery_boundary":"ADEBench measured delivered_text_after_cut only; source_inventory_text never scores","expected_resolution":"CONFLICT_UNRESOLVED","historical_event_ids":[],"independent_dimensions":["operative_value","authority_outcome","source_event_delivery"],"interpretation_note":"Expected event and authority evidence pertains to delivered text after cut, not retrieval pool or final model answer.","operative_values":[],"required_event_ids":["E1","E2"],"required_grant_ids":["G1","G2"],"supersession_event_ids":[],"temporal_basis":"valid_at; recorded_at provenance only"},{"case_id":"AR-04-S20261022-V0001","conflicting_event_ids":[],"delivery_boundary":"ADEBench measured delivered_text_after_cut only; source_inventory_text never scores","expected_resolution":"NO_AUTHORITY","historical_event_ids":["E1"],"independent_dimensions":["operative_value","authority_outcome","source_event_delivery"],"interpretation_note":"Expected event and authority evidence pertains to delivered text after cut, not retrieval pool or final model answer.","operative_values":[],"required_event_ids":["E1","E2"],"required_grant_ids":["G1"],"supersession_event_ids":["E2"],"temporal_basis":"valid_at; recorded_at provenance only"},{"case_id":"AR-04-S20261022-V0002","conflicting_event_ids":[],"delivery_boundary":"ADEBench measured delivered_text_after_cut only; source_inventory_text never scores","expected_resolution":"NO_AUTHORITY","historical_event_ids":["E1"],"independent_dimensions":["operative_value","authority_outcome","source_event_delivery"],"interpretation_note":"Expected event and authority evidence pertains to delivered text after cut, not retrieval pool or final model answer.","operative_values":[],"required_event_ids":["E1","E2"],"required_grant_ids":["G1"],"supersession_event_ids":["E2"],"temporal_basis":"valid_at; recorded_at provenance only"},{"case_id":"AR-05-S20261022-V0001","conflicting_event_ids":[],"delivery_boundary":"ADEBench measured delivered_text_after_cut only; source_inventory_text never scores","expected_resolution":"RESOLVED","future_event_ids":["E2"],"historical_event_ids":[],"independent_dimensions":["operative_value","authority_outcome","source_event_delivery"],"interpretation_note":"Expected event and authority evidence pertains to delivered text after cut, not retrieval pool or final model answer.","operative_values":["Fern"],"required_event_ids":["E1"],"required_grant_ids":["G1"],"supersession_event_ids":[],"temporal_basis":"valid_at; recorded_at provenance only"},{"case_id":"AR-05-S20261022-V0002","conflicting_event_ids":[],"delivery_boundary":"ADEBench measured delivered_text_after_cut only; source_inventory_text never scores","expected_resolution":"RESOLVED","future_event_ids":["E2"],"historical_event_ids":[],"independent_dimensions":["operative_value","authority_outcome","source_event_delivery"],"interpretation_note":"Expected event and authority evidence pertains to delivered text after cut, not retrieval pool or final model answer.","operative_values":["Granite"],"required_event_ids":["E1"],"required_grant_ids":["G1"],"supersession_event_ids":[],"temporal_basis":"valid_at; recorded_at provenance only"},{"case_id":"AR-06-S20261022-V0001","conflicting_event_ids":[],"delivery_boundary":"ADEBench measured delivered_text_after_cut only; source_inventory_text never scores","expected_resolution":"NO_AUTHORITY_IF_COVERAGE_PROVEN","historical_event_ids":[],"incomplete_coverage_resolution":"AUTHORITY_UNRESOLVED","independent_dimensions":["operative_value","authority_outcome","source_event_delivery"],"interpretation_note":"Expected event and authority evidence pertains to delivered text after cut, not retrieval pool or final model answer.","operative_values":[],"required_coverage_refs":["SOURCE-REGISTRY"],"required_event_ids":["E1"],"required_grant_ids":[],"supersession_event_ids":[],"temporal_basis":"valid_at; recorded_at provenance only"},{"case_id":"AR-06-S20261022-V0002","conflicting_event_ids":[],"delivery_boundary":"ADEBench measured delivered_text_after_cut only; source_inventory_text never scores","expected_resolution":"NO_AUTHORITY_IF_COVERAGE_PROVEN","historical_event_ids":[],"incomplete_coverage_resolution":"AUTHORITY_UNRESOLVED","independent_dimensions":["operative_value","authority_outcome","source_event_delivery"],"interpretation_note":"Expected event and authority evidence pertains to delivered text after cut, not retrieval pool or final model answer.","operative_values":[],"required_coverage_refs":["SOURCE-REGISTRY"],"required_event_ids":["E1"],"required_grant_ids":[],"supersession_event_ids":[],"temporal_basis":"valid_at; recorded_at provenance only"}] diff --git a/experimental/authority_resolution/examples/public_seed_20261022/metadata.json b/experimental/authority_resolution/examples/public_seed_20261022/metadata.json new file mode 100644 index 0000000..03f3ed7 --- /dev/null +++ b/experimental/authority_resolution/examples/public_seed_20261022/metadata.json @@ -0,0 +1 @@ +{"case_count":12,"classification":"PUBLIC_SYNTHETIC","count_per_pattern":2,"generator_version":"0.2.0","patterns":["AR-01","AR-02","AR-03","AR-04","AR-05","AR-06"],"prng":"python.random.Random","python_version":"3.13.5","schema_version":"2.0","seed":20261022,"source_inventory_is_not_measured_delivery":true} diff --git a/experimental/authority_resolution/examples/public_seed_20261022/sha256_manifest.json b/experimental/authority_resolution/examples/public_seed_20261022/sha256_manifest.json new file mode 100644 index 0000000..33de682 --- /dev/null +++ b/experimental/authority_resolution/examples/public_seed_20261022/sha256_manifest.json @@ -0,0 +1 @@ +{"algorithm":"SHA-256","files":{"cases.json":"f6c2515f64ed6ddc2226cfef19c0e9db0426092893ee8d211195cbd119f1e0a5","expected_outcomes.json":"b8f64ef4f018958db56a4ed0a828dab006a56cfc09c26093e91cb4410232c192","generate_authority_cases.py":"a2321ffed53aaa7a9c42fb47293c8f4875581effe76bf9760b83e676fbfe4d58","metadata.json":"e169842f562b59cd3bad28e0ac8c710338f29df2712ee797cd761c9a02895908","tests/test_generator.py":"f9d4570b94002eac7ee51ccf4cf25994554b35ae9d207a73727a2d2e79e77dbc"},"generator_version":"0.2.0","scope":"public fixture files and local generator/test sources only"} diff --git a/experimental/authority_resolution/generate_authority_cases.py b/experimental/authority_resolution/generate_authority_cases.py new file mode 100644 index 0000000..5564a5d --- /dev/null +++ b/experimental/authority_resolution/generate_authority_cases.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""GovKM synthetic authority-fixture generator; no scoring or product execution.""" +import argparse +import hashlib +import json +import random +from datetime import datetime, timedelta, timezone +from pathlib import Path + +VERSION = '0.2.0' +SCHEMA_VERSION = '2.0' +MAX_SEED = 2**63 - 1 +PATTERNS = ('AR-01', 'AR-02', 'AR-03', 'AR-04', 'AR-05', 'AR-06') +NAMES = ('Aster', 'Birch', 'Cobalt', 'Delta', 'Elm', 'Fable', 'Garnet', 'Harbor', 'Ibis', 'Juniper', 'Kestrel', 'Lumen') +VALUES = ('Amber', 'Blue', 'Copper', 'Dawn', 'Ember', 'Fern', 'Granite', 'Indigo', 'Jade', 'Lilac', 'Moss', 'Pearl') +LOCATIONS = ('Northgate', 'Westhaven', 'Stonefield', 'Ashport', 'Silverbank', 'Redcliff') + + +def iso(dt): + return dt.astimezone(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') + + +def canonical(obj): + return (json.dumps(obj, ensure_ascii=True, sort_keys=True, separators=(',', ':')) + '\n').encode('utf-8') + + +def make_case(rng, pattern, idx, seed): + a, b = rng.sample(VALUES, 2) + principal, outsider = rng.sample(NAMES, 2) + project = rng.choice(LOCATIONS) + base = datetime(2031 + rng.randrange(8), rng.randrange(1, 12), rng.randrange(1, 20), 12, tzinfo=timezone.utc) + t1, t2, t3, t4 = (base + timedelta(days=d) for d in (0, 3, 9, 15)) + case_id = f'{pattern}-S{seed}-V{idx:04d}' + grants = [{ 'grant_id':'G1','issuer':'Synthetic Council', 'grantee':principal, 'capability':'approve', 'scope':project, 'valid_from':iso(t1-timedelta(days=1)), 'valid_to':iso(t4+timedelta(days=5)), 'source_ref':'SOURCE-G1'}] + if pattern == 'AR-03': + grants.append({'grant_id':'G2','issuer':'Synthetic Council','grantee':outsider,'capability':'approve','scope':project,'valid_from':iso(t1-timedelta(days=1)),'valid_to':iso(t4+timedelta(days=5)),'source_ref':'SOURCE-G2'}) + if pattern == 'AR-06': + grants = [] + events = [] + def event(eid, when, who, operation, value, grant_ids, target=None, recorded_lag=1): + events.append({'event_id':eid,'recorded_at':iso(when+timedelta(days=recorded_lag)), 'valid_at':iso(when), 'principal_id':who, 'authority_grant_ids':grant_ids,'operation':operation,'target_id':target,'value':value,'claim':f'{who} {operation} configuration {value} for {project}', 'source_ref':f'SOURCE-{eid}'}) + status='RESOLVED'; operative=[]; required=[]; conflict=[]; historical=[]; supersession=[] + if pattern == 'AR-01': + event('E1',t1,principal,'approve',a,['G1'], recorded_lag=14) + event('E2',t3,principal,'supersede',b,['G1'],target='E1') + operative=[b]; required=['E2']; historical=['E1']; supersession=['E2']; query=t4 + elif pattern == 'AR-02': + event('E1',t1,principal,'approve',a,['G1']) + event('E2',t3,outsider,'approve',b,[]) + operative=[a]; required=['E1']; historical=[]; query=t4 + elif pattern == 'AR-03': + event('E1',t1,principal,'approve',a,['G1']) + event('E2',t1,outsider,'approve',b,['G2']) + status='CONFLICT_UNRESOLVED'; required=['E1','E2']; conflict=['E1','E2']; query=t2 + elif pattern == 'AR-04': + event('E1',t1,principal,'approve',a,['G1']) + event('E2',t3,principal,'revoke',a,['G1'],target='E1') + status='NO_AUTHORITY'; required=['E1','E2']; historical=['E1']; supersession=['E2']; query=t4 + elif pattern == 'AR-05': + event('E1',t1,principal,'approve',a,['G1']) + event('E2',t3,principal,'supersede',b,['G1'],target='E1') + operative=[a]; required=['E1']; query=t2 + else: + event('E1',t1,principal,'approve',a,[]) + status='NO_AUTHORITY'; required=['E1']; query=t2 + case={'case_id':case_id,'schema_version':SCHEMA_VERSION,'generator_version':VERSION,'pattern':pattern,'variant_index':idx,'query_at':iso(query),'query_text':f'Which configuration governs {project} at {iso(query)}, and what source evidence establishes it?', 'grants':grants,'events':events} + # Only fictional source inventory. This is input to a memory under test, NEVER scored delivery. + lines=[f'Case {case_id} / Scope {project}', 'Authority grants:'] + for g in grants: lines.append(f"[{g['source_ref']}] {g['grantee']} may {g['capability']} in {project}, {g['valid_from']} through {g['valid_to']}") + if not grants: lines.append('No authority grants recorded for this scope.') + lines.append('Event records:') + for e in events: + lines.append(f"[{e['source_ref']}] {e['event_id']} {e['claim']}; effective={e['valid_at']}; recorded={e['recorded_at']}; grants={','.join(e['authority_grant_ids']) or 'NONE'}; target={e['target_id'] or 'NONE'}") + case['source_inventory_text']='\n'.join(lines)+'\n' + case['source_coverage']={'grant_registry_complete': True, 'event_log_complete_through':iso(query), 'scope':project, 'registry_source_ref':'SOURCE-REGISTRY', 'coverage_declared_at':iso(query)} + case['evidence_inventory_role']='unscored_source_input; not delivered_text_after_cut' + oracle={'case_id':case_id,'expected_resolution':status,'operative_values':operative, 'required_event_ids':required,'conflicting_event_ids':conflict,'historical_event_ids':historical,'supersession_event_ids':supersession, 'required_grant_ids':sorted(set(g for e in events if e['event_id'] in required for g in e['authority_grant_ids'])), 'interpretation_note':'Expected event and authority evidence pertains to delivered text after cut, not retrieval pool or final model answer.'} + if pattern == 'AR-01': + oracle['required_event_ids']=['E1','E2'] + oracle['required_grant_ids']=['G1'] + if pattern == 'AR-04': + oracle['required_event_ids']=['E1','E2'] + oracle['required_grant_ids']=['G1'] + if pattern == 'AR-06': + oracle['expected_resolution']='NO_AUTHORITY_IF_COVERAGE_PROVEN' + oracle['required_coverage_refs']=['SOURCE-REGISTRY'] + oracle['incomplete_coverage_resolution']='AUTHORITY_UNRESOLVED' + if pattern == 'AR-02': oracle['disallowed_event_ids']=['E2'] + if pattern == 'AR-05': oracle['future_event_ids']=['E2'] + oracle['delivery_boundary']='ADEBench measured delivered_text_after_cut only; source_inventory_text never scores' + oracle['independent_dimensions']=['operative_value','authority_outcome','source_event_delivery'] + oracle['temporal_basis']='valid_at; recorded_at provenance only' + return case, oracle + + +def generate(seed, count): + if count < 1 or count > 10000: raise ValueError('count must be between 1 and 10000') + if isinstance(seed,bool) or not isinstance(seed,int) or seed < 0 or seed > MAX_SEED: raise ValueError('seed must be an integer between 0 and 2**63-1') + rng=random.Random(seed) + cases=[]; oracles=[] + for pattern in PATTERNS: + for i in range(1,count+1): + case, oracle=make_case(rng,pattern,i,seed) + cases.append(case); oracles.append(oracle) + return cases, oracles + + +def write_set(seed,count,outdir): + out=Path(outdir) + if out.exists(): raise ValueError('output directory must not exist (no overwrite)') + if out.is_symlink(): raise ValueError('symlink output not allowed') + cases, oracles=generate(seed,count) + out.mkdir(parents=True,exist_ok=True) + assets={'cases.json':cases,'expected_outcomes.json':oracles,'metadata.json':{'generator_version':VERSION,'schema_version':SCHEMA_VERSION,'seed':seed,'count_per_pattern':count,'patterns':list(PATTERNS),'case_count':len(cases),'classification':'PUBLIC_SYNTHETIC','python_version':__import__('sys').version.split()[0],'prng':'python.random.Random','source_inventory_is_not_measured_delivery':True}} + hashes={} + for name,data in assets.items(): + buf=canonical(data) + (out/name).write_bytes(buf) + hashes[name]=hashlib.sha256(buf).hexdigest() + for codefile in (Path(__file__),Path(__file__).parent/'tests'/'test_generator.py'): + hashes[codefile.relative_to(Path(__file__).parent).as_posix()]=hashlib.sha256(codefile.read_bytes()).hexdigest() + manifest={'algorithm':'SHA-256','files':hashes,'generator_version':VERSION,'scope':'public fixture files and local generator/test sources only'} + (out/'sha256_manifest.json').write_bytes(canonical(manifest)) + return hashes + + +def main(): + p=argparse.ArgumentParser(description=__doc__) + p.add_argument('--seed',type=int,required=True) + p.add_argument('--count',type=int,required=True,help='N variants PER pattern (total 6*N)') + p.add_argument('--output-dir',type=Path,required=True) + p.add_argument('--set',choices=('public','holdout'),default='public') + args=p.parse_args() + if args.set=='holdout': p.error('private holdout generation and persistence are reserved for ADEBench; no holdout export supported') + hashes=write_set(args.seed,args.count,args.output_dir) + print(f'Generated {6*args.count} PUBLIC SYNTHETIC fixtures; SHA-256 cases.json={hashes["cases.json"]}') + +if __name__=='__main__': main() diff --git a/experimental/authority_resolution/tests/test_generator.py b/experimental/authority_resolution/tests/test_generator.py new file mode 100644 index 0000000..9897e66 --- /dev/null +++ b/experimental/authority_resolution/tests/test_generator.py @@ -0,0 +1,118 @@ +import hashlib +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT)) +from generate_authority_cases import generate, write_set, PATTERNS, canonical + +class GeneratorTests(unittest.TestCase): + def test_repeatability(self): + with tempfile.TemporaryDirectory() as td: + a=Path(td)/'a'; b=Path(td)/'b' + x=write_set(47,2,a); y=write_set(47,2,b) + self.assertEqual(x,y) + self.assertEqual(sorted(p.name for p in a.iterdir()),sorted(p.name for p in b.iterdir())) + for p in a.iterdir(): self.assertEqual(p.read_bytes(),(b/p.name).read_bytes()) + def test_different_seed_changes_fictions(self): + a,_=generate(1,2); b,_=generate(2,2) + self.assertNotEqual([x['source_inventory_text'] for x in a],[x['source_inventory_text'] for x in b]) + def test_count_uniqueness(self): + for n in (1,3): + c,o=generate(9,n) + self.assertEqual(len(c),6*n);self.assertEqual(len(o),6*n) + self.assertEqual(set(x['pattern'] for x in c),set(PATTERNS)) + self.assertEqual(len(set(x['case_id'] for x in c)),len(c)) + def test_grants_and_deliberate_invalids(self): + c,o=generate(42,1) + for case in c: + grantids={g['grant_id'] for g in case['grants']} + for e in case['events']: + self.assertTrue(set(e['authority_grant_ids']) <=grantids) + if e['authority_grant_ids']: self.assertTrue(e['principal_id'] in [g['grantee'] for g in case['grants'] if g['grant_id'] in e['authority_grant_ids']]) + self.assertFalse(c[-1]['grants']) + def test_temporal_fields(self): + c,_=generate(42,1) + for x in c: + for e in x['events']: self.assertGreaterEqual(e['recorded_at'],e['valid_at']) + self.assertGreater(c[4]['events'][1]['valid_at'],c[4]['query_at']) + def test_conflict_all_evidence(self): + c,o=generate(12,1) + self.assertEqual(o[2]['expected_resolution'],'CONFLICT_UNRESOLVED') + self.assertEqual(set(o[2]['required_event_ids']),{'E1','E2'}) + def test_retired_companion(self): + c,o=generate(12,1) + self.assertEqual(o[0]['historical_event_ids'],['E1']) + self.assertEqual(o[0]['supersession_event_ids'],['E2']) + def test_cut_metadata_not_scoring(self): + c,o=generate(12,1) + self.assertIn('after cut',o[0]['interpretation_note']) + self.assertNotIn('score',c[0]) + def test_fiction_only(self): + c,_=generate(12,1) + serialized=json.dumps(c) + for forbidden in ('govkm.com','SYN-VGR','robnoyes','Nishant','Simone','pipeshub'): + self.assertNotIn(forbidden,serialized) + def test_private_holdout_block(self): + with tempfile.TemporaryDirectory() as td: + result=subprocess.run([sys.executable,str(ROOT/'generate_authority_cases.py'),'--seed','12','--count','1','--output-dir',str(Path(td)/'hidden'),'--set','holdout'],capture_output=True,text=True) + self.assertNotEqual(result.returncode,0) + self.assertFalse((Path(td)/'hidden').exists()) + +if __name__=='__main__': unittest.main() + +class ProtocolV02Tests(unittest.TestCase): + def test_inventory_is_not_measured_delivery(self): + c, o = generate(13, 1) + for case, oracle in zip(c, o): + self.assertNotIn('public_delivery_text', case) + self.assertNotIn('delivered_text_after_cut', case) + self.assertIn('unscored_source_input', case['evidence_inventory_role']) + self.assertIn('delivered_text_after_cut only', oracle['delivery_boundary']) + self.assertNotIn('operative_values', case) + + def test_supersession_requires_original_and_replacement_sources(self): + _, oracle = generate(13, 1) + self.assertEqual(oracle[0]['required_event_ids'], ['E1', 'E2']) + self.assertIn('G1', oracle[0]['required_grant_ids']) + + def test_revocation_requires_authority_chain(self): + _, oracle = generate(13, 1) + self.assertEqual(oracle[3]['required_event_ids'], ['E1', 'E2']) + self.assertEqual(oracle[3]['required_grant_ids'], ['G1']) + + def test_missing_grants_provisional_without_coverage(self): + c, o = generate(13, 1) + self.assertEqual(o[5]['expected_resolution'],'NO_AUTHORITY_IF_COVERAGE_PROVEN') + self.assertEqual(o[5]['incomplete_coverage_resolution'],'AUTHORITY_UNRESOLVED') + self.assertIn('SOURCE-REGISTRY',o[5]['required_coverage_refs']) + self.assertTrue(c[5]['source_coverage']['grant_registry_complete']) + + def test_independent_dimensions_and_temporal_basis(self): + _, o = generate(13, 1) + for row in o: + self.assertEqual(len(row['independent_dimensions']), 3) + self.assertIn('valid_at', row['temporal_basis']) + + def test_non_overwrite_and_seed_limits(self): + from generate_authority_cases import write_set + with tempfile.TemporaryDirectory() as d: + folder=Path(d)/'public' + write_set(99, 1, folder) + with self.assertRaises(ValueError): write_set(99, 1, folder) + for seed in (-1,2**63,True): + with self.assertRaises(ValueError): generate(seed,1) + + def test_manifest_includes_source_and_test(self): + from generate_authority_cases import write_set + with tempfile.TemporaryDirectory() as d: + folder=Path(d)/'public' + write_set(13,1,folder) + manifest=json.loads((folder/'sha256_manifest.json').read_text()) + for key in ('generate_authority_cases.py','tests/test_generator.py','cases.json','expected_outcomes.json','metadata.json'): + self.assertIn(key,manifest['files']) +