diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0bd1357..6a7b87f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,8 @@ jobs: cmp benchmarks/real-mutation-results/evaluation/summary.json /tmp/real-mutations/summary.json python scripts/compare_replay.py benchmarks/real-mutation-results/evaluation/cases.json /tmp/real-mutations/cases.json --old-version 0.1.0a2 --new-version 0.1.0a4 --report-path result.report python examples/five_minute_demo.py + - name: Replay immutable validator studies with MateProbe + run: python benchmarks/study_replay.py --output /tmp/mateprobe-study-replay - name: Audit existing validators offline run: python examples/audit_existing_validator.py --output /tmp/validator-audit - run: python -m build && python -m build packages/pytest-mateprobe diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 07346d6..76ee4e3 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -28,6 +28,7 @@ jobs: -r requirements-integrations.txt - run: python scripts/check_alpha_install.py --output /tmp/published-docs-evidence - run: python scripts/check_launch_package.py --output /tmp/launch-evidence + - run: python benchmarks/study_replay.py --output /tmp/published-study-replay - uses: actions/upload-artifact@v4 if: always() with: @@ -35,6 +36,7 @@ jobs: path: | /tmp/published-docs-evidence /tmp/launch-evidence + /tmp/published-study-replay historical-a3-trial: runs-on: ubuntu-latest @@ -46,11 +48,27 @@ jobs: - name: Install the historical a3 trial dependencies run: python -m pip install narrative-contracts==0.1.0a3 pytest-narrative-contracts==0.1.0a3 - run: python benchmarks/replay_agent_readiness.py --output /tmp/a3-agent-readiness + - name: Replay frozen a3 studies byte-for-byte + run: | + python benchmarks/validator_study.py replay --prepared benchmarks/validator-study/prepared --captured benchmarks/validator-study/evaluation --output /tmp/a3-controlled + python benchmarks/historical_validator_study.py replay --captured benchmarks/historical-validator-results/captured --output /tmp/a3-historical + python benchmarks/historical_followup.py replay --captured benchmarks/historical-validator-results/compatibility-followup-v2/captured --output /tmp/a3-followup + for file in summary.json audit-reports.json comparisons.json observations.json; do + cmp "benchmarks/validator-study/evaluation/$file" "/tmp/a3-controlled/$file" + done + for file in summary.json audit-reports.json; do + cmp "benchmarks/historical-validator-results/evaluation/$file" "/tmp/a3-historical/$file" + cmp "benchmarks/historical-validator-results/compatibility-followup-v2/evaluation/$file" "/tmp/a3-followup/$file" + done - uses: actions/upload-artifact@v4 if: always() with: - name: historical-a3-agent-readiness - path: /tmp/a3-agent-readiness + name: historical-a3-agent-readiness-and-studies + path: | + /tmp/a3-agent-readiness + /tmp/a3-controlled + /tmp/a3-historical + /tmp/a3-followup historical-a2-trial: runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index 91a58cb..7215c85 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,6 +21,14 @@ and `pydantic==2.13.5`, then run `python scripts/check_published_docs.py`; an editable checkout cannot verify published compatibility. See [the agent integration guide](docs/agent-guide.md) for consumer instructions. +## Project identity + +The current project is **MateProbe by Mate4B** (`mateprobe`, `pytest-mateprobe`, a4). +Frozen a3 study drivers, protocols, captures and hashes retain their original names +and source bytes. Use `benchmarks/study_replay.py` to replay those studies against +MateProbe; direct historical commands require an isolated a3 environment. +See docs/project-name.md and docs/validator-study.md. + ## Version boundary The current release is `0.1.0a4`, including independent audits and state bindings. diff --git a/README.md b/README.md index 3af2840..a0d807b 100644 --- a/README.md +++ b/README.md @@ -244,6 +244,15 @@ trusted Python state transitions, checks branch selection and declarations, and valid paraphrases and failures that remain outside the prose guarantee. It is independent of LifeCard; this is an executable second integration, not evidence of broad domain generalization. +## Controlled and historical validator evidence + +The [a3 validator study](docs/validator-study.md) compares 144 paired cases under eight +authored profiles. Detailed Python assertions and the audit agree on all 1,152 +classifications; boolean rejection alone loses diagnostic attribution. A separate +version-pinned study reproduces three public historical fixes across jsonschema and +Marshmallow, retaining a fourth unsuccessful reproduction and all compatibility +failures. This is author-run evidence, not external adoption or natural-text accuracy. + ## Reproduce or challenge the results ```sh diff --git a/benchmarks/historical-validator-results/README.md b/benchmarks/historical-validator-results/README.md new file mode 100644 index 0000000..26d1742 --- /dev/null +++ b/benchmarks/historical-validator-results/README.md @@ -0,0 +1,29 @@ +# Historical evidence: read the final and initial results together + +The final compatibility run is `compatibility-followup-v2/evaluation/summary.json`: +three reproduced historical fixes across two packages and one non-reproduction. +There are four issue comparisons, eight changed pairs, and two versions per issue. +These are independently originated reports reproduced by this project's authors, +not independent users of Narrative Contracts or newly discovered defects. + +- `preflight-v1/`: flawed initial preparation preserved for transparency; not results. +- `prepared-v2-bootstrap/`: preparation before the macOS venv bootstrap correction. +- `prepared/`: original executed inputs, adapter and protocol snapshots. +- `captured/` and `evaluation/`: original run, with one reproduced fix, unavailable + jsonschema 4.17.2, missing pkg_resources in old versions and the unchanged IDN case. +- `compatibility-followup/`: first compatibility run; a conflicting setuptools pin + prevented two environments from installing. Other results and the driver are retained. +- `compatibility-followup-v2/`: corrected, pinned dependency environments, raw + outcomes, artifact hashes, locks, and final replay results. Case labels and inputs + were never changed to improve results after the original package execution. + +The initial screening ledger is purposive, with ten candidates and six exclusions. +Null versions on excluded entries mean a validation-fix pair was not established. +Preflight-v1 guesses must not be used as established release information. + +See [the study](../../docs/validator-study.md), [screening protocol](../../paper/historical-validator-protocol.md) +and [follow-up deviations](../../paper/historical-followup-protocol.md). Reproduction +commands distinguish offline replay from package downloads and fresh execution. +No package installations or inference calls are needed for CI replay. The original +and final runs are checked byte-for-byte by the test suite; the intermediate +compatibility run is an archived failed setup attempt, not the recommended entrypoint. diff --git a/benchmarks/historical-validator-results/captured/captured.json b/benchmarks/historical-validator-results/captured/captured.json new file mode 100644 index 0000000..37a8609 --- /dev/null +++ b/benchmarks/historical-validator-results/captured/captured.json @@ -0,0 +1,423 @@ +{ + "jsonschema-bool-ref-cache": { + "4.17.2": { + "install_returncode": 1, + "setup_error": "ERROR: Ignored the following yanked versions: 4.0.0\nERROR: Could not find a version that satisfies the requirement jsonschema==4.17.2 (from versions: 0.1a0, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8.0, 1.0.0, 1.1.0, 1.2.0, 1.3.0, 2.0.0, 2.1.0, 2.2.0, 2.3.0, 2.4.0, 2.5.0, 2.5.1, 2.6.0, 3.0.0a1, 3.0.0a2, 3.0.0a3, 3.0.0a4, 3.0.0a5, 3.0.0a6, 3.0.0b1, 3.0.0b2, 3.0.0b3, 3.0.0, 3.0.1, 3.0.2, 3.1.0, 3.1.1, 3.2.0, 4.0.0a1, 4.0.0a2, 4.0.0a3, 4.0.0a4, 4.0.0a5, 4.0.0a6, 4.0.1, 4.1.0a1, 4.1.0, 4.1.1, 4.1.2, 4.2.0, 4.2.1, 4.3.0, 4.3.1, 4.3.2, 4.3.3, 4.4.0, 4.5.0, 4.5.1, 4.6.0, 4.6.1, 4.6.2, 4.7.0, 4.7.1, 4.7.2, 4.8.0, 4.9.0, 4.9.1, 4.10.0, 4.10.1, 4.10.2, 4.10.3, 4.11.0, 4.12.0, 4.12.1, 4.13.0, 4.14.0, 4.15.0, 4.16.0, 4.17.0, 4.17.1, 4.17.3, 4.18.0a1, 4.18.0a2, 4.18.0a3, 4.18.0a4, 4.18.0a5, 4.18.0a6, 4.18.0a7, 4.18.0a8, 4.18.0a9, 4.18.0a10, 4.18.0, 4.18.1, 4.18.2, 4.18.3, 4.18.4, 4.18.5, 4.18.6, 4.19.0, 4.19.1, 4.19.2, 4.20.0, 4.21.0, 4.21.1, 4.22.0, 4.23.0, 4.24.0, 4.24.1, 4.25.0, 4.25.1, 4.26.0)\nERROR: No matching distribution found for jsonschema==4.17.2\n" + }, + "4.17.3": { + "artifacts": [ + { + "hashes": { + "sha256": "a870ad254da1a8ca84b6a2905cac29d265f805acc57af304784962a2aa6508f6" + }, + "name": "jsonschema", + "url": "https://files.pythonhosted.org/packages/c1/97/c698bd9350f307daad79dd740806e1a59becd693bd11443a0f531e3229b3/jsonschema-4.17.3-py3-none-any.whl", + "version": "4.17.3" + }, + { + "hashes": { + "sha256": "c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309" + }, + "name": "attrs", + "url": "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", + "version": "26.1.0" + }, + { + "hashes": { + "sha256": "09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e" + }, + "name": "pyrsistent", + "url": "https://files.pythonhosted.org/packages/15/ee/ff2ed52032ac1ce2e7ba19e79bd5b05d152ebfb77956cf08fcd6e8d760ea/pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", + "version": "0.20.0" + } + ], + "install_returncode": 0, + "lock_sha256": "839c46ca0b682f2f69ae74c921b15761456ea9107d3fb315ca2e0d7941107f8a", + "payload": { + "package_version": "4.17.3", + "python": "3.12.13", + "results": [ + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + } + ] + }, + "returncode": 0, + "stderr": "", + "stdout": "{\"package_version\": \"4.17.3\", \"python\": \"3.12.13\", \"results\": [{\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}]}\n" + } + }, + "jsonschema-enum-bool-int": { + "3.0.1": { + "artifacts": [ + { + "hashes": { + "sha256": "a5f6559964a3851f59040d3b961de5e68e70971afb88ba519d27e6a039efff1a" + }, + "name": "jsonschema", + "url": "https://files.pythonhosted.org/packages/aa/69/df679dfbdd051568b53c38ec8152a3ab6bc533434fc7ed11ab034bf5e82f/jsonschema-3.0.1-py2.py3-none-any.whl", + "version": "3.0.1" + }, + { + "hashes": { + "sha256": "c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309" + }, + "name": "attrs", + "url": "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", + "version": "26.1.0" + }, + { + "hashes": { + "sha256": "09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e" + }, + "name": "pyrsistent", + "url": "https://files.pythonhosted.org/packages/15/ee/ff2ed52032ac1ce2e7ba19e79bd5b05d152ebfb77956cf08fcd6e8d760ea/pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", + "version": "0.20.0" + }, + { + "hashes": { + "sha256": "4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274" + }, + "name": "six", + "url": "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", + "version": "1.17.0" + }, + { + "hashes": { + "sha256": "51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670" + }, + "name": "setuptools", + "url": "https://files.pythonhosted.org/packages/95/9c/c510029fc6ef33a6275cd2c5d3cecd6613dfd6aa401d57c54f1c18852ccf/setuptools-84.0.0-py3-none-any.whl", + "version": "84.0.0" + } + ], + "install_returncode": 0, + "lock_sha256": "c8f12a893bfcd6b9a55dbef0d18fcbb5cc3071f0d9d70e3d84c0f062c1142523", + "payload": { + "package_version": "3.0.1", + "python": "3.12.13", + "results": [ + { + "error": "ModuleNotFoundError: No module named 'pkg_resources'" + }, + { + "error": "ModuleNotFoundError: No module named 'pkg_resources'" + }, + { + "error": "ModuleNotFoundError: No module named 'pkg_resources'" + }, + { + "error": "ModuleNotFoundError: No module named 'pkg_resources'" + } + ] + }, + "returncode": 0, + "stderr": "", + "stdout": "{\"package_version\": \"3.0.1\", \"python\": \"3.12.13\", \"results\": [{\"error\": \"ModuleNotFoundError: No module named 'pkg_resources'\"}, {\"error\": \"ModuleNotFoundError: No module named 'pkg_resources'\"}, {\"error\": \"ModuleNotFoundError: No module named 'pkg_resources'\"}, {\"error\": \"ModuleNotFoundError: No module named 'pkg_resources'\"}]}\n" + }, + "3.0.2": { + "artifacts": [ + { + "hashes": { + "sha256": "5f9c0a719ca2ce14c5de2fd350a64fd2d13e8539db29836a86adc990bb1a068f" + }, + "name": "jsonschema", + "url": "https://files.pythonhosted.org/packages/54/48/f5f11003ceddcd4ad292d4d9b5677588e9169eef41f88e38b2888e7ec6c4/jsonschema-3.0.2-py2.py3-none-any.whl", + "version": "3.0.2" + }, + { + "hashes": { + "sha256": "c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309" + }, + "name": "attrs", + "url": "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", + "version": "26.1.0" + }, + { + "hashes": { + "sha256": "09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e" + }, + "name": "pyrsistent", + "url": "https://files.pythonhosted.org/packages/15/ee/ff2ed52032ac1ce2e7ba19e79bd5b05d152ebfb77956cf08fcd6e8d760ea/pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", + "version": "0.20.0" + }, + { + "hashes": { + "sha256": "4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274" + }, + "name": "six", + "url": "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", + "version": "1.17.0" + }, + { + "hashes": { + "sha256": "51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670" + }, + "name": "setuptools", + "url": "https://files.pythonhosted.org/packages/95/9c/c510029fc6ef33a6275cd2c5d3cecd6613dfd6aa401d57c54f1c18852ccf/setuptools-84.0.0-py3-none-any.whl", + "version": "84.0.0" + } + ], + "install_returncode": 0, + "lock_sha256": "d1d2d4e7390399505321ce57ddf20c3e67a83e84dedaf7189f51d1819000da3d", + "payload": { + "package_version": "3.0.2", + "python": "3.12.13", + "results": [ + { + "error": "ModuleNotFoundError: No module named 'pkg_resources'" + }, + { + "error": "ModuleNotFoundError: No module named 'pkg_resources'" + }, + { + "error": "ModuleNotFoundError: No module named 'pkg_resources'" + }, + { + "error": "ModuleNotFoundError: No module named 'pkg_resources'" + } + ] + }, + "returncode": 0, + "stderr": "", + "stdout": "{\"package_version\": \"3.0.2\", \"python\": \"3.12.13\", \"results\": [{\"error\": \"ModuleNotFoundError: No module named 'pkg_resources'\"}, {\"error\": \"ModuleNotFoundError: No module named 'pkg_resources'\"}, {\"error\": \"ModuleNotFoundError: No module named 'pkg_resources'\"}, {\"error\": \"ModuleNotFoundError: No module named 'pkg_resources'\"}]}\n" + } + }, + "marshmallow-file-url": { + "4.2.0": { + "artifacts": [ + { + "hashes": { + "sha256": "1dc369bd13a8708a9566d6f73d1db07d50142a7580f04fd81e1c29a4d2e10af4" + }, + "name": "marshmallow", + "url": "https://files.pythonhosted.org/packages/1c/b6/0a907f92c2158c9841da0227c7074ce1490f578f34d67cbba82ba8f9146e/marshmallow-4.2.0-py3-none-any.whl", + "version": "4.2.0" + } + ], + "install_returncode": 0, + "lock_sha256": "42d87bc7dbcbece58f644985e692238b2a6273d097e85deffc337316a6aa0ca0", + "payload": { + "package_version": "4.2.0", + "python": "3.12.13", + "results": [ + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": false, + "complete": true, + "error": null, + "messages": [ + "['Not a valid URL.']" + ], + "violations": [ + "field@$" + ] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + } + ] + }, + "returncode": 0, + "stderr": "", + "stdout": "{\"package_version\": \"4.2.0\", \"python\": \"3.12.13\", \"results\": [{\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": false, \"complete\": true, \"error\": null, \"messages\": [\"['Not a valid URL.']\"], \"violations\": [\"field@$\"]}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}]}\n" + }, + "4.2.1": { + "artifacts": [ + { + "hashes": { + "sha256": "d82b1a83cfbb4667d050850fbed4e9d4435576cb95f5ff37894f375dce201768" + }, + "name": "marshmallow", + "url": "https://files.pythonhosted.org/packages/6c/41/e8cfdebc3a475c397e074ceaa12e86dca1b23e8e10c2ab4d4875f9a01418/marshmallow-4.2.1-py3-none-any.whl", + "version": "4.2.1" + } + ], + "install_returncode": 0, + "lock_sha256": "58dc973cc15ec968b9e44708e7ea902faaddb5a380096a4f794ae370dc944b02", + "payload": { + "package_version": "4.2.1", + "python": "3.12.13", + "results": [ + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + } + ] + }, + "returncode": 0, + "stderr": "", + "stdout": "{\"package_version\": \"4.2.1\", \"python\": \"3.12.13\", \"results\": [{\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}]}\n" + } + }, + "marshmallow-idn-email": { + "4.2.3": { + "artifacts": [ + { + "hashes": { + "sha256": "c84fd89817ecea690bde1eb925036070e5e9883148217585adc56d5cfbc082b8" + }, + "name": "marshmallow", + "url": "https://files.pythonhosted.org/packages/58/2e/ce3278954d44903ccf8606c2a16e6b3ad8e10146c21de1b7b9b0f5812b23/marshmallow-4.2.3-py3-none-any.whl", + "version": "4.2.3" + } + ], + "install_returncode": 0, + "lock_sha256": "1d7fa2cd4b51cd71b9a58cfd7bfb0a657d9f94dfc44cf957829af622faa14e57", + "payload": { + "package_version": "4.2.3", + "python": "3.12.13", + "results": [ + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + } + ] + }, + "returncode": 0, + "stderr": "", + "stdout": "{\"package_version\": \"4.2.3\", \"python\": \"3.12.13\", \"results\": [{\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}]}\n" + }, + "4.2.4": { + "artifacts": [ + { + "hashes": { + "sha256": "06f3a7ec7081e54e8026a8bd5e7c4e0e88bff82a8fa33714a33166917f2b353a" + }, + "name": "marshmallow", + "url": "https://files.pythonhosted.org/packages/dc/df/7d6d6625ec822d5adf05b55a1a0f53e2f03627c6e841ea5655310b8fda22/marshmallow-4.2.4-py3-none-any.whl", + "version": "4.2.4" + } + ], + "install_returncode": 0, + "lock_sha256": "201c63e7fd2b26bde60fece7e85d6f35d651e3362edfa787cf6e536cd5fdd14b", + "payload": { + "package_version": "4.2.4", + "python": "3.12.13", + "results": [ + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + } + ] + }, + "returncode": 0, + "stderr": "", + "stdout": "{\"package_version\": \"4.2.4\", \"python\": \"3.12.13\", \"results\": [{\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}]}\n" + } + } +} diff --git a/benchmarks/historical-validator-results/captured/jsonschema-3.0.1.lock.txt b/benchmarks/historical-validator-results/captured/jsonschema-3.0.1.lock.txt new file mode 100644 index 0000000..e330a96 --- /dev/null +++ b/benchmarks/historical-validator-results/captured/jsonschema-3.0.1.lock.txt @@ -0,0 +1,5 @@ +attrs==26.1.0 +jsonschema==3.0.1 +pyrsistent==0.20.0 +setuptools==84.0.0 +six==1.17.0 diff --git a/benchmarks/historical-validator-results/captured/jsonschema-3.0.2.lock.txt b/benchmarks/historical-validator-results/captured/jsonschema-3.0.2.lock.txt new file mode 100644 index 0000000..a7b31b2 --- /dev/null +++ b/benchmarks/historical-validator-results/captured/jsonschema-3.0.2.lock.txt @@ -0,0 +1,5 @@ +attrs==26.1.0 +jsonschema==3.0.2 +pyrsistent==0.20.0 +setuptools==84.0.0 +six==1.17.0 diff --git a/benchmarks/historical-validator-results/captured/jsonschema-4.17.3.lock.txt b/benchmarks/historical-validator-results/captured/jsonschema-4.17.3.lock.txt new file mode 100644 index 0000000..184ab11 --- /dev/null +++ b/benchmarks/historical-validator-results/captured/jsonschema-4.17.3.lock.txt @@ -0,0 +1,3 @@ +attrs==26.1.0 +jsonschema==4.17.3 +pyrsistent==0.20.0 diff --git a/benchmarks/historical-validator-results/captured/manifest.json b/benchmarks/historical-validator-results/captured/manifest.json new file mode 100644 index 0000000..d9fbaa6 --- /dev/null +++ b/benchmarks/historical-validator-results/captured/manifest.json @@ -0,0 +1,13 @@ +{ + "files": { + "captured.json": "d2459059f2a64c2f7a2ce508d3a9de3ca0be70f0c15d94c48bc35d7e4a7482a8", + "jsonschema-3.0.1.lock.txt": "c8f12a893bfcd6b9a55dbef0d18fcbb5cc3071f0d9d70e3d84c0f062c1142523", + "jsonschema-3.0.2.lock.txt": "d1d2d4e7390399505321ce57ddf20c3e67a83e84dedaf7189f51d1819000da3d", + "jsonschema-4.17.3.lock.txt": "839c46ca0b682f2f69ae74c921b15761456ea9107d3fb315ca2e0d7941107f8a", + "marshmallow-4.2.0.lock.txt": "42d87bc7dbcbece58f644985e692238b2a6273d097e85deffc337316a6aa0ca0", + "marshmallow-4.2.1.lock.txt": "58dc973cc15ec968b9e44708e7ea902faaddb5a380096a4f794ae370dc944b02", + "marshmallow-4.2.3.lock.txt": "1d7fa2cd4b51cd71b9a58cfd7bfb0a657d9f94dfc44cf957829af622faa14e57", + "marshmallow-4.2.4.lock.txt": "201c63e7fd2b26bde60fece7e85d6f35d651e3362edfa787cf6e536cd5fdd14b" + }, + "prepared_manifest_sha256": "3f8b51289a0caed4c608f9a26ddf7b9575ff9bb577cf570b34489375a7353543" +} diff --git a/benchmarks/historical-validator-results/captured/marshmallow-4.2.0.lock.txt b/benchmarks/historical-validator-results/captured/marshmallow-4.2.0.lock.txt new file mode 100644 index 0000000..410f96d --- /dev/null +++ b/benchmarks/historical-validator-results/captured/marshmallow-4.2.0.lock.txt @@ -0,0 +1 @@ +marshmallow==4.2.0 diff --git a/benchmarks/historical-validator-results/captured/marshmallow-4.2.1.lock.txt b/benchmarks/historical-validator-results/captured/marshmallow-4.2.1.lock.txt new file mode 100644 index 0000000..8592200 --- /dev/null +++ b/benchmarks/historical-validator-results/captured/marshmallow-4.2.1.lock.txt @@ -0,0 +1 @@ +marshmallow==4.2.1 diff --git a/benchmarks/historical-validator-results/captured/marshmallow-4.2.3.lock.txt b/benchmarks/historical-validator-results/captured/marshmallow-4.2.3.lock.txt new file mode 100644 index 0000000..f15c45d --- /dev/null +++ b/benchmarks/historical-validator-results/captured/marshmallow-4.2.3.lock.txt @@ -0,0 +1 @@ +marshmallow==4.2.3 diff --git a/benchmarks/historical-validator-results/captured/marshmallow-4.2.4.lock.txt b/benchmarks/historical-validator-results/captured/marshmallow-4.2.4.lock.txt new file mode 100644 index 0000000..90e1042 --- /dev/null +++ b/benchmarks/historical-validator-results/captured/marshmallow-4.2.4.lock.txt @@ -0,0 +1 @@ +marshmallow==4.2.4 diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/captured/captured.json b/benchmarks/historical-validator-results/compatibility-followup-v2/captured/captured.json new file mode 100644 index 0000000..fb953dd --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/captured/captured.json @@ -0,0 +1,518 @@ +{ + "jsonschema-bool-ref-cache": { + "4.17.1": { + "artifacts": [ + { + "hashes": { + "sha256": "c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309" + }, + "name": "attrs", + "url": "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", + "version": "26.1.0" + }, + { + "hashes": { + "sha256": "410ef23dcdbca4eaedc08b850079179883c2ed09378bd1f760d4af4aacfa28d7" + }, + "name": "jsonschema", + "url": "https://files.pythonhosted.org/packages/9f/df/824fdaa0d7228fa2e8a5171a408dbabe2c66955afd5be5211725389640b5/jsonschema-4.17.1-py3-none-any.whl", + "version": "4.17.1" + }, + { + "hashes": { + "sha256": "09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e" + }, + "name": "pyrsistent", + "url": "https://files.pythonhosted.org/packages/15/ee/ff2ed52032ac1ce2e7ba19e79bd5b05d152ebfb77956cf08fcd6e8d760ea/pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", + "version": "0.20.0" + } + ], + "install_returncode": 0, + "lock_sha256": "cbcdc8d2ed03fbdbf57a9aaa3c6dc09bb67ca99ae89d16901549ce33d737c9c0", + "payload": { + "package_version": "4.17.1", + "python": "3.12.13", + "results": [ + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "error": "TypeError: argument of type 'bool' is not iterable" + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + } + ] + }, + "returncode": 0, + "stderr": "", + "stdout": "{\"package_version\": \"4.17.1\", \"python\": \"3.12.13\", \"results\": [{\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"error\": \"TypeError: argument of type 'bool' is not iterable\"}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}]}\n" + }, + "4.17.3": { + "artifacts": [ + { + "hashes": { + "sha256": "c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309" + }, + "name": "attrs", + "url": "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", + "version": "26.1.0" + }, + { + "hashes": { + "sha256": "a870ad254da1a8ca84b6a2905cac29d265f805acc57af304784962a2aa6508f6" + }, + "name": "jsonschema", + "url": "https://files.pythonhosted.org/packages/c1/97/c698bd9350f307daad79dd740806e1a59becd693bd11443a0f531e3229b3/jsonschema-4.17.3-py3-none-any.whl", + "version": "4.17.3" + }, + { + "hashes": { + "sha256": "09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e" + }, + "name": "pyrsistent", + "url": "https://files.pythonhosted.org/packages/15/ee/ff2ed52032ac1ce2e7ba19e79bd5b05d152ebfb77956cf08fcd6e8d760ea/pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", + "version": "0.20.0" + } + ], + "install_returncode": 0, + "lock_sha256": "839c46ca0b682f2f69ae74c921b15761456ea9107d3fb315ca2e0d7941107f8a", + "payload": { + "package_version": "4.17.3", + "python": "3.12.13", + "results": [ + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + } + ] + }, + "returncode": 0, + "stderr": "", + "stdout": "{\"package_version\": \"4.17.3\", \"python\": \"3.12.13\", \"results\": [{\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}]}\n" + } + }, + "jsonschema-enum-bool-int": { + "3.0.1": { + "artifacts": [ + { + "hashes": { + "sha256": "c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309" + }, + "name": "attrs", + "url": "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", + "version": "26.1.0" + }, + { + "hashes": { + "sha256": "a5f6559964a3851f59040d3b961de5e68e70971afb88ba519d27e6a039efff1a" + }, + "name": "jsonschema", + "url": "https://files.pythonhosted.org/packages/aa/69/df679dfbdd051568b53c38ec8152a3ab6bc533434fc7ed11ab034bf5e82f/jsonschema-3.0.1-py2.py3-none-any.whl", + "version": "3.0.1" + }, + { + "hashes": { + "sha256": "09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e" + }, + "name": "pyrsistent", + "url": "https://files.pythonhosted.org/packages/15/ee/ff2ed52032ac1ce2e7ba19e79bd5b05d152ebfb77956cf08fcd6e8d760ea/pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", + "version": "0.20.0" + }, + { + "hashes": { + "sha256": "4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274" + }, + "name": "six", + "url": "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", + "version": "1.17.0" + }, + { + "hashes": { + "sha256": "fe384da74336c398e0d956d1cae0669bc02eed936cdb1d49b57de1990dc11ffc" + }, + "name": "setuptools", + "url": "https://files.pythonhosted.org/packages/ef/15/88e46eb9387e905704b69849618e699dc2f54407d8953cc4ec4b8b46528d/setuptools-70.3.0-py3-none-any.whl", + "version": "70.3.0" + } + ], + "install_returncode": 0, + "lock_sha256": "6c9323fdb37a347ad757ef9a48d859caa2d428e5105ac987977326cd7c260dbd", + "payload": { + "package_version": "3.0.1", + "python": "3.12.13", + "results": [ + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + } + ] + }, + "returncode": 0, + "stderr": "", + "stdout": "{\"package_version\": \"3.0.1\", \"python\": \"3.12.13\", \"results\": [{\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}]}\n" + }, + "3.0.2": { + "artifacts": [ + { + "hashes": { + "sha256": "c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309" + }, + "name": "attrs", + "url": "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", + "version": "26.1.0" + }, + { + "hashes": { + "sha256": "5f9c0a719ca2ce14c5de2fd350a64fd2d13e8539db29836a86adc990bb1a068f" + }, + "name": "jsonschema", + "url": "https://files.pythonhosted.org/packages/54/48/f5f11003ceddcd4ad292d4d9b5677588e9169eef41f88e38b2888e7ec6c4/jsonschema-3.0.2-py2.py3-none-any.whl", + "version": "3.0.2" + }, + { + "hashes": { + "sha256": "09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e" + }, + "name": "pyrsistent", + "url": "https://files.pythonhosted.org/packages/15/ee/ff2ed52032ac1ce2e7ba19e79bd5b05d152ebfb77956cf08fcd6e8d760ea/pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", + "version": "0.20.0" + }, + { + "hashes": { + "sha256": "4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274" + }, + "name": "six", + "url": "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", + "version": "1.17.0" + }, + { + "hashes": { + "sha256": "fe384da74336c398e0d956d1cae0669bc02eed936cdb1d49b57de1990dc11ffc" + }, + "name": "setuptools", + "url": "https://files.pythonhosted.org/packages/ef/15/88e46eb9387e905704b69849618e699dc2f54407d8953cc4ec4b8b46528d/setuptools-70.3.0-py3-none-any.whl", + "version": "70.3.0" + } + ], + "install_returncode": 0, + "lock_sha256": "6eba25f854a8c1f6b17936f8d312a0e71e529630f22f7e216c9e71727595b89e", + "payload": { + "package_version": "3.0.2", + "python": "3.12.13", + "results": [ + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": false, + "complete": true, + "error": null, + "messages": [ + "0 is not one of [False]" + ], + "violations": [ + "enum@$" + ] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + } + ] + }, + "returncode": 0, + "stderr": "", + "stdout": "{\"package_version\": \"3.0.2\", \"python\": \"3.12.13\", \"results\": [{\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": false, \"complete\": true, \"error\": null, \"messages\": [\"0 is not one of [False]\"], \"violations\": [\"enum@$\"]}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}]}\n" + } + }, + "marshmallow-file-url": { + "4.2.0": { + "artifacts": [ + { + "hashes": { + "sha256": "1dc369bd13a8708a9566d6f73d1db07d50142a7580f04fd81e1c29a4d2e10af4" + }, + "name": "marshmallow", + "url": "https://files.pythonhosted.org/packages/1c/b6/0a907f92c2158c9841da0227c7074ce1490f578f34d67cbba82ba8f9146e/marshmallow-4.2.0-py3-none-any.whl", + "version": "4.2.0" + } + ], + "install_returncode": 0, + "lock_sha256": "42d87bc7dbcbece58f644985e692238b2a6273d097e85deffc337316a6aa0ca0", + "payload": { + "package_version": "4.2.0", + "python": "3.12.13", + "results": [ + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": false, + "complete": true, + "error": null, + "messages": [ + "['Not a valid URL.']" + ], + "violations": [ + "field@$" + ] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + } + ] + }, + "returncode": 0, + "stderr": "", + "stdout": "{\"package_version\": \"4.2.0\", \"python\": \"3.12.13\", \"results\": [{\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": false, \"complete\": true, \"error\": null, \"messages\": [\"['Not a valid URL.']\"], \"violations\": [\"field@$\"]}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}]}\n" + }, + "4.2.1": { + "artifacts": [ + { + "hashes": { + "sha256": "d82b1a83cfbb4667d050850fbed4e9d4435576cb95f5ff37894f375dce201768" + }, + "name": "marshmallow", + "url": "https://files.pythonhosted.org/packages/6c/41/e8cfdebc3a475c397e074ceaa12e86dca1b23e8e10c2ab4d4875f9a01418/marshmallow-4.2.1-py3-none-any.whl", + "version": "4.2.1" + } + ], + "install_returncode": 0, + "lock_sha256": "58dc973cc15ec968b9e44708e7ea902faaddb5a380096a4f794ae370dc944b02", + "payload": { + "package_version": "4.2.1", + "python": "3.12.13", + "results": [ + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + } + ] + }, + "returncode": 0, + "stderr": "", + "stdout": "{\"package_version\": \"4.2.1\", \"python\": \"3.12.13\", \"results\": [{\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}]}\n" + } + }, + "marshmallow-idn-email": { + "4.2.3": { + "artifacts": [ + { + "hashes": { + "sha256": "c84fd89817ecea690bde1eb925036070e5e9883148217585adc56d5cfbc082b8" + }, + "name": "marshmallow", + "url": "https://files.pythonhosted.org/packages/58/2e/ce3278954d44903ccf8606c2a16e6b3ad8e10146c21de1b7b9b0f5812b23/marshmallow-4.2.3-py3-none-any.whl", + "version": "4.2.3" + } + ], + "install_returncode": 0, + "lock_sha256": "1d7fa2cd4b51cd71b9a58cfd7bfb0a657d9f94dfc44cf957829af622faa14e57", + "payload": { + "package_version": "4.2.3", + "python": "3.12.13", + "results": [ + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + } + ] + }, + "returncode": 0, + "stderr": "", + "stdout": "{\"package_version\": \"4.2.3\", \"python\": \"3.12.13\", \"results\": [{\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}]}\n" + }, + "4.2.4": { + "artifacts": [ + { + "hashes": { + "sha256": "06f3a7ec7081e54e8026a8bd5e7c4e0e88bff82a8fa33714a33166917f2b353a" + }, + "name": "marshmallow", + "url": "https://files.pythonhosted.org/packages/dc/df/7d6d6625ec822d5adf05b55a1a0f53e2f03627c6e841ea5655310b8fda22/marshmallow-4.2.4-py3-none-any.whl", + "version": "4.2.4" + } + ], + "install_returncode": 0, + "lock_sha256": "201c63e7fd2b26bde60fece7e85d6f35d651e3362edfa787cf6e536cd5fdd14b", + "payload": { + "package_version": "4.2.4", + "python": "3.12.13", + "results": [ + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + } + ] + }, + "returncode": 0, + "stderr": "", + "stdout": "{\"package_version\": \"4.2.4\", \"python\": \"3.12.13\", \"results\": [{\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}]}\n" + } + } +} diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/captured/jsonschema-3.0.1.lock.txt b/benchmarks/historical-validator-results/compatibility-followup-v2/captured/jsonschema-3.0.1.lock.txt new file mode 100644 index 0000000..f6086b3 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/captured/jsonschema-3.0.1.lock.txt @@ -0,0 +1,5 @@ +attrs==26.1.0 +jsonschema==3.0.1 +pyrsistent==0.20.0 +setuptools==70.3.0 +six==1.17.0 diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/captured/jsonschema-3.0.2.lock.txt b/benchmarks/historical-validator-results/compatibility-followup-v2/captured/jsonschema-3.0.2.lock.txt new file mode 100644 index 0000000..efe369e --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/captured/jsonschema-3.0.2.lock.txt @@ -0,0 +1,5 @@ +attrs==26.1.0 +jsonschema==3.0.2 +pyrsistent==0.20.0 +setuptools==70.3.0 +six==1.17.0 diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/captured/jsonschema-4.17.1.lock.txt b/benchmarks/historical-validator-results/compatibility-followup-v2/captured/jsonschema-4.17.1.lock.txt new file mode 100644 index 0000000..eac7b5b --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/captured/jsonschema-4.17.1.lock.txt @@ -0,0 +1,3 @@ +attrs==26.1.0 +jsonschema==4.17.1 +pyrsistent==0.20.0 diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/captured/jsonschema-4.17.3.lock.txt b/benchmarks/historical-validator-results/compatibility-followup-v2/captured/jsonschema-4.17.3.lock.txt new file mode 100644 index 0000000..184ab11 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/captured/jsonschema-4.17.3.lock.txt @@ -0,0 +1,3 @@ +attrs==26.1.0 +jsonschema==4.17.3 +pyrsistent==0.20.0 diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/captured/manifest.json b/benchmarks/historical-validator-results/compatibility-followup-v2/captured/manifest.json new file mode 100644 index 0000000..dc75bbd --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/captured/manifest.json @@ -0,0 +1,14 @@ +{ + "files": { + "captured.json": "cff5b666aacfc848c557a36bbe1c28403e05265b0918e1dca808c637f63c8875", + "jsonschema-3.0.1.lock.txt": "6c9323fdb37a347ad757ef9a48d859caa2d428e5105ac987977326cd7c260dbd", + "jsonschema-3.0.2.lock.txt": "6eba25f854a8c1f6b17936f8d312a0e71e529630f22f7e216c9e71727595b89e", + "jsonschema-4.17.1.lock.txt": "cbcdc8d2ed03fbdbf57a9aaa3c6dc09bb67ca99ae89d16901549ce33d737c9c0", + "jsonschema-4.17.3.lock.txt": "839c46ca0b682f2f69ae74c921b15761456ea9107d3fb315ca2e0d7941107f8a", + "marshmallow-4.2.0.lock.txt": "42d87bc7dbcbece58f644985e692238b2a6273d097e85deffc337316a6aa0ca0", + "marshmallow-4.2.1.lock.txt": "58dc973cc15ec968b9e44708e7ea902faaddb5a380096a4f794ae370dc944b02", + "marshmallow-4.2.3.lock.txt": "1d7fa2cd4b51cd71b9a58cfd7bfb0a657d9f94dfc44cf957829af622faa14e57", + "marshmallow-4.2.4.lock.txt": "201c63e7fd2b26bde60fece7e85d6f35d651e3362edfa787cf6e536cd5fdd14b" + }, + "prepared_manifest_sha256": "59890219a60da350c31377fa16743d8efead43ded32c1a331f72f9a47908d92b" +} diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/captured/marshmallow-4.2.0.lock.txt b/benchmarks/historical-validator-results/compatibility-followup-v2/captured/marshmallow-4.2.0.lock.txt new file mode 100644 index 0000000..410f96d --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/captured/marshmallow-4.2.0.lock.txt @@ -0,0 +1 @@ +marshmallow==4.2.0 diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/captured/marshmallow-4.2.1.lock.txt b/benchmarks/historical-validator-results/compatibility-followup-v2/captured/marshmallow-4.2.1.lock.txt new file mode 100644 index 0000000..8592200 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/captured/marshmallow-4.2.1.lock.txt @@ -0,0 +1 @@ +marshmallow==4.2.1 diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/captured/marshmallow-4.2.3.lock.txt b/benchmarks/historical-validator-results/compatibility-followup-v2/captured/marshmallow-4.2.3.lock.txt new file mode 100644 index 0000000..f15c45d --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/captured/marshmallow-4.2.3.lock.txt @@ -0,0 +1 @@ +marshmallow==4.2.3 diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/captured/marshmallow-4.2.4.lock.txt b/benchmarks/historical-validator-results/compatibility-followup-v2/captured/marshmallow-4.2.4.lock.txt new file mode 100644 index 0000000..90e1042 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/captured/marshmallow-4.2.4.lock.txt @@ -0,0 +1 @@ +marshmallow==4.2.4 diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/evaluation/audit-reports.json b/benchmarks/historical-validator-results/compatibility-followup-v2/evaluation/audit-reports.json new file mode 100644 index 0000000..7ab87cf --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/evaluation/audit-reports.json @@ -0,0 +1,785 @@ +{ + "jsonschema-bool-ref-cache/4.17.1": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "23d1edd41d6aeba96e7014ab4f960b6025b66424e01c5f6445f9e687d47c0c0b", + "expected": [], + "id": "cached-boolean-valid", + "obligation": "jsonschema-bool-ref-cache", + "outcome": "error", + "provenance": "Authored minimal reproduction of https://github.com/python-jsonschema/jsonschema/issues/1018", + "reason": "variant_error: RuntimeError: TypeError: argument of type 'bool' is not iterable", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "637d75abb2919f760e0cca7887cfffe980cd2fabdb0ea3cbce6fdddcda31f01c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "23d1edd41d6aeba96e7014ab4f960b6025b66424e01c5f6445f9e687d47c0c0b", + "expected": [], + "id": "cached-object-control", + "obligation": "jsonschema-bool-ref-cache", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/python-jsonschema/jsonschema/issues/1018", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "a1f217c76d39d422f165a35ee37ddd15f340ccbf3babf622ac450c8cf9909bdb" + } + ], + "corpus_digest": "51a25513137f3ebef8a66e4f4f413dee971902077d4e04ed8d8d25e126bace62", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Validator instantiation accepts cached boolean schemas", + "id": "jsonschema-bool-ref-cache", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "error": 1, + "preserved": 1 + }, + "detected_faults": 0, + "detection_score": null, + "eligible_controls": 2, + "eligible_faults": 0, + "obligations": [ + { + "assessment": "incomplete_evidence", + "case_ids": [ + "cached-boolean-valid", + "cached-object-control" + ], + "counts": { + "error": 1, + "preserved": 1 + }, + "description": "Validator instantiation accepts cached boolean schemas", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": true, + "has_known_gaps": false, + "id": "jsonschema-bool-ref-cache", + "scope": "contract" + } + ], + "preservation_rate": 0.5, + "preserved_controls": 1, + "total": 2 + }, + "validator_id": "jsonschema/4.17.1" + }, + "jsonschema-bool-ref-cache/4.17.3": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "23d1edd41d6aeba96e7014ab4f960b6025b66424e01c5f6445f9e687d47c0c0b", + "expected": [], + "id": "cached-boolean-valid", + "obligation": "jsonschema-bool-ref-cache", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/python-jsonschema/jsonschema/issues/1018", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "637d75abb2919f760e0cca7887cfffe980cd2fabdb0ea3cbce6fdddcda31f01c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "23d1edd41d6aeba96e7014ab4f960b6025b66424e01c5f6445f9e687d47c0c0b", + "expected": [], + "id": "cached-object-control", + "obligation": "jsonschema-bool-ref-cache", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/python-jsonschema/jsonschema/issues/1018", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "a1f217c76d39d422f165a35ee37ddd15f340ccbf3babf622ac450c8cf9909bdb" + } + ], + "corpus_digest": "51a25513137f3ebef8a66e4f4f413dee971902077d4e04ed8d8d25e126bace62", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Validator instantiation accepts cached boolean schemas", + "id": "jsonschema-bool-ref-cache", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 2 + }, + "detected_faults": 0, + "detection_score": null, + "eligible_controls": 2, + "eligible_faults": 0, + "obligations": [ + { + "assessment": "controls_only_no_faults", + "case_ids": [ + "cached-boolean-valid", + "cached-object-control" + ], + "counts": { + "preserved": 2 + }, + "description": "Validator instantiation accepts cached boolean schemas", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "jsonschema-bool-ref-cache", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 2, + "total": 2 + }, + "validator_id": "jsonschema/4.17.3" + }, + "jsonschema-enum-bool-int/3.0.1": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7a3567c53252eb64648dedc0bd2efa892f4d90c8d43ca7a3e4c95ec095426acd", + "expected": [ + "enum@$" + ], + "id": "enum-int-fault", + "obligation": "jsonschema-enum-bool-int", + "outcome": "survived", + "provenance": "Authored minimal reproduction of https://github.com/python-jsonschema/jsonschema/issues/575", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "85caeae8034324bdbc9eb23223c792bcf7a1f2730f4837aec95750e807539ffe" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7a3567c53252eb64648dedc0bd2efa892f4d90c8d43ca7a3e4c95ec095426acd", + "expected": [], + "id": "enum-valid-control", + "obligation": "jsonschema-enum-bool-int", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/python-jsonschema/jsonschema/issues/575", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "2446758f1b7798b41baa464d7c0d72d91815b392d2f797b7afd30ab1f5fba147" + } + ], + "corpus_digest": "d918c96e79ae9e403b38755f3e55e55787ed7bec5e66923471224eb534a8b12b", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "JSON Schema enum/const distinguish 0 from false", + "id": "jsonschema-enum-bool-int", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 1, + "survived": 1 + }, + "detected_faults": 0, + "detection_score": 0.0, + "eligible_controls": 1, + "eligible_faults": 1, + "obligations": [ + { + "assessment": "known_gaps", + "case_ids": [ + "enum-int-fault", + "enum-valid-control" + ], + "counts": { + "preserved": 1, + "survived": 1 + }, + "description": "JSON Schema enum/const distinguish 0 from false", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": true, + "has_incomplete_evidence": false, + "has_known_gaps": true, + "id": "jsonschema-enum-bool-int", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 1, + "total": 2 + }, + "validator_id": "jsonschema/3.0.1" + }, + "jsonschema-enum-bool-int/3.0.2": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7a3567c53252eb64648dedc0bd2efa892f4d90c8d43ca7a3e4c95ec095426acd", + "expected": [ + "enum@$" + ], + "id": "enum-int-fault", + "obligation": "jsonschema-enum-bool-int", + "outcome": "detected", + "provenance": "Authored minimal reproduction of https://github.com/python-jsonschema/jsonschema/issues/575", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [ + "0 is not one of [False]" + ], + "violations": [ + "enum@$" + ] + }, + "variant_digest": "85caeae8034324bdbc9eb23223c792bcf7a1f2730f4837aec95750e807539ffe" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7a3567c53252eb64648dedc0bd2efa892f4d90c8d43ca7a3e4c95ec095426acd", + "expected": [], + "id": "enum-valid-control", + "obligation": "jsonschema-enum-bool-int", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/python-jsonschema/jsonschema/issues/575", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "2446758f1b7798b41baa464d7c0d72d91815b392d2f797b7afd30ab1f5fba147" + } + ], + "corpus_digest": "d918c96e79ae9e403b38755f3e55e55787ed7bec5e66923471224eb534a8b12b", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "JSON Schema enum/const distinguish 0 from false", + "id": "jsonschema-enum-bool-int", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "detected": 1, + "preserved": 1 + }, + "detected_faults": 1, + "detection_score": 1.0, + "eligible_controls": 1, + "eligible_faults": 1, + "obligations": [ + { + "assessment": "no_failures_observed", + "case_ids": [ + "enum-int-fault", + "enum-valid-control" + ], + "counts": { + "detected": 1, + "preserved": 1 + }, + "description": "JSON Schema enum/const distinguish 0 from false", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": true, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "jsonschema-enum-bool-int", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 1, + "total": 2 + }, + "validator_id": "jsonschema/3.0.2" + }, + "marshmallow-file-url/4.2.0": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "8e0181a9a7ffb31fcbba12bac11a8e779ffb29d858ae6f80e2497df03de25d6a", + "expected": [], + "id": "uppercase-scheme-valid", + "obligation": "marshmallow-file-url", + "outcome": "regressed", + "provenance": "Authored minimal reproduction of https://github.com/marshmallow-code/marshmallow/issues/2891", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [ + "['Not a valid URL.']" + ], + "violations": [ + "field@$" + ] + }, + "variant_digest": "1c11eec5aa6a9d3631ee25994ad82fe370e5ebf0d0f1896c35db84ec1cbaeb31" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "8e0181a9a7ffb31fcbba12bac11a8e779ffb29d858ae6f80e2497df03de25d6a", + "expected": [], + "id": "lowercase-control", + "obligation": "marshmallow-file-url", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/marshmallow-code/marshmallow/issues/2891", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "fc2859890cc70e6029c2a2e07130c519ee56b0b8305960a2a6319888e012b5b3" + } + ], + "corpus_digest": "8372dc7ce05eca31612829b3af45404209289e23a319853266a2fd943026b844", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "URL validator handles uppercase FILE scheme", + "id": "marshmallow-file-url", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 1, + "regressed": 1 + }, + "detected_faults": 0, + "detection_score": null, + "eligible_controls": 2, + "eligible_faults": 0, + "obligations": [ + { + "assessment": "known_gaps", + "case_ids": [ + "uppercase-scheme-valid", + "lowercase-control" + ], + "counts": { + "preserved": 1, + "regressed": 1 + }, + "description": "URL validator handles uppercase FILE scheme", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": true, + "id": "marshmallow-file-url", + "scope": "contract" + } + ], + "preservation_rate": 0.5, + "preserved_controls": 1, + "total": 2 + }, + "validator_id": "marshmallow/4.2.0" + }, + "marshmallow-file-url/4.2.1": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "8e0181a9a7ffb31fcbba12bac11a8e779ffb29d858ae6f80e2497df03de25d6a", + "expected": [], + "id": "uppercase-scheme-valid", + "obligation": "marshmallow-file-url", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/marshmallow-code/marshmallow/issues/2891", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "1c11eec5aa6a9d3631ee25994ad82fe370e5ebf0d0f1896c35db84ec1cbaeb31" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "8e0181a9a7ffb31fcbba12bac11a8e779ffb29d858ae6f80e2497df03de25d6a", + "expected": [], + "id": "lowercase-control", + "obligation": "marshmallow-file-url", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/marshmallow-code/marshmallow/issues/2891", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "fc2859890cc70e6029c2a2e07130c519ee56b0b8305960a2a6319888e012b5b3" + } + ], + "corpus_digest": "8372dc7ce05eca31612829b3af45404209289e23a319853266a2fd943026b844", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "URL validator handles uppercase FILE scheme", + "id": "marshmallow-file-url", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 2 + }, + "detected_faults": 0, + "detection_score": null, + "eligible_controls": 2, + "eligible_faults": 0, + "obligations": [ + { + "assessment": "controls_only_no_faults", + "case_ids": [ + "uppercase-scheme-valid", + "lowercase-control" + ], + "counts": { + "preserved": 2 + }, + "description": "URL validator handles uppercase FILE scheme", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "marshmallow-file-url", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 2, + "total": 2 + }, + "validator_id": "marshmallow/4.2.1" + }, + "marshmallow-idn-email/4.2.3": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "e102831f1946c821ebba3d7d3e4eb088d810b857cf1169c07f5e9d16d06733e0", + "expected": [], + "id": "unicode-tld-valid", + "obligation": "marshmallow-idn-email", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/marshmallow-code/marshmallow/issues/2936", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "2d95c74f71d180a0d5ac62a81fafb19471a9d04abede546ea08e380c5789dc24" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "e102831f1946c821ebba3d7d3e4eb088d810b857cf1169c07f5e9d16d06733e0", + "expected": [], + "id": "ascii-control", + "obligation": "marshmallow-idn-email", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/marshmallow-code/marshmallow/issues/2936", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "01fd15a9b9c17e7048fc00711f58e979c1a0007be6d1e330f5f117bf706b3525" + } + ], + "corpus_digest": "be1c84141ec7eb9991b7399fdf0a9c0c209bfb654d2a24024f0a0cb14c258f33", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Email validator accepts valid internationalized domain names", + "id": "marshmallow-idn-email", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 2 + }, + "detected_faults": 0, + "detection_score": null, + "eligible_controls": 2, + "eligible_faults": 0, + "obligations": [ + { + "assessment": "controls_only_no_faults", + "case_ids": [ + "unicode-tld-valid", + "ascii-control" + ], + "counts": { + "preserved": 2 + }, + "description": "Email validator accepts valid internationalized domain names", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "marshmallow-idn-email", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 2, + "total": 2 + }, + "validator_id": "marshmallow/4.2.3" + }, + "marshmallow-idn-email/4.2.4": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "e102831f1946c821ebba3d7d3e4eb088d810b857cf1169c07f5e9d16d06733e0", + "expected": [], + "id": "unicode-tld-valid", + "obligation": "marshmallow-idn-email", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/marshmallow-code/marshmallow/issues/2936", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "2d95c74f71d180a0d5ac62a81fafb19471a9d04abede546ea08e380c5789dc24" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "e102831f1946c821ebba3d7d3e4eb088d810b857cf1169c07f5e9d16d06733e0", + "expected": [], + "id": "ascii-control", + "obligation": "marshmallow-idn-email", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/marshmallow-code/marshmallow/issues/2936", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "01fd15a9b9c17e7048fc00711f58e979c1a0007be6d1e330f5f117bf706b3525" + } + ], + "corpus_digest": "be1c84141ec7eb9991b7399fdf0a9c0c209bfb654d2a24024f0a0cb14c258f33", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Email validator accepts valid internationalized domain names", + "id": "marshmallow-idn-email", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 2 + }, + "detected_faults": 0, + "detection_score": null, + "eligible_controls": 2, + "eligible_faults": 0, + "obligations": [ + { + "assessment": "controls_only_no_faults", + "case_ids": [ + "unicode-tld-valid", + "ascii-control" + ], + "counts": { + "preserved": 2 + }, + "description": "Email validator accepts valid internationalized domain names", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "marshmallow-idn-email", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 2, + "total": 2 + }, + "validator_id": "marshmallow/4.2.4" + } +} diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/evaluation/summary.json b/benchmarks/historical-validator-results/compatibility-followup-v2/evaluation/summary.json new file mode 100644 index 0000000..c180fb0 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/evaluation/summary.json @@ -0,0 +1,77 @@ +{ + "excluded": 6, + "fixes_reproduced": 3, + "included": 4, + "results": [ + { + "after_version": "3.0.2", + "before_version": "3.0.1", + "fix_reproduced": true, + "id": "jsonschema-enum-bool-int", + "outcomes": { + "3.0.1": { + "enum-int-fault": "survived", + "enum-valid-control": "preserved" + }, + "3.0.2": { + "enum-int-fault": "detected", + "enum-valid-control": "preserved" + } + }, + "source": "https://github.com/python-jsonschema/jsonschema/issues/575" + }, + { + "after_version": "4.17.3", + "before_version": "4.17.1", + "fix_reproduced": true, + "id": "jsonschema-bool-ref-cache", + "outcomes": { + "4.17.1": { + "cached-boolean-valid": "error", + "cached-object-control": "preserved" + }, + "4.17.3": { + "cached-boolean-valid": "preserved", + "cached-object-control": "preserved" + } + }, + "source": "https://github.com/python-jsonschema/jsonschema/issues/1018" + }, + { + "after_version": "4.2.1", + "before_version": "4.2.0", + "fix_reproduced": true, + "id": "marshmallow-file-url", + "outcomes": { + "4.2.0": { + "lowercase-control": "preserved", + "uppercase-scheme-valid": "regressed" + }, + "4.2.1": { + "lowercase-control": "preserved", + "uppercase-scheme-valid": "preserved" + } + }, + "source": "https://github.com/marshmallow-code/marshmallow/issues/2891" + }, + { + "after_version": "4.2.4", + "before_version": "4.2.3", + "fix_reproduced": false, + "id": "marshmallow-idn-email", + "outcomes": { + "4.2.3": { + "ascii-control": "preserved", + "unicode-tld-valid": "preserved" + }, + "4.2.4": { + "ascii-control": "preserved", + "unicode-tld-valid": "preserved" + } + }, + "source": "https://github.com/marshmallow-code/marshmallow/issues/2936" + } + ], + "schema_version": 1, + "screened": 10 +} diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/frozen-inputs.json b/benchmarks/historical-validator-results/compatibility-followup-v2/frozen-inputs.json new file mode 100644 index 0000000..9932c7a --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/frozen-inputs.json @@ -0,0 +1,304 @@ +{ + "authored_inputs": { + "jsonschema-bool-ref-cache": [ + { + "baseline": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:ok": { + "type": "object" + } + }, + "value": {} + }, + "expected": [], + "id": "cached-boolean-valid", + "relation": "preserve", + "variant": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:any": true, + "urn:study:ok": { + "type": "object" + } + }, + "value": {} + } + }, + { + "baseline": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:ok": { + "type": "object" + } + }, + "value": {} + }, + "expected": [], + "id": "cached-object-control", + "relation": "preserve", + "variant": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:ok": { + "type": "object" + }, + "urn:study:other": { + "type": "array" + } + }, + "value": {} + } + } + ], + "jsonschema-enum-bool-int": [ + { + "baseline": { + "kind": "enum", + "schema": { + "enum": [ + false + ] + }, + "value": false + }, + "expected": [ + "enum@$" + ], + "id": "enum-int-fault", + "relation": "violation", + "variant": { + "kind": "enum", + "schema": { + "enum": [ + false + ] + }, + "value": 0 + } + }, + { + "baseline": { + "kind": "enum", + "schema": { + "enum": [ + false + ] + }, + "value": false + }, + "expected": [], + "id": "enum-valid-control", + "relation": "preserve", + "variant": { + "kind": "enum", + "schema": { + "enum": [ + false, + true + ] + }, + "value": false + } + } + ], + "marshmallow-file-url": [ + { + "baseline": { + "kind": "file", + "value": "file:///tmp/a" + }, + "expected": [], + "id": "uppercase-scheme-valid", + "relation": "preserve", + "variant": { + "kind": "file", + "value": "FILE:///tmp/a" + } + }, + { + "baseline": { + "kind": "file", + "value": "file:///tmp/a" + }, + "expected": [], + "id": "lowercase-control", + "relation": "preserve", + "variant": { + "kind": "file", + "value": "file:///tmp/b" + } + } + ], + "marshmallow-idn-email": [ + { + "baseline": { + "kind": "email", + "value": "user@example.com" + }, + "expected": [], + "id": "unicode-tld-valid", + "relation": "preserve", + "variant": { + "kind": "email", + "value": "user@пример.рф" + } + }, + { + "baseline": { + "kind": "email", + "value": "user@example.com" + }, + "expected": [], + "id": "ascii-control", + "relation": "preserve", + "variant": { + "kind": "email", + "value": "user2@example.com" + } + } + ] + }, + "candidates": [ + { + "after": "3.0.2", + "before": "3.0.1", + "behavior": "JSON Schema enum/const distinguish 0 from false", + "decision": "include", + "id": "jsonschema-enum-bool-int", + "package": "jsonschema", + "reason": "Release notes explicitly call this a bug fix and identify before/after releases.", + "release_source": "https://github.com/python-jsonschema/jsonschema/blob/51cd75e399c760e5aa3adce600dcce1385756ab0/CHANGELOG.rst", + "source": "https://github.com/python-jsonschema/jsonschema/issues/575", + "source_id": "issue #575" + }, + { + "after": "4.17.3", + "before": "4.17.1", + "behavior": "Validator instantiation accepts cached boolean schemas", + "decision": "include", + "id": "jsonschema-bool-ref-cache", + "package": "jsonschema", + "reason": "Release notes explicitly call this a bug fix with a fixed release. Follow-up uses available 4.17.1; initial 4.17.2 could not be installed.", + "release_source": "https://github.com/python-jsonschema/jsonschema/blob/51cd75e399c760e5aa3adce600dcce1385756ab0/CHANGELOG.rst", + "source": "https://github.com/python-jsonschema/jsonschema/issues/1018", + "source_id": "issue #1018" + }, + { + "after": "4.2.1", + "before": "4.2.0", + "behavior": "URL validator handles uppercase FILE scheme", + "decision": "include", + "id": "marshmallow-file-url", + "package": "marshmallow", + "reason": "Release notes identify a validation fix and adjacent releases.", + "release_source": "https://github.com/marshmallow-code/marshmallow/blob/7f0792bd7a06f72e393ec866ac1e89e1502ccfe1/CHANGELOG.rst", + "source": "https://github.com/marshmallow-code/marshmallow/issues/2891", + "source_id": "issue #2891" + }, + { + "after": "4.2.4", + "before": "4.2.3", + "behavior": "Email validator accepts valid internationalized domain names", + "decision": "include", + "id": "marshmallow-idn-email", + "package": "marshmallow", + "reason": "Release notes identify a validator behavior fix and adjacent releases.", + "release_source": "https://github.com/marshmallow-code/marshmallow/blob/7f0792bd7a06f72e393ec866ac1e89e1502ccfe1/CHANGELOG.rst", + "source": "https://github.com/marshmallow-code/marshmallow/issues/2936", + "source_id": "issue #2936" + }, + { + "after": null, + "before": null, + "behavior": "format checker is not enabled by default", + "decision": "exclude", + "id": "jsonschema-format-default", + "package": "jsonschema", + "reason": "Documented configuration/default behavior; not a library bug under the frozen rule.", + "source": "https://github.com/python-jsonschema/jsonschema/issues/945", + "source_id": "issue #945" + }, + { + "after": null, + "before": null, + "behavior": "email format checker is incomplete", + "decision": "exclude", + "id": "jsonschema-email-completeness", + "package": "jsonschema", + "reason": "Optional format-check completeness dispute; no documented before/after fix established in screening.", + "source": "https://github.com/python-jsonschema/jsonschema/issues/403", + "source_id": "issue #403" + }, + { + "after": null, + "before": null, + "behavior": "built-in generic forward references", + "decision": "exclude", + "id": "pydantic-forward-ref-generics", + "package": "pydantic", + "reason": "Model/type resolution rather than value validation; no fixed before/after release established in screening.", + "source": "https://github.com/pydantic/pydantic/issues/6130", + "source_id": "issue #6130" + }, + { + "after": null, + "before": null, + "behavior": "IPv4Address default JSON schema warning", + "decision": "exclude", + "id": "pydantic-ipv4-default", + "package": "pydantic", + "reason": "Serialization/schema warning, with no documented before/after validation fix.", + "source": "https://github.com/pydantic/pydantic/issues/8942", + "source_id": "issue #8942" + }, + { + "after": null, + "before": null, + "behavior": "default/missing parameter transition", + "decision": "exclude", + "id": "marshmallow-required-default", + "package": "marshmallow", + "reason": "Intentional API/deprecation transition and configuration semantics, not a validator defect.", + "source": "https://github.com/marshmallow-code/marshmallow/blob/dev/CHANGELOG.rst", + "source_id": "issue #1742" + }, + { + "after": null, + "before": null, + "behavior": "ModuleType serialization fails with serialize_as_any", + "decision": "exclude", + "id": "pydantic-serialize-any-regression", + "package": "pydantic", + "reason": "Serialization-only report; outside this study. No validation fix release was established.", + "source": "https://github.com/pydantic/pydantic/issues/12348", + "source_id": "issue #12348" + } + ], + "followup_driver_sha256": "4436a838c7995f02dd33943bfb45fbee229ad8bd43a982d084ab7c648960820d", + "parent_capture_manifest_sha256": "7e2c9e33a5a63fe60c57ae03508c6a203f332c86f05704615db05fbf8e14c8e5", + "preflight_revision": 2, + "requirements_sha256": { + "jsonschema-3.0.1.lock.txt": "6c2d65702342d2dbfa8b04ffb04d3105143d627730676b8146c360ff2693ba31", + "jsonschema-3.0.2.lock.txt": "9735a5cd726df701dc69da9cb7275fe7f59204f81d48f288015a864c13e785a7", + "jsonschema-4.17.1.lock.txt": "cbcdc8d2ed03fbdbf57a9aaa3c6dc09bb67ca99ae89d16901549ce33d737c9c0", + "jsonschema-4.17.3.lock.txt": "839c46ca0b682f2f69ae74c921b15761456ea9107d3fb315ca2e0d7941107f8a", + "marshmallow-4.2.0.lock.txt": "42d87bc7dbcbece58f644985e692238b2a6273d097e85deffc337316a6aa0ca0", + "marshmallow-4.2.1.lock.txt": "58dc973cc15ec968b9e44708e7ea902faaddb5a380096a4f794ae370dc944b02", + "marshmallow-4.2.3.lock.txt": "1d7fa2cd4b51cd71b9a58cfd7bfb0a657d9f94dfc44cf957829af622faa14e57", + "marshmallow-4.2.4.lock.txt": "201c63e7fd2b26bde60fece7e85d6f35d651e3362edfa787cf6e536cd5fdd14b" + } +} diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/prepared/inputs.json b/benchmarks/historical-validator-results/compatibility-followup-v2/prepared/inputs.json new file mode 100644 index 0000000..9932c7a --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/prepared/inputs.json @@ -0,0 +1,304 @@ +{ + "authored_inputs": { + "jsonschema-bool-ref-cache": [ + { + "baseline": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:ok": { + "type": "object" + } + }, + "value": {} + }, + "expected": [], + "id": "cached-boolean-valid", + "relation": "preserve", + "variant": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:any": true, + "urn:study:ok": { + "type": "object" + } + }, + "value": {} + } + }, + { + "baseline": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:ok": { + "type": "object" + } + }, + "value": {} + }, + "expected": [], + "id": "cached-object-control", + "relation": "preserve", + "variant": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:ok": { + "type": "object" + }, + "urn:study:other": { + "type": "array" + } + }, + "value": {} + } + } + ], + "jsonschema-enum-bool-int": [ + { + "baseline": { + "kind": "enum", + "schema": { + "enum": [ + false + ] + }, + "value": false + }, + "expected": [ + "enum@$" + ], + "id": "enum-int-fault", + "relation": "violation", + "variant": { + "kind": "enum", + "schema": { + "enum": [ + false + ] + }, + "value": 0 + } + }, + { + "baseline": { + "kind": "enum", + "schema": { + "enum": [ + false + ] + }, + "value": false + }, + "expected": [], + "id": "enum-valid-control", + "relation": "preserve", + "variant": { + "kind": "enum", + "schema": { + "enum": [ + false, + true + ] + }, + "value": false + } + } + ], + "marshmallow-file-url": [ + { + "baseline": { + "kind": "file", + "value": "file:///tmp/a" + }, + "expected": [], + "id": "uppercase-scheme-valid", + "relation": "preserve", + "variant": { + "kind": "file", + "value": "FILE:///tmp/a" + } + }, + { + "baseline": { + "kind": "file", + "value": "file:///tmp/a" + }, + "expected": [], + "id": "lowercase-control", + "relation": "preserve", + "variant": { + "kind": "file", + "value": "file:///tmp/b" + } + } + ], + "marshmallow-idn-email": [ + { + "baseline": { + "kind": "email", + "value": "user@example.com" + }, + "expected": [], + "id": "unicode-tld-valid", + "relation": "preserve", + "variant": { + "kind": "email", + "value": "user@пример.рф" + } + }, + { + "baseline": { + "kind": "email", + "value": "user@example.com" + }, + "expected": [], + "id": "ascii-control", + "relation": "preserve", + "variant": { + "kind": "email", + "value": "user2@example.com" + } + } + ] + }, + "candidates": [ + { + "after": "3.0.2", + "before": "3.0.1", + "behavior": "JSON Schema enum/const distinguish 0 from false", + "decision": "include", + "id": "jsonschema-enum-bool-int", + "package": "jsonschema", + "reason": "Release notes explicitly call this a bug fix and identify before/after releases.", + "release_source": "https://github.com/python-jsonschema/jsonschema/blob/51cd75e399c760e5aa3adce600dcce1385756ab0/CHANGELOG.rst", + "source": "https://github.com/python-jsonschema/jsonschema/issues/575", + "source_id": "issue #575" + }, + { + "after": "4.17.3", + "before": "4.17.1", + "behavior": "Validator instantiation accepts cached boolean schemas", + "decision": "include", + "id": "jsonschema-bool-ref-cache", + "package": "jsonschema", + "reason": "Release notes explicitly call this a bug fix with a fixed release. Follow-up uses available 4.17.1; initial 4.17.2 could not be installed.", + "release_source": "https://github.com/python-jsonschema/jsonschema/blob/51cd75e399c760e5aa3adce600dcce1385756ab0/CHANGELOG.rst", + "source": "https://github.com/python-jsonschema/jsonschema/issues/1018", + "source_id": "issue #1018" + }, + { + "after": "4.2.1", + "before": "4.2.0", + "behavior": "URL validator handles uppercase FILE scheme", + "decision": "include", + "id": "marshmallow-file-url", + "package": "marshmallow", + "reason": "Release notes identify a validation fix and adjacent releases.", + "release_source": "https://github.com/marshmallow-code/marshmallow/blob/7f0792bd7a06f72e393ec866ac1e89e1502ccfe1/CHANGELOG.rst", + "source": "https://github.com/marshmallow-code/marshmallow/issues/2891", + "source_id": "issue #2891" + }, + { + "after": "4.2.4", + "before": "4.2.3", + "behavior": "Email validator accepts valid internationalized domain names", + "decision": "include", + "id": "marshmallow-idn-email", + "package": "marshmallow", + "reason": "Release notes identify a validator behavior fix and adjacent releases.", + "release_source": "https://github.com/marshmallow-code/marshmallow/blob/7f0792bd7a06f72e393ec866ac1e89e1502ccfe1/CHANGELOG.rst", + "source": "https://github.com/marshmallow-code/marshmallow/issues/2936", + "source_id": "issue #2936" + }, + { + "after": null, + "before": null, + "behavior": "format checker is not enabled by default", + "decision": "exclude", + "id": "jsonschema-format-default", + "package": "jsonschema", + "reason": "Documented configuration/default behavior; not a library bug under the frozen rule.", + "source": "https://github.com/python-jsonschema/jsonschema/issues/945", + "source_id": "issue #945" + }, + { + "after": null, + "before": null, + "behavior": "email format checker is incomplete", + "decision": "exclude", + "id": "jsonschema-email-completeness", + "package": "jsonschema", + "reason": "Optional format-check completeness dispute; no documented before/after fix established in screening.", + "source": "https://github.com/python-jsonschema/jsonschema/issues/403", + "source_id": "issue #403" + }, + { + "after": null, + "before": null, + "behavior": "built-in generic forward references", + "decision": "exclude", + "id": "pydantic-forward-ref-generics", + "package": "pydantic", + "reason": "Model/type resolution rather than value validation; no fixed before/after release established in screening.", + "source": "https://github.com/pydantic/pydantic/issues/6130", + "source_id": "issue #6130" + }, + { + "after": null, + "before": null, + "behavior": "IPv4Address default JSON schema warning", + "decision": "exclude", + "id": "pydantic-ipv4-default", + "package": "pydantic", + "reason": "Serialization/schema warning, with no documented before/after validation fix.", + "source": "https://github.com/pydantic/pydantic/issues/8942", + "source_id": "issue #8942" + }, + { + "after": null, + "before": null, + "behavior": "default/missing parameter transition", + "decision": "exclude", + "id": "marshmallow-required-default", + "package": "marshmallow", + "reason": "Intentional API/deprecation transition and configuration semantics, not a validator defect.", + "source": "https://github.com/marshmallow-code/marshmallow/blob/dev/CHANGELOG.rst", + "source_id": "issue #1742" + }, + { + "after": null, + "before": null, + "behavior": "ModuleType serialization fails with serialize_as_any", + "decision": "exclude", + "id": "pydantic-serialize-any-regression", + "package": "pydantic", + "reason": "Serialization-only report; outside this study. No validation fix release was established.", + "source": "https://github.com/pydantic/pydantic/issues/12348", + "source_id": "issue #12348" + } + ], + "followup_driver_sha256": "4436a838c7995f02dd33943bfb45fbee229ad8bd43a982d084ab7c648960820d", + "parent_capture_manifest_sha256": "7e2c9e33a5a63fe60c57ae03508c6a203f332c86f05704615db05fbf8e14c8e5", + "preflight_revision": 2, + "requirements_sha256": { + "jsonschema-3.0.1.lock.txt": "6c2d65702342d2dbfa8b04ffb04d3105143d627730676b8146c360ff2693ba31", + "jsonschema-3.0.2.lock.txt": "9735a5cd726df701dc69da9cb7275fe7f59204f81d48f288015a864c13e785a7", + "jsonschema-4.17.1.lock.txt": "cbcdc8d2ed03fbdbf57a9aaa3c6dc09bb67ca99ae89d16901549ce33d737c9c0", + "jsonschema-4.17.3.lock.txt": "839c46ca0b682f2f69ae74c921b15761456ea9107d3fb315ca2e0d7941107f8a", + "marshmallow-4.2.0.lock.txt": "42d87bc7dbcbece58f644985e692238b2a6273d097e85deffc337316a6aa0ca0", + "marshmallow-4.2.1.lock.txt": "58dc973cc15ec968b9e44708e7ea902faaddb5a380096a4f794ae370dc944b02", + "marshmallow-4.2.3.lock.txt": "1d7fa2cd4b51cd71b9a58cfd7bfb0a657d9f94dfc44cf957829af622faa14e57", + "marshmallow-4.2.4.lock.txt": "201c63e7fd2b26bde60fece7e85d6f35d651e3362edfa787cf6e536cd5fdd14b" + } +} diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/prepared/manifest.json b/benchmarks/historical-validator-results/compatibility-followup-v2/prepared/manifest.json new file mode 100644 index 0000000..60e3848 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/prepared/manifest.json @@ -0,0 +1,5 @@ +{ + "inputs.json": "acc11e561074a04eabb46e69db7a414bb97acb523cae61fd8420766aeb7229e6", + "protocol.md": "7f85beb0d64822f6ef610f266fad1562ac99090992ec8181fccb978bc99ec74c", + "runner.py.txt": "cab4f5433556cf3c4448084ed716da2daa80eefdab1125d47fec264e83ca71df" +} diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/prepared/protocol.md b/benchmarks/historical-validator-results/compatibility-followup-v2/prepared/protocol.md new file mode 100644 index 0000000..ee4ac78 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/prepared/protocol.md @@ -0,0 +1,42 @@ +# Historical compatibility follow-up protocol + +Prepared after the first historical run and before executing this follow-up. +This is explicitly exploratory compatibility recovery, not a preplanned success. +Original ledger, inputs, failures and results remain available unchanged. + +The first run reproduced the uppercase FILE-scheme fix, but both jsonschema 3.0.x +versions failed importing pkg_resources, jsonschema 4.17.2 was unavailable from the +package index, and the chosen IDN email was accepted by both Marshmallow versions. + +Two changes are fixed in advance of the new run: + +1. Add setuptools 70.3.0 (which supplies pkg_resources) to both jsonschema 3.0.x + environments. Keep all other captured dependencies pinned. +2. Use jsonschema 4.17.1 as the available predecessor to 4.17.3. Issue #1018 describes + the regression starting in 4.16.0; the pinned changelog associates the fix with + 4.17.3. This is not evidence about the unavailable 4.17.2 package. + +All eight paired inputs, labels, policies and adapters remain identical. The IDN +case is rerun unchanged; no search for a different successful reproducer is made. +The same two Marshmallow version comparisons act as checks on repeat execution. + +The follow-up driver, base collector, protocol, dependency requirements and parent +capture identity are frozen before new collection. The driver hash is included in +the prepared inputs; requirements hashes are verified before installing. Actual +installation reports retain artifact URLs and hashes, along with pip-freeze locks. +Offline replay and fresh package execution are distinct operations. Both the +initial and follow-up tables must be reported, including failed reproduction. + +A successful reproduction requires the issue-triggering outcome to change from +incorrect to correct and the preservation control to pass in both versions. +A fixed example is a historical regression reproduced, not a newly discovered bug, +an external adopter, or evidence of a sensitivity advantage over detailed pytest. + +## Lockfile correction after the first follow-up attempt + +The first follow-up accidentally retained setuptools 84.0.0 while also requesting +70.3.0. Pip refused these two environments before executing their samples. Other +candidates ran; that capture and replay remain under `compatibility-followup/`. +Version 2 replaces the existing setuptools pin instead of appending a conflicting +one. The corrected driver and requirements were frozen again before new execution. +Case inputs and labels remain unchanged. Results of all attempts are retained. diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/prepared/runner.py.txt b/benchmarks/historical-validator-results/compatibility-followup-v2/prepared/runner.py.txt new file mode 100644 index 0000000..8017b3a --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/prepared/runner.py.txt @@ -0,0 +1,327 @@ +"""Version-pinned historical captures; isolated collection and verified offline replay.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import platform +import shutil +import subprocess +import sys +import venv +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "benchmarks/historical-validator-results" +PROTOCOL = ROOT / "paper/historical-validator-protocol.md" + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def write(path, value): + path.write_text(json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n") + + +def probe(sample): + """Only public payloads enter the third-party validator: no expected labels.""" + if sample["kind"] in ("enum", "refs"): + from jsonschema import Draft7Validator, RefResolver + + options = {} + if sample["kind"] == "refs": + options["resolver"] = RefResolver("", sample["schema"], store=sample["store"]) + errors = list(Draft7Validator(sample["schema"], **options).iter_errors(sample["value"])) + return { + "accepted": not errors, + "complete": True, + "violations": sorted({str(e.validator) + "@$" for e in errors}), + "messages": [e.message for e in errors], + "error": None, + } + from marshmallow import ValidationError, fields + + field = fields.Url(schemes={"file"}) if sample["kind"] == "file" else fields.Email() + try: + field.deserialize(sample["value"]) + except ValidationError as exc: + return { + "accepted": False, + "complete": True, + "violations": ["field@$"], + "messages": [str(exc)], + "error": None, + } + return {"accepted": True, "complete": True, "violations": [], "messages": [], "error": None} + + +def child(package): + # This process has only the pinned historical package and its dependencies. + samples = json.load(sys.stdin) + results = [] + for sample in samples: + try: + results.append(probe(sample)) + except Exception as exc: + results.append({"error": type(exc).__name__ + ": " + str(exc)}) + print( + json.dumps( + { + "python": platform.python_version(), + "package_version": importlib.metadata.version(package), + "results": results, + }, + sort_keys=True, + ) + ) + + +def freeze(): + target = OUT / "prepared" + target.mkdir(exist_ok=False) + shutil.copyfile(OUT / "frozen-inputs.json", target / "inputs.json") + shutil.copyfile(PROTOCOL, target / "protocol.md") + shutil.copyfile(__file__, target / "runner.py.txt") + write( + target / "manifest.json", + {name: sha(target / name) for name in ("inputs.json", "protocol.md", "runner.py.txt")}, + ) + + +def load_prepared(): + target = OUT / "prepared" + manifest = json.loads((target / "manifest.json").read_text()) + if set(manifest) != {"inputs.json", "protocol.md", "runner.py.txt"}: + raise ValueError("Unexpected prepared files") + for name, expected in manifest.items(): + if sha(target / name) != expected: + raise ValueError("Prepared content changed: " + name) + if sha(Path(__file__)) != manifest["runner.py.txt"] or sha(PROTOCOL) != manifest["protocol.md"]: + raise ValueError("Code or protocol differs from frozen source") + return json.loads((target / "inputs.json").read_text()) + + +def collect(work, output, lock_from=None): + data = load_prepared() + work.mkdir(parents=True, exist_ok=True) + output.mkdir(parents=True, exist_ok=False) + records = {} + for candidate in data["candidates"]: + if candidate["decision"] != "include": + continue + pairs = data["authored_inputs"][candidate["id"]] + samples = [pair[side] for pair in pairs for side in ("baseline", "variant")] + records[candidate["id"]] = {} + for version in (candidate["before"], candidate["after"]): + name = candidate["package"] + "-" + version + env = work / name + if env.exists(): + raise ValueError("Use a fresh environment directory: " + str(env)) + venv.EnvBuilder(with_pip=True, symlinks=True).create(env) + python = str(env / "bin/python") + pip_report = env / "install.json" + install_args = [candidate["package"] + "==" + version] + if lock_from: + install_args = ["-r", str(lock_from / (name + ".lock.txt"))] + install = subprocess.run( + [ + python, + "-m", + "pip", + "--disable-pip-version-check", + "install", + "--report", + str(pip_report), + *install_args, + ], + capture_output=True, + text=True, + timeout=180, + ) + record = {"install_returncode": install.returncode} + if install.returncode: + record["setup_error"] = install.stderr + records[candidate["id"]][version] = record + continue + installed = json.loads(pip_report.read_text()) + record["artifacts"] = [ + { + "name": item["metadata"]["name"], + "version": item["metadata"]["version"], + "url": item["download_info"]["url"], + "hashes": item["download_info"]["archive_info"].get("hashes", {}), + } + for item in installed["install"] + ] + lock = subprocess.check_output([python, "-m", "pip", "freeze"], text=True) + (output / (name + ".lock.txt")).write_text(lock) + record["lock_sha256"] = sha(output / (name + ".lock.txt")) + proc = subprocess.run( + [ + python, + "-I", + str(Path(__file__).resolve()), + "child", + "--package", + candidate["package"], + ], + input=json.dumps(samples), + capture_output=True, + text=True, + timeout=30, + ) + record.update( + {"returncode": proc.returncode, "stdout": proc.stdout, "stderr": proc.stderr} + ) + if proc.returncode == 0: + record["payload"] = json.loads(proc.stdout) + records[candidate["id"]][version] = record + print(name, "captured", flush=True) + write(output / "captured.json", records) + write( + output / "manifest.json", + { + "prepared_manifest_sha256": sha(OUT / "prepared/manifest.json"), + "files": {p.name: sha(p) for p in sorted(output.iterdir()) if p.is_file()}, + }, + ) + + +def replay(captured, output): + from narrative_contracts.model import digest + from narrative_contracts.mutations import Relation, Validity + from narrative_contracts.validator_audit import AuditCase, Obligation, Verdict, audit_validator + + data = load_prepared() + manifest = json.loads((captured / "manifest.json").read_text()) + if manifest["prepared_manifest_sha256"] != sha(OUT / "prepared/manifest.json"): + raise ValueError("Capture does not match preparation") + if "captured.json" not in manifest["files"] or any( + Path(n).name != n for n in manifest["files"] + ): + raise ValueError("Invalid capture manifest") + for name, expected in manifest["files"].items(): + if sha(captured / name) != expected: + raise ValueError("Capture changed: " + name) + records = json.loads((captured / "captured.json").read_text()) + reports, results = {}, [] + included = [c for c in data["candidates"] if c["decision"] == "include"] + if set(records) != {c["id"] for c in included}: + raise ValueError("Missing or extra candidate capture") + for candidate in included: + pairs = data["authored_inputs"][candidate["id"]] + versions = records[candidate["id"]] + outcomes = {} + for version in (candidate["before"], candidate["after"]): + record = versions[version] + if record["install_returncode"] or record["returncode"]: + outcomes[version] = {"execution": "unavailable"} + continue + payload = record["payload"] + if payload != json.loads(record["stdout"]) or payload["package_version"] != version: + raise ValueError("Captured version or raw output mismatch") + if len(payload["results"]) != 2 * len(pairs): + raise ValueError("Incomplete paired capture") + lookup = {} + for pair_index, pair in enumerate(pairs): + for side_index, side in enumerate(("baseline", "variant")): + key = digest(pair[side]) + observed = payload["results"][pair_index * 2 + side_index] + if key in lookup and lookup[key] != observed: + raise ValueError("Inconsistent repeated baseline capture") + lookup[key] = observed + + def adapter(sample): + record = lookup[digest(sample)] + if record["error"]: + raise RuntimeError(record["error"]) + return Verdict( + record["accepted"], + tuple(record["violations"]), + record["complete"], + tuple(record["messages"]), + ) + + cases = tuple( + AuditCase( + p["id"], + candidate["id"], + p["baseline"], + p["variant"], + Relation(p["relation"]), + tuple(p["expected"]), + Validity.VALID, + "Authored minimal reproduction of " + candidate["source"], + ) + for p in pairs + ) + report = audit_validator( + adapter, + cases, + obligations=(Obligation(candidate["id"], candidate["behavior"]),), + validator_id=f"{candidate['package']}/{version}", + ) + reports[candidate["id"] + "/" + version] = report.to_dict() + outcomes[version] = {c.id: c.outcome for c in report.cases} + trigger, control = pairs + before, after = outcomes[candidate["before"]], outcomes[candidate["after"]] + desired = "detected" if trigger["relation"] == "violation" else "preserved" + reproduced = ( + before.get(trigger["id"]) in ("survived", "regressed", "error") + and after.get(trigger["id"]) == desired + and before.get(control["id"]) == after.get(control["id"]) == "preserved" + ) + results.append( + { + "id": candidate["id"], + "source": candidate["source"], + "before_version": candidate["before"], + "after_version": candidate["after"], + "outcomes": outcomes, + "fix_reproduced": reproduced, + } + ) + summary = { + "schema_version": 1, + "screened": len(data["candidates"]), + "included": len(included), + "excluded": len(data["candidates"]) - len(included), + "fixes_reproduced": sum(r["fix_reproduced"] for r in results), + "results": results, + } + output.mkdir(parents=True, exist_ok=False) + write(output / "summary.json", summary) + write(output / "audit-reports.json", reports) + print(json.dumps(summary, indent=2)) + return summary + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("freeze", "collect", "replay", "child")) + parser.add_argument("--network", action="store_true") + parser.add_argument("--work", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument("--captured", type=Path) + parser.add_argument("--lock-from", type=Path) + parser.add_argument("--package") + args = parser.parse_args() + if args.action == "child": + child(args.package) + elif args.action == "freeze": + freeze() + elif args.action == "collect": + if not args.network or args.work is None or args.output is None: + parser.error("collect requires --network, --work and --output") + collect(args.work, args.output, args.lock_from) + else: + if args.captured is None or args.output is None: + parser.error("replay requires --captured and --output") + replay(args.captured, args.output) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/requirements/jsonschema-3.0.1.lock.txt b/benchmarks/historical-validator-results/compatibility-followup-v2/requirements/jsonschema-3.0.1.lock.txt new file mode 100644 index 0000000..a0d77d8 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/requirements/jsonschema-3.0.1.lock.txt @@ -0,0 +1,5 @@ +attrs==26.1.0 +jsonschema==3.0.1 +pyrsistent==0.20.0 +six==1.17.0 +setuptools==70.3.0 diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/requirements/jsonschema-3.0.2.lock.txt b/benchmarks/historical-validator-results/compatibility-followup-v2/requirements/jsonschema-3.0.2.lock.txt new file mode 100644 index 0000000..ed7f8d3 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/requirements/jsonschema-3.0.2.lock.txt @@ -0,0 +1,5 @@ +attrs==26.1.0 +jsonschema==3.0.2 +pyrsistent==0.20.0 +six==1.17.0 +setuptools==70.3.0 diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/requirements/jsonschema-4.17.1.lock.txt b/benchmarks/historical-validator-results/compatibility-followup-v2/requirements/jsonschema-4.17.1.lock.txt new file mode 100644 index 0000000..eac7b5b --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/requirements/jsonschema-4.17.1.lock.txt @@ -0,0 +1,3 @@ +attrs==26.1.0 +jsonschema==4.17.1 +pyrsistent==0.20.0 diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/requirements/jsonschema-4.17.3.lock.txt b/benchmarks/historical-validator-results/compatibility-followup-v2/requirements/jsonschema-4.17.3.lock.txt new file mode 100644 index 0000000..184ab11 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/requirements/jsonschema-4.17.3.lock.txt @@ -0,0 +1,3 @@ +attrs==26.1.0 +jsonschema==4.17.3 +pyrsistent==0.20.0 diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/requirements/marshmallow-4.2.0.lock.txt b/benchmarks/historical-validator-results/compatibility-followup-v2/requirements/marshmallow-4.2.0.lock.txt new file mode 100644 index 0000000..410f96d --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/requirements/marshmallow-4.2.0.lock.txt @@ -0,0 +1 @@ +marshmallow==4.2.0 diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/requirements/marshmallow-4.2.1.lock.txt b/benchmarks/historical-validator-results/compatibility-followup-v2/requirements/marshmallow-4.2.1.lock.txt new file mode 100644 index 0000000..8592200 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/requirements/marshmallow-4.2.1.lock.txt @@ -0,0 +1 @@ +marshmallow==4.2.1 diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/requirements/marshmallow-4.2.3.lock.txt b/benchmarks/historical-validator-results/compatibility-followup-v2/requirements/marshmallow-4.2.3.lock.txt new file mode 100644 index 0000000..f15c45d --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/requirements/marshmallow-4.2.3.lock.txt @@ -0,0 +1 @@ +marshmallow==4.2.3 diff --git a/benchmarks/historical-validator-results/compatibility-followup-v2/requirements/marshmallow-4.2.4.lock.txt b/benchmarks/historical-validator-results/compatibility-followup-v2/requirements/marshmallow-4.2.4.lock.txt new file mode 100644 index 0000000..90e1042 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup-v2/requirements/marshmallow-4.2.4.lock.txt @@ -0,0 +1 @@ +marshmallow==4.2.4 diff --git a/benchmarks/historical-validator-results/compatibility-followup/captured/captured.json b/benchmarks/historical-validator-results/compatibility-followup/captured/captured.json new file mode 100644 index 0000000..6ebbcd4 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup/captured/captured.json @@ -0,0 +1,356 @@ +{ + "jsonschema-bool-ref-cache": { + "4.17.1": { + "artifacts": [ + { + "hashes": { + "sha256": "c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309" + }, + "name": "attrs", + "url": "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", + "version": "26.1.0" + }, + { + "hashes": { + "sha256": "410ef23dcdbca4eaedc08b850079179883c2ed09378bd1f760d4af4aacfa28d7" + }, + "name": "jsonschema", + "url": "https://files.pythonhosted.org/packages/9f/df/824fdaa0d7228fa2e8a5171a408dbabe2c66955afd5be5211725389640b5/jsonschema-4.17.1-py3-none-any.whl", + "version": "4.17.1" + }, + { + "hashes": { + "sha256": "09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e" + }, + "name": "pyrsistent", + "url": "https://files.pythonhosted.org/packages/15/ee/ff2ed52032ac1ce2e7ba19e79bd5b05d152ebfb77956cf08fcd6e8d760ea/pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", + "version": "0.20.0" + } + ], + "install_returncode": 0, + "lock_sha256": "cbcdc8d2ed03fbdbf57a9aaa3c6dc09bb67ca99ae89d16901549ce33d737c9c0", + "payload": { + "package_version": "4.17.1", + "python": "3.12.13", + "results": [ + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "error": "TypeError: argument of type 'bool' is not iterable" + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + } + ] + }, + "returncode": 0, + "stderr": "", + "stdout": "{\"package_version\": \"4.17.1\", \"python\": \"3.12.13\", \"results\": [{\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"error\": \"TypeError: argument of type 'bool' is not iterable\"}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}]}\n" + }, + "4.17.3": { + "artifacts": [ + { + "hashes": { + "sha256": "c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309" + }, + "name": "attrs", + "url": "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", + "version": "26.1.0" + }, + { + "hashes": { + "sha256": "a870ad254da1a8ca84b6a2905cac29d265f805acc57af304784962a2aa6508f6" + }, + "name": "jsonschema", + "url": "https://files.pythonhosted.org/packages/c1/97/c698bd9350f307daad79dd740806e1a59becd693bd11443a0f531e3229b3/jsonschema-4.17.3-py3-none-any.whl", + "version": "4.17.3" + }, + { + "hashes": { + "sha256": "09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e" + }, + "name": "pyrsistent", + "url": "https://files.pythonhosted.org/packages/15/ee/ff2ed52032ac1ce2e7ba19e79bd5b05d152ebfb77956cf08fcd6e8d760ea/pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", + "version": "0.20.0" + } + ], + "install_returncode": 0, + "lock_sha256": "839c46ca0b682f2f69ae74c921b15761456ea9107d3fb315ca2e0d7941107f8a", + "payload": { + "package_version": "4.17.3", + "python": "3.12.13", + "results": [ + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + } + ] + }, + "returncode": 0, + "stderr": "", + "stdout": "{\"package_version\": \"4.17.3\", \"python\": \"3.12.13\", \"results\": [{\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}]}\n" + } + }, + "jsonschema-enum-bool-int": { + "3.0.1": { + "install_returncode": 1, + "setup_error": "ERROR: Cannot install setuptools==70.3.0 and setuptools==84.0.0 because these package versions have conflicting dependencies.\nERROR: ResolutionImpossible: for help visit https://pip.pypa.io/en/latest/topics/dependency-resolution/#dealing-with-dependency-conflicts\n" + }, + "3.0.2": { + "install_returncode": 1, + "setup_error": "ERROR: Cannot install setuptools==70.3.0 and setuptools==84.0.0 because these package versions have conflicting dependencies.\nERROR: ResolutionImpossible: for help visit https://pip.pypa.io/en/latest/topics/dependency-resolution/#dealing-with-dependency-conflicts\n" + } + }, + "marshmallow-file-url": { + "4.2.0": { + "artifacts": [ + { + "hashes": { + "sha256": "1dc369bd13a8708a9566d6f73d1db07d50142a7580f04fd81e1c29a4d2e10af4" + }, + "name": "marshmallow", + "url": "https://files.pythonhosted.org/packages/1c/b6/0a907f92c2158c9841da0227c7074ce1490f578f34d67cbba82ba8f9146e/marshmallow-4.2.0-py3-none-any.whl", + "version": "4.2.0" + } + ], + "install_returncode": 0, + "lock_sha256": "42d87bc7dbcbece58f644985e692238b2a6273d097e85deffc337316a6aa0ca0", + "payload": { + "package_version": "4.2.0", + "python": "3.12.13", + "results": [ + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": false, + "complete": true, + "error": null, + "messages": [ + "['Not a valid URL.']" + ], + "violations": [ + "field@$" + ] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + } + ] + }, + "returncode": 0, + "stderr": "", + "stdout": "{\"package_version\": \"4.2.0\", \"python\": \"3.12.13\", \"results\": [{\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": false, \"complete\": true, \"error\": null, \"messages\": [\"['Not a valid URL.']\"], \"violations\": [\"field@$\"]}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}]}\n" + }, + "4.2.1": { + "artifacts": [ + { + "hashes": { + "sha256": "d82b1a83cfbb4667d050850fbed4e9d4435576cb95f5ff37894f375dce201768" + }, + "name": "marshmallow", + "url": "https://files.pythonhosted.org/packages/6c/41/e8cfdebc3a475c397e074ceaa12e86dca1b23e8e10c2ab4d4875f9a01418/marshmallow-4.2.1-py3-none-any.whl", + "version": "4.2.1" + } + ], + "install_returncode": 0, + "lock_sha256": "58dc973cc15ec968b9e44708e7ea902faaddb5a380096a4f794ae370dc944b02", + "payload": { + "package_version": "4.2.1", + "python": "3.12.13", + "results": [ + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + } + ] + }, + "returncode": 0, + "stderr": "", + "stdout": "{\"package_version\": \"4.2.1\", \"python\": \"3.12.13\", \"results\": [{\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}]}\n" + } + }, + "marshmallow-idn-email": { + "4.2.3": { + "artifacts": [ + { + "hashes": { + "sha256": "c84fd89817ecea690bde1eb925036070e5e9883148217585adc56d5cfbc082b8" + }, + "name": "marshmallow", + "url": "https://files.pythonhosted.org/packages/58/2e/ce3278954d44903ccf8606c2a16e6b3ad8e10146c21de1b7b9b0f5812b23/marshmallow-4.2.3-py3-none-any.whl", + "version": "4.2.3" + } + ], + "install_returncode": 0, + "lock_sha256": "1d7fa2cd4b51cd71b9a58cfd7bfb0a657d9f94dfc44cf957829af622faa14e57", + "payload": { + "package_version": "4.2.3", + "python": "3.12.13", + "results": [ + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + } + ] + }, + "returncode": 0, + "stderr": "", + "stdout": "{\"package_version\": \"4.2.3\", \"python\": \"3.12.13\", \"results\": [{\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}]}\n" + }, + "4.2.4": { + "artifacts": [ + { + "hashes": { + "sha256": "06f3a7ec7081e54e8026a8bd5e7c4e0e88bff82a8fa33714a33166917f2b353a" + }, + "name": "marshmallow", + "url": "https://files.pythonhosted.org/packages/dc/df/7d6d6625ec822d5adf05b55a1a0f53e2f03627c6e841ea5655310b8fda22/marshmallow-4.2.4-py3-none-any.whl", + "version": "4.2.4" + } + ], + "install_returncode": 0, + "lock_sha256": "201c63e7fd2b26bde60fece7e85d6f35d651e3362edfa787cf6e536cd5fdd14b", + "payload": { + "package_version": "4.2.4", + "python": "3.12.13", + "results": [ + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + }, + { + "accepted": true, + "complete": true, + "error": null, + "messages": [], + "violations": [] + } + ] + }, + "returncode": 0, + "stderr": "", + "stdout": "{\"package_version\": \"4.2.4\", \"python\": \"3.12.13\", \"results\": [{\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}, {\"accepted\": true, \"complete\": true, \"error\": null, \"messages\": [], \"violations\": []}]}\n" + } + } +} diff --git a/benchmarks/historical-validator-results/compatibility-followup/captured/jsonschema-4.17.1.lock.txt b/benchmarks/historical-validator-results/compatibility-followup/captured/jsonschema-4.17.1.lock.txt new file mode 100644 index 0000000..eac7b5b --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup/captured/jsonschema-4.17.1.lock.txt @@ -0,0 +1,3 @@ +attrs==26.1.0 +jsonschema==4.17.1 +pyrsistent==0.20.0 diff --git a/benchmarks/historical-validator-results/compatibility-followup/captured/jsonschema-4.17.3.lock.txt b/benchmarks/historical-validator-results/compatibility-followup/captured/jsonschema-4.17.3.lock.txt new file mode 100644 index 0000000..184ab11 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup/captured/jsonschema-4.17.3.lock.txt @@ -0,0 +1,3 @@ +attrs==26.1.0 +jsonschema==4.17.3 +pyrsistent==0.20.0 diff --git a/benchmarks/historical-validator-results/compatibility-followup/captured/manifest.json b/benchmarks/historical-validator-results/compatibility-followup/captured/manifest.json new file mode 100644 index 0000000..957dc68 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup/captured/manifest.json @@ -0,0 +1,12 @@ +{ + "files": { + "captured.json": "fdac565e46b4f0317040629742e3d778313aa43c53916aaa93535f338e5877a2", + "jsonschema-4.17.1.lock.txt": "cbcdc8d2ed03fbdbf57a9aaa3c6dc09bb67ca99ae89d16901549ce33d737c9c0", + "jsonschema-4.17.3.lock.txt": "839c46ca0b682f2f69ae74c921b15761456ea9107d3fb315ca2e0d7941107f8a", + "marshmallow-4.2.0.lock.txt": "42d87bc7dbcbece58f644985e692238b2a6273d097e85deffc337316a6aa0ca0", + "marshmallow-4.2.1.lock.txt": "58dc973cc15ec968b9e44708e7ea902faaddb5a380096a4f794ae370dc944b02", + "marshmallow-4.2.3.lock.txt": "1d7fa2cd4b51cd71b9a58cfd7bfb0a657d9f94dfc44cf957829af622faa14e57", + "marshmallow-4.2.4.lock.txt": "201c63e7fd2b26bde60fece7e85d6f35d651e3362edfa787cf6e536cd5fdd14b" + }, + "prepared_manifest_sha256": "fb9546c41166155f999af9995a92dd9e3d82af4376edb5beca0f72857cbdd156" +} diff --git a/benchmarks/historical-validator-results/compatibility-followup/captured/marshmallow-4.2.0.lock.txt b/benchmarks/historical-validator-results/compatibility-followup/captured/marshmallow-4.2.0.lock.txt new file mode 100644 index 0000000..410f96d --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup/captured/marshmallow-4.2.0.lock.txt @@ -0,0 +1 @@ +marshmallow==4.2.0 diff --git a/benchmarks/historical-validator-results/compatibility-followup/captured/marshmallow-4.2.1.lock.txt b/benchmarks/historical-validator-results/compatibility-followup/captured/marshmallow-4.2.1.lock.txt new file mode 100644 index 0000000..8592200 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup/captured/marshmallow-4.2.1.lock.txt @@ -0,0 +1 @@ +marshmallow==4.2.1 diff --git a/benchmarks/historical-validator-results/compatibility-followup/captured/marshmallow-4.2.3.lock.txt b/benchmarks/historical-validator-results/compatibility-followup/captured/marshmallow-4.2.3.lock.txt new file mode 100644 index 0000000..f15c45d --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup/captured/marshmallow-4.2.3.lock.txt @@ -0,0 +1 @@ +marshmallow==4.2.3 diff --git a/benchmarks/historical-validator-results/compatibility-followup/captured/marshmallow-4.2.4.lock.txt b/benchmarks/historical-validator-results/compatibility-followup/captured/marshmallow-4.2.4.lock.txt new file mode 100644 index 0000000..90e1042 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup/captured/marshmallow-4.2.4.lock.txt @@ -0,0 +1 @@ +marshmallow==4.2.4 diff --git a/benchmarks/historical-validator-results/compatibility-followup/driver.py.txt b/benchmarks/historical-validator-results/compatibility-followup/driver.py.txt new file mode 100644 index 0000000..2d0dcc7 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup/driver.py.txt @@ -0,0 +1,97 @@ +"""Explicit compatibility follow-up; preserves the original historical captures.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BASE = ROOT / "benchmarks/historical-validator-results" +OUT = BASE / "compatibility-followup" +spec = importlib.util.spec_from_file_location( + "historical_base", ROOT / "benchmarks/historical_validator_study.py" +) +base = importlib.util.module_from_spec(spec) +spec.loader.exec_module(base) +base.OUT = OUT +base.PROTOCOL = ROOT / "paper/historical-followup-protocol.md" + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def prepare(): + OUT.mkdir(exist_ok=False) + requirements = OUT / "requirements" + requirements.mkdir() + data = json.loads((BASE / "prepared/inputs.json").read_text()) + for c in data["candidates"]: + if c["id"] == "jsonschema-bool-ref-cache": + c["before"] = "4.17.1" + c["reason"] += ( + " Follow-up uses available 4.17.1; initial 4.17.2 could not be installed." + ) + if c["decision"] != "include": + continue + for version in (c["before"], c["after"]): + name = c["package"] + "-" + version + ".lock.txt" + original = BASE / "captured" / name + if version == "4.17.1": + content = ( + (BASE / "captured/jsonschema-4.17.3.lock.txt") + .read_text() + .replace("jsonschema==4.17.3", "jsonschema==4.17.1") + ) + else: + content = original.read_text() + if c["package"] == "jsonschema" and version.startswith("3."): + content += "setuptools==70.3.0\n" + (requirements / name).write_text(content) + data["followup_driver_sha256"] = sha(Path(__file__)) + data["requirements_sha256"] = {p.name: sha(p) for p in requirements.iterdir()} + data["parent_capture_manifest_sha256"] = sha(BASE / "captured/manifest.json") + (OUT / "frozen-inputs.json").write_text( + json.dumps(data, indent=2, sort_keys=True, ensure_ascii=False) + "\n" + ) + base.freeze() + + +def verify(): + data = base.load_prepared() + if sha(Path(__file__)) != data["followup_driver_sha256"]: + raise ValueError("Follow-up driver changed") + for name, expected in data["requirements_sha256"].items(): + if Path(name).name != name or sha(OUT / "requirements" / name) != expected: + raise ValueError("Follow-up requirements changed") + if sha(BASE / "captured/manifest.json") != data["parent_capture_manifest_sha256"]: + raise ValueError("Parent capture identity changed") + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("action", choices=("prepare", "collect", "replay")) + p.add_argument("--network", action="store_true") + p.add_argument("--work", type=Path) + p.add_argument("--output", type=Path) + p.add_argument("--captured", type=Path) + args = p.parse_args() + if args.action == "prepare": + prepare() + return + verify() + if args.action == "collect": + if not args.network or args.work is None or args.output is None: + p.error("collect requires --network, --work and --output") + base.collect(args.work, args.output, OUT / "requirements") + else: + if args.captured is None or args.output is None: + p.error("replay requires --captured and --output") + base.replay(args.captured, args.output) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/historical-validator-results/compatibility-followup/evaluation/audit-reports.json b/benchmarks/historical-validator-results/compatibility-followup/evaluation/audit-reports.json new file mode 100644 index 0000000..3aec560 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup/evaluation/audit-reports.json @@ -0,0 +1,581 @@ +{ + "jsonschema-bool-ref-cache/4.17.1": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "23d1edd41d6aeba96e7014ab4f960b6025b66424e01c5f6445f9e687d47c0c0b", + "expected": [], + "id": "cached-boolean-valid", + "obligation": "jsonschema-bool-ref-cache", + "outcome": "error", + "provenance": "Authored minimal reproduction of https://github.com/python-jsonschema/jsonschema/issues/1018", + "reason": "variant_error: RuntimeError: TypeError: argument of type 'bool' is not iterable", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "637d75abb2919f760e0cca7887cfffe980cd2fabdb0ea3cbce6fdddcda31f01c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "23d1edd41d6aeba96e7014ab4f960b6025b66424e01c5f6445f9e687d47c0c0b", + "expected": [], + "id": "cached-object-control", + "obligation": "jsonschema-bool-ref-cache", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/python-jsonschema/jsonschema/issues/1018", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "a1f217c76d39d422f165a35ee37ddd15f340ccbf3babf622ac450c8cf9909bdb" + } + ], + "corpus_digest": "51a25513137f3ebef8a66e4f4f413dee971902077d4e04ed8d8d25e126bace62", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Validator instantiation accepts cached boolean schemas", + "id": "jsonschema-bool-ref-cache", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "error": 1, + "preserved": 1 + }, + "detected_faults": 0, + "detection_score": null, + "eligible_controls": 2, + "eligible_faults": 0, + "obligations": [ + { + "assessment": "incomplete_evidence", + "case_ids": [ + "cached-boolean-valid", + "cached-object-control" + ], + "counts": { + "error": 1, + "preserved": 1 + }, + "description": "Validator instantiation accepts cached boolean schemas", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": true, + "has_known_gaps": false, + "id": "jsonschema-bool-ref-cache", + "scope": "contract" + } + ], + "preservation_rate": 0.5, + "preserved_controls": 1, + "total": 2 + }, + "validator_id": "jsonschema/4.17.1" + }, + "jsonschema-bool-ref-cache/4.17.3": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "23d1edd41d6aeba96e7014ab4f960b6025b66424e01c5f6445f9e687d47c0c0b", + "expected": [], + "id": "cached-boolean-valid", + "obligation": "jsonschema-bool-ref-cache", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/python-jsonschema/jsonschema/issues/1018", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "637d75abb2919f760e0cca7887cfffe980cd2fabdb0ea3cbce6fdddcda31f01c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "23d1edd41d6aeba96e7014ab4f960b6025b66424e01c5f6445f9e687d47c0c0b", + "expected": [], + "id": "cached-object-control", + "obligation": "jsonschema-bool-ref-cache", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/python-jsonschema/jsonschema/issues/1018", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "a1f217c76d39d422f165a35ee37ddd15f340ccbf3babf622ac450c8cf9909bdb" + } + ], + "corpus_digest": "51a25513137f3ebef8a66e4f4f413dee971902077d4e04ed8d8d25e126bace62", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Validator instantiation accepts cached boolean schemas", + "id": "jsonschema-bool-ref-cache", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 2 + }, + "detected_faults": 0, + "detection_score": null, + "eligible_controls": 2, + "eligible_faults": 0, + "obligations": [ + { + "assessment": "controls_only_no_faults", + "case_ids": [ + "cached-boolean-valid", + "cached-object-control" + ], + "counts": { + "preserved": 2 + }, + "description": "Validator instantiation accepts cached boolean schemas", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "jsonschema-bool-ref-cache", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 2, + "total": 2 + }, + "validator_id": "jsonschema/4.17.3" + }, + "marshmallow-file-url/4.2.0": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "8e0181a9a7ffb31fcbba12bac11a8e779ffb29d858ae6f80e2497df03de25d6a", + "expected": [], + "id": "uppercase-scheme-valid", + "obligation": "marshmallow-file-url", + "outcome": "regressed", + "provenance": "Authored minimal reproduction of https://github.com/marshmallow-code/marshmallow/issues/2891", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [ + "['Not a valid URL.']" + ], + "violations": [ + "field@$" + ] + }, + "variant_digest": "1c11eec5aa6a9d3631ee25994ad82fe370e5ebf0d0f1896c35db84ec1cbaeb31" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "8e0181a9a7ffb31fcbba12bac11a8e779ffb29d858ae6f80e2497df03de25d6a", + "expected": [], + "id": "lowercase-control", + "obligation": "marshmallow-file-url", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/marshmallow-code/marshmallow/issues/2891", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "fc2859890cc70e6029c2a2e07130c519ee56b0b8305960a2a6319888e012b5b3" + } + ], + "corpus_digest": "8372dc7ce05eca31612829b3af45404209289e23a319853266a2fd943026b844", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "URL validator handles uppercase FILE scheme", + "id": "marshmallow-file-url", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 1, + "regressed": 1 + }, + "detected_faults": 0, + "detection_score": null, + "eligible_controls": 2, + "eligible_faults": 0, + "obligations": [ + { + "assessment": "known_gaps", + "case_ids": [ + "uppercase-scheme-valid", + "lowercase-control" + ], + "counts": { + "preserved": 1, + "regressed": 1 + }, + "description": "URL validator handles uppercase FILE scheme", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": true, + "id": "marshmallow-file-url", + "scope": "contract" + } + ], + "preservation_rate": 0.5, + "preserved_controls": 1, + "total": 2 + }, + "validator_id": "marshmallow/4.2.0" + }, + "marshmallow-file-url/4.2.1": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "8e0181a9a7ffb31fcbba12bac11a8e779ffb29d858ae6f80e2497df03de25d6a", + "expected": [], + "id": "uppercase-scheme-valid", + "obligation": "marshmallow-file-url", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/marshmallow-code/marshmallow/issues/2891", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "1c11eec5aa6a9d3631ee25994ad82fe370e5ebf0d0f1896c35db84ec1cbaeb31" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "8e0181a9a7ffb31fcbba12bac11a8e779ffb29d858ae6f80e2497df03de25d6a", + "expected": [], + "id": "lowercase-control", + "obligation": "marshmallow-file-url", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/marshmallow-code/marshmallow/issues/2891", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "fc2859890cc70e6029c2a2e07130c519ee56b0b8305960a2a6319888e012b5b3" + } + ], + "corpus_digest": "8372dc7ce05eca31612829b3af45404209289e23a319853266a2fd943026b844", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "URL validator handles uppercase FILE scheme", + "id": "marshmallow-file-url", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 2 + }, + "detected_faults": 0, + "detection_score": null, + "eligible_controls": 2, + "eligible_faults": 0, + "obligations": [ + { + "assessment": "controls_only_no_faults", + "case_ids": [ + "uppercase-scheme-valid", + "lowercase-control" + ], + "counts": { + "preserved": 2 + }, + "description": "URL validator handles uppercase FILE scheme", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "marshmallow-file-url", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 2, + "total": 2 + }, + "validator_id": "marshmallow/4.2.1" + }, + "marshmallow-idn-email/4.2.3": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "e102831f1946c821ebba3d7d3e4eb088d810b857cf1169c07f5e9d16d06733e0", + "expected": [], + "id": "unicode-tld-valid", + "obligation": "marshmallow-idn-email", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/marshmallow-code/marshmallow/issues/2936", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "2d95c74f71d180a0d5ac62a81fafb19471a9d04abede546ea08e380c5789dc24" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "e102831f1946c821ebba3d7d3e4eb088d810b857cf1169c07f5e9d16d06733e0", + "expected": [], + "id": "ascii-control", + "obligation": "marshmallow-idn-email", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/marshmallow-code/marshmallow/issues/2936", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "01fd15a9b9c17e7048fc00711f58e979c1a0007be6d1e330f5f117bf706b3525" + } + ], + "corpus_digest": "be1c84141ec7eb9991b7399fdf0a9c0c209bfb654d2a24024f0a0cb14c258f33", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Email validator accepts valid internationalized domain names", + "id": "marshmallow-idn-email", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 2 + }, + "detected_faults": 0, + "detection_score": null, + "eligible_controls": 2, + "eligible_faults": 0, + "obligations": [ + { + "assessment": "controls_only_no_faults", + "case_ids": [ + "unicode-tld-valid", + "ascii-control" + ], + "counts": { + "preserved": 2 + }, + "description": "Email validator accepts valid internationalized domain names", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "marshmallow-idn-email", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 2, + "total": 2 + }, + "validator_id": "marshmallow/4.2.3" + }, + "marshmallow-idn-email/4.2.4": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "e102831f1946c821ebba3d7d3e4eb088d810b857cf1169c07f5e9d16d06733e0", + "expected": [], + "id": "unicode-tld-valid", + "obligation": "marshmallow-idn-email", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/marshmallow-code/marshmallow/issues/2936", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "2d95c74f71d180a0d5ac62a81fafb19471a9d04abede546ea08e380c5789dc24" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "e102831f1946c821ebba3d7d3e4eb088d810b857cf1169c07f5e9d16d06733e0", + "expected": [], + "id": "ascii-control", + "obligation": "marshmallow-idn-email", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/marshmallow-code/marshmallow/issues/2936", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "01fd15a9b9c17e7048fc00711f58e979c1a0007be6d1e330f5f117bf706b3525" + } + ], + "corpus_digest": "be1c84141ec7eb9991b7399fdf0a9c0c209bfb654d2a24024f0a0cb14c258f33", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Email validator accepts valid internationalized domain names", + "id": "marshmallow-idn-email", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 2 + }, + "detected_faults": 0, + "detection_score": null, + "eligible_controls": 2, + "eligible_faults": 0, + "obligations": [ + { + "assessment": "controls_only_no_faults", + "case_ids": [ + "unicode-tld-valid", + "ascii-control" + ], + "counts": { + "preserved": 2 + }, + "description": "Email validator accepts valid internationalized domain names", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "marshmallow-idn-email", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 2, + "total": 2 + }, + "validator_id": "marshmallow/4.2.4" + } +} diff --git a/benchmarks/historical-validator-results/compatibility-followup/evaluation/summary.json b/benchmarks/historical-validator-results/compatibility-followup/evaluation/summary.json new file mode 100644 index 0000000..dbf60b8 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup/evaluation/summary.json @@ -0,0 +1,75 @@ +{ + "excluded": 6, + "fixes_reproduced": 2, + "included": 4, + "results": [ + { + "after_version": "3.0.2", + "before_version": "3.0.1", + "fix_reproduced": false, + "id": "jsonschema-enum-bool-int", + "outcomes": { + "3.0.1": { + "execution": "unavailable" + }, + "3.0.2": { + "execution": "unavailable" + } + }, + "source": "https://github.com/python-jsonschema/jsonschema/issues/575" + }, + { + "after_version": "4.17.3", + "before_version": "4.17.1", + "fix_reproduced": true, + "id": "jsonschema-bool-ref-cache", + "outcomes": { + "4.17.1": { + "cached-boolean-valid": "error", + "cached-object-control": "preserved" + }, + "4.17.3": { + "cached-boolean-valid": "preserved", + "cached-object-control": "preserved" + } + }, + "source": "https://github.com/python-jsonschema/jsonschema/issues/1018" + }, + { + "after_version": "4.2.1", + "before_version": "4.2.0", + "fix_reproduced": true, + "id": "marshmallow-file-url", + "outcomes": { + "4.2.0": { + "lowercase-control": "preserved", + "uppercase-scheme-valid": "regressed" + }, + "4.2.1": { + "lowercase-control": "preserved", + "uppercase-scheme-valid": "preserved" + } + }, + "source": "https://github.com/marshmallow-code/marshmallow/issues/2891" + }, + { + "after_version": "4.2.4", + "before_version": "4.2.3", + "fix_reproduced": false, + "id": "marshmallow-idn-email", + "outcomes": { + "4.2.3": { + "ascii-control": "preserved", + "unicode-tld-valid": "preserved" + }, + "4.2.4": { + "ascii-control": "preserved", + "unicode-tld-valid": "preserved" + } + }, + "source": "https://github.com/marshmallow-code/marshmallow/issues/2936" + } + ], + "schema_version": 1, + "screened": 10 +} diff --git a/benchmarks/historical-validator-results/compatibility-followup/frozen-inputs.json b/benchmarks/historical-validator-results/compatibility-followup/frozen-inputs.json new file mode 100644 index 0000000..0284873 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup/frozen-inputs.json @@ -0,0 +1,304 @@ +{ + "authored_inputs": { + "jsonschema-bool-ref-cache": [ + { + "baseline": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:ok": { + "type": "object" + } + }, + "value": {} + }, + "expected": [], + "id": "cached-boolean-valid", + "relation": "preserve", + "variant": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:any": true, + "urn:study:ok": { + "type": "object" + } + }, + "value": {} + } + }, + { + "baseline": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:ok": { + "type": "object" + } + }, + "value": {} + }, + "expected": [], + "id": "cached-object-control", + "relation": "preserve", + "variant": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:ok": { + "type": "object" + }, + "urn:study:other": { + "type": "array" + } + }, + "value": {} + } + } + ], + "jsonschema-enum-bool-int": [ + { + "baseline": { + "kind": "enum", + "schema": { + "enum": [ + false + ] + }, + "value": false + }, + "expected": [ + "enum@$" + ], + "id": "enum-int-fault", + "relation": "violation", + "variant": { + "kind": "enum", + "schema": { + "enum": [ + false + ] + }, + "value": 0 + } + }, + { + "baseline": { + "kind": "enum", + "schema": { + "enum": [ + false + ] + }, + "value": false + }, + "expected": [], + "id": "enum-valid-control", + "relation": "preserve", + "variant": { + "kind": "enum", + "schema": { + "enum": [ + false, + true + ] + }, + "value": false + } + } + ], + "marshmallow-file-url": [ + { + "baseline": { + "kind": "file", + "value": "file:///tmp/a" + }, + "expected": [], + "id": "uppercase-scheme-valid", + "relation": "preserve", + "variant": { + "kind": "file", + "value": "FILE:///tmp/a" + } + }, + { + "baseline": { + "kind": "file", + "value": "file:///tmp/a" + }, + "expected": [], + "id": "lowercase-control", + "relation": "preserve", + "variant": { + "kind": "file", + "value": "file:///tmp/b" + } + } + ], + "marshmallow-idn-email": [ + { + "baseline": { + "kind": "email", + "value": "user@example.com" + }, + "expected": [], + "id": "unicode-tld-valid", + "relation": "preserve", + "variant": { + "kind": "email", + "value": "user@пример.рф" + } + }, + { + "baseline": { + "kind": "email", + "value": "user@example.com" + }, + "expected": [], + "id": "ascii-control", + "relation": "preserve", + "variant": { + "kind": "email", + "value": "user2@example.com" + } + } + ] + }, + "candidates": [ + { + "after": "3.0.2", + "before": "3.0.1", + "behavior": "JSON Schema enum/const distinguish 0 from false", + "decision": "include", + "id": "jsonschema-enum-bool-int", + "package": "jsonschema", + "reason": "Release notes explicitly call this a bug fix and identify before/after releases.", + "release_source": "https://github.com/python-jsonschema/jsonschema/blob/51cd75e399c760e5aa3adce600dcce1385756ab0/CHANGELOG.rst", + "source": "https://github.com/python-jsonschema/jsonschema/issues/575", + "source_id": "issue #575" + }, + { + "after": "4.17.3", + "before": "4.17.1", + "behavior": "Validator instantiation accepts cached boolean schemas", + "decision": "include", + "id": "jsonschema-bool-ref-cache", + "package": "jsonschema", + "reason": "Release notes explicitly call this a bug fix with a fixed release. Follow-up uses available 4.17.1; initial 4.17.2 could not be installed.", + "release_source": "https://github.com/python-jsonschema/jsonschema/blob/51cd75e399c760e5aa3adce600dcce1385756ab0/CHANGELOG.rst", + "source": "https://github.com/python-jsonschema/jsonschema/issues/1018", + "source_id": "issue #1018" + }, + { + "after": "4.2.1", + "before": "4.2.0", + "behavior": "URL validator handles uppercase FILE scheme", + "decision": "include", + "id": "marshmallow-file-url", + "package": "marshmallow", + "reason": "Release notes identify a validation fix and adjacent releases.", + "release_source": "https://github.com/marshmallow-code/marshmallow/blob/7f0792bd7a06f72e393ec866ac1e89e1502ccfe1/CHANGELOG.rst", + "source": "https://github.com/marshmallow-code/marshmallow/issues/2891", + "source_id": "issue #2891" + }, + { + "after": "4.2.4", + "before": "4.2.3", + "behavior": "Email validator accepts valid internationalized domain names", + "decision": "include", + "id": "marshmallow-idn-email", + "package": "marshmallow", + "reason": "Release notes identify a validator behavior fix and adjacent releases.", + "release_source": "https://github.com/marshmallow-code/marshmallow/blob/7f0792bd7a06f72e393ec866ac1e89e1502ccfe1/CHANGELOG.rst", + "source": "https://github.com/marshmallow-code/marshmallow/issues/2936", + "source_id": "issue #2936" + }, + { + "after": null, + "before": null, + "behavior": "format checker is not enabled by default", + "decision": "exclude", + "id": "jsonschema-format-default", + "package": "jsonschema", + "reason": "Documented configuration/default behavior; not a library bug under the frozen rule.", + "source": "https://github.com/python-jsonschema/jsonschema/issues/945", + "source_id": "issue #945" + }, + { + "after": null, + "before": null, + "behavior": "email format checker is incomplete", + "decision": "exclude", + "id": "jsonschema-email-completeness", + "package": "jsonschema", + "reason": "Optional format-check completeness dispute; no documented before/after fix established in screening.", + "source": "https://github.com/python-jsonschema/jsonschema/issues/403", + "source_id": "issue #403" + }, + { + "after": null, + "before": null, + "behavior": "built-in generic forward references", + "decision": "exclude", + "id": "pydantic-forward-ref-generics", + "package": "pydantic", + "reason": "Model/type resolution rather than value validation; no fixed before/after release established in screening.", + "source": "https://github.com/pydantic/pydantic/issues/6130", + "source_id": "issue #6130" + }, + { + "after": null, + "before": null, + "behavior": "IPv4Address default JSON schema warning", + "decision": "exclude", + "id": "pydantic-ipv4-default", + "package": "pydantic", + "reason": "Serialization/schema warning, with no documented before/after validation fix.", + "source": "https://github.com/pydantic/pydantic/issues/8942", + "source_id": "issue #8942" + }, + { + "after": null, + "before": null, + "behavior": "default/missing parameter transition", + "decision": "exclude", + "id": "marshmallow-required-default", + "package": "marshmallow", + "reason": "Intentional API/deprecation transition and configuration semantics, not a validator defect.", + "source": "https://github.com/marshmallow-code/marshmallow/blob/dev/CHANGELOG.rst", + "source_id": "issue #1742" + }, + { + "after": null, + "before": null, + "behavior": "ModuleType serialization fails with serialize_as_any", + "decision": "exclude", + "id": "pydantic-serialize-any-regression", + "package": "pydantic", + "reason": "Serialization-only report; outside this study. No validation fix release was established.", + "source": "https://github.com/pydantic/pydantic/issues/12348", + "source_id": "issue #12348" + } + ], + "followup_driver_sha256": "8b47d741fa6ba5afb359bb359224b1be0d2fd546e591486063b010e3e3a41418", + "parent_capture_manifest_sha256": "7e2c9e33a5a63fe60c57ae03508c6a203f332c86f05704615db05fbf8e14c8e5", + "preflight_revision": 2, + "requirements_sha256": { + "jsonschema-3.0.1.lock.txt": "667cfa6465f0b419f24a96f4f4ba4bf9ae53b6bce4cec84610b3319522da2e90", + "jsonschema-3.0.2.lock.txt": "bccf527bf2a1e4e26f1faa39459b0b9c78cce8c3e58e4087d055c122e11c760b", + "jsonschema-4.17.1.lock.txt": "cbcdc8d2ed03fbdbf57a9aaa3c6dc09bb67ca99ae89d16901549ce33d737c9c0", + "jsonschema-4.17.3.lock.txt": "839c46ca0b682f2f69ae74c921b15761456ea9107d3fb315ca2e0d7941107f8a", + "marshmallow-4.2.0.lock.txt": "42d87bc7dbcbece58f644985e692238b2a6273d097e85deffc337316a6aa0ca0", + "marshmallow-4.2.1.lock.txt": "58dc973cc15ec968b9e44708e7ea902faaddb5a380096a4f794ae370dc944b02", + "marshmallow-4.2.3.lock.txt": "1d7fa2cd4b51cd71b9a58cfd7bfb0a657d9f94dfc44cf957829af622faa14e57", + "marshmallow-4.2.4.lock.txt": "201c63e7fd2b26bde60fece7e85d6f35d651e3362edfa787cf6e536cd5fdd14b" + } +} diff --git a/benchmarks/historical-validator-results/compatibility-followup/prepared/inputs.json b/benchmarks/historical-validator-results/compatibility-followup/prepared/inputs.json new file mode 100644 index 0000000..0284873 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup/prepared/inputs.json @@ -0,0 +1,304 @@ +{ + "authored_inputs": { + "jsonschema-bool-ref-cache": [ + { + "baseline": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:ok": { + "type": "object" + } + }, + "value": {} + }, + "expected": [], + "id": "cached-boolean-valid", + "relation": "preserve", + "variant": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:any": true, + "urn:study:ok": { + "type": "object" + } + }, + "value": {} + } + }, + { + "baseline": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:ok": { + "type": "object" + } + }, + "value": {} + }, + "expected": [], + "id": "cached-object-control", + "relation": "preserve", + "variant": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:ok": { + "type": "object" + }, + "urn:study:other": { + "type": "array" + } + }, + "value": {} + } + } + ], + "jsonschema-enum-bool-int": [ + { + "baseline": { + "kind": "enum", + "schema": { + "enum": [ + false + ] + }, + "value": false + }, + "expected": [ + "enum@$" + ], + "id": "enum-int-fault", + "relation": "violation", + "variant": { + "kind": "enum", + "schema": { + "enum": [ + false + ] + }, + "value": 0 + } + }, + { + "baseline": { + "kind": "enum", + "schema": { + "enum": [ + false + ] + }, + "value": false + }, + "expected": [], + "id": "enum-valid-control", + "relation": "preserve", + "variant": { + "kind": "enum", + "schema": { + "enum": [ + false, + true + ] + }, + "value": false + } + } + ], + "marshmallow-file-url": [ + { + "baseline": { + "kind": "file", + "value": "file:///tmp/a" + }, + "expected": [], + "id": "uppercase-scheme-valid", + "relation": "preserve", + "variant": { + "kind": "file", + "value": "FILE:///tmp/a" + } + }, + { + "baseline": { + "kind": "file", + "value": "file:///tmp/a" + }, + "expected": [], + "id": "lowercase-control", + "relation": "preserve", + "variant": { + "kind": "file", + "value": "file:///tmp/b" + } + } + ], + "marshmallow-idn-email": [ + { + "baseline": { + "kind": "email", + "value": "user@example.com" + }, + "expected": [], + "id": "unicode-tld-valid", + "relation": "preserve", + "variant": { + "kind": "email", + "value": "user@пример.рф" + } + }, + { + "baseline": { + "kind": "email", + "value": "user@example.com" + }, + "expected": [], + "id": "ascii-control", + "relation": "preserve", + "variant": { + "kind": "email", + "value": "user2@example.com" + } + } + ] + }, + "candidates": [ + { + "after": "3.0.2", + "before": "3.0.1", + "behavior": "JSON Schema enum/const distinguish 0 from false", + "decision": "include", + "id": "jsonschema-enum-bool-int", + "package": "jsonschema", + "reason": "Release notes explicitly call this a bug fix and identify before/after releases.", + "release_source": "https://github.com/python-jsonschema/jsonschema/blob/51cd75e399c760e5aa3adce600dcce1385756ab0/CHANGELOG.rst", + "source": "https://github.com/python-jsonschema/jsonschema/issues/575", + "source_id": "issue #575" + }, + { + "after": "4.17.3", + "before": "4.17.1", + "behavior": "Validator instantiation accepts cached boolean schemas", + "decision": "include", + "id": "jsonschema-bool-ref-cache", + "package": "jsonschema", + "reason": "Release notes explicitly call this a bug fix with a fixed release. Follow-up uses available 4.17.1; initial 4.17.2 could not be installed.", + "release_source": "https://github.com/python-jsonschema/jsonschema/blob/51cd75e399c760e5aa3adce600dcce1385756ab0/CHANGELOG.rst", + "source": "https://github.com/python-jsonschema/jsonschema/issues/1018", + "source_id": "issue #1018" + }, + { + "after": "4.2.1", + "before": "4.2.0", + "behavior": "URL validator handles uppercase FILE scheme", + "decision": "include", + "id": "marshmallow-file-url", + "package": "marshmallow", + "reason": "Release notes identify a validation fix and adjacent releases.", + "release_source": "https://github.com/marshmallow-code/marshmallow/blob/7f0792bd7a06f72e393ec866ac1e89e1502ccfe1/CHANGELOG.rst", + "source": "https://github.com/marshmallow-code/marshmallow/issues/2891", + "source_id": "issue #2891" + }, + { + "after": "4.2.4", + "before": "4.2.3", + "behavior": "Email validator accepts valid internationalized domain names", + "decision": "include", + "id": "marshmallow-idn-email", + "package": "marshmallow", + "reason": "Release notes identify a validator behavior fix and adjacent releases.", + "release_source": "https://github.com/marshmallow-code/marshmallow/blob/7f0792bd7a06f72e393ec866ac1e89e1502ccfe1/CHANGELOG.rst", + "source": "https://github.com/marshmallow-code/marshmallow/issues/2936", + "source_id": "issue #2936" + }, + { + "after": null, + "before": null, + "behavior": "format checker is not enabled by default", + "decision": "exclude", + "id": "jsonschema-format-default", + "package": "jsonschema", + "reason": "Documented configuration/default behavior; not a library bug under the frozen rule.", + "source": "https://github.com/python-jsonschema/jsonschema/issues/945", + "source_id": "issue #945" + }, + { + "after": null, + "before": null, + "behavior": "email format checker is incomplete", + "decision": "exclude", + "id": "jsonschema-email-completeness", + "package": "jsonschema", + "reason": "Optional format-check completeness dispute; no documented before/after fix established in screening.", + "source": "https://github.com/python-jsonschema/jsonschema/issues/403", + "source_id": "issue #403" + }, + { + "after": null, + "before": null, + "behavior": "built-in generic forward references", + "decision": "exclude", + "id": "pydantic-forward-ref-generics", + "package": "pydantic", + "reason": "Model/type resolution rather than value validation; no fixed before/after release established in screening.", + "source": "https://github.com/pydantic/pydantic/issues/6130", + "source_id": "issue #6130" + }, + { + "after": null, + "before": null, + "behavior": "IPv4Address default JSON schema warning", + "decision": "exclude", + "id": "pydantic-ipv4-default", + "package": "pydantic", + "reason": "Serialization/schema warning, with no documented before/after validation fix.", + "source": "https://github.com/pydantic/pydantic/issues/8942", + "source_id": "issue #8942" + }, + { + "after": null, + "before": null, + "behavior": "default/missing parameter transition", + "decision": "exclude", + "id": "marshmallow-required-default", + "package": "marshmallow", + "reason": "Intentional API/deprecation transition and configuration semantics, not a validator defect.", + "source": "https://github.com/marshmallow-code/marshmallow/blob/dev/CHANGELOG.rst", + "source_id": "issue #1742" + }, + { + "after": null, + "before": null, + "behavior": "ModuleType serialization fails with serialize_as_any", + "decision": "exclude", + "id": "pydantic-serialize-any-regression", + "package": "pydantic", + "reason": "Serialization-only report; outside this study. No validation fix release was established.", + "source": "https://github.com/pydantic/pydantic/issues/12348", + "source_id": "issue #12348" + } + ], + "followup_driver_sha256": "8b47d741fa6ba5afb359bb359224b1be0d2fd546e591486063b010e3e3a41418", + "parent_capture_manifest_sha256": "7e2c9e33a5a63fe60c57ae03508c6a203f332c86f05704615db05fbf8e14c8e5", + "preflight_revision": 2, + "requirements_sha256": { + "jsonschema-3.0.1.lock.txt": "667cfa6465f0b419f24a96f4f4ba4bf9ae53b6bce4cec84610b3319522da2e90", + "jsonschema-3.0.2.lock.txt": "bccf527bf2a1e4e26f1faa39459b0b9c78cce8c3e58e4087d055c122e11c760b", + "jsonschema-4.17.1.lock.txt": "cbcdc8d2ed03fbdbf57a9aaa3c6dc09bb67ca99ae89d16901549ce33d737c9c0", + "jsonschema-4.17.3.lock.txt": "839c46ca0b682f2f69ae74c921b15761456ea9107d3fb315ca2e0d7941107f8a", + "marshmallow-4.2.0.lock.txt": "42d87bc7dbcbece58f644985e692238b2a6273d097e85deffc337316a6aa0ca0", + "marshmallow-4.2.1.lock.txt": "58dc973cc15ec968b9e44708e7ea902faaddb5a380096a4f794ae370dc944b02", + "marshmallow-4.2.3.lock.txt": "1d7fa2cd4b51cd71b9a58cfd7bfb0a657d9f94dfc44cf957829af622faa14e57", + "marshmallow-4.2.4.lock.txt": "201c63e7fd2b26bde60fece7e85d6f35d651e3362edfa787cf6e536cd5fdd14b" + } +} diff --git a/benchmarks/historical-validator-results/compatibility-followup/prepared/manifest.json b/benchmarks/historical-validator-results/compatibility-followup/prepared/manifest.json new file mode 100644 index 0000000..f18ca82 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup/prepared/manifest.json @@ -0,0 +1,5 @@ +{ + "inputs.json": "09f08e3fed5cb93f5f71008ce18f91273428bb598a5682a79eaf90a03a7a46e1", + "protocol.md": "6aed28a758c7e3a25206e2bb4d61b0664db0b8e6f54f77a7832518732c724e5d", + "runner.py.txt": "cab4f5433556cf3c4448084ed716da2daa80eefdab1125d47fec264e83ca71df" +} diff --git a/benchmarks/historical-validator-results/compatibility-followup/prepared/protocol.md b/benchmarks/historical-validator-results/compatibility-followup/prepared/protocol.md new file mode 100644 index 0000000..b14b6a8 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup/prepared/protocol.md @@ -0,0 +1,33 @@ +# Historical compatibility follow-up protocol + +Prepared after the first historical run and before executing this follow-up. +This is explicitly exploratory compatibility recovery, not a preplanned success. +Original ledger, inputs, failures and results remain available unchanged. + +The first run reproduced the uppercase FILE-scheme fix, but both jsonschema 3.0.x +versions failed importing pkg_resources, jsonschema 4.17.2 was unavailable from the +package index, and the chosen IDN email was accepted by both Marshmallow versions. + +Two changes are fixed in advance of the new run: + +1. Add setuptools 70.3.0 (which supplies pkg_resources) to both jsonschema 3.0.x + environments. Keep all other captured dependencies pinned. +2. Use jsonschema 4.17.1 as the available predecessor to 4.17.3. Issue #1018 describes + the regression starting in 4.16.0; the pinned changelog associates the fix with + 4.17.3. This is not evidence about the unavailable 4.17.2 package. + +All eight paired inputs, labels, policies and adapters remain identical. The IDN +case is rerun unchanged; no search for a different successful reproducer is made. +The same two Marshmallow version comparisons act as checks on repeat execution. + +The follow-up driver, base collector, protocol, dependency requirements and parent +capture identity are frozen before new collection. The driver hash is included in +the prepared inputs; requirements hashes are verified before installing. Actual +installation reports retain artifact URLs and hashes, along with pip-freeze locks. +Offline replay and fresh package execution are distinct operations. Both the +initial and follow-up tables must be reported, including failed reproduction. + +A successful reproduction requires the issue-triggering outcome to change from +incorrect to correct and the preservation control to pass in both versions. +A fixed example is a historical regression reproduced, not a newly discovered bug, +an external adopter, or evidence of a sensitivity advantage over detailed pytest. diff --git a/benchmarks/historical-validator-results/compatibility-followup/prepared/runner.py.txt b/benchmarks/historical-validator-results/compatibility-followup/prepared/runner.py.txt new file mode 100644 index 0000000..8017b3a --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup/prepared/runner.py.txt @@ -0,0 +1,327 @@ +"""Version-pinned historical captures; isolated collection and verified offline replay.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import platform +import shutil +import subprocess +import sys +import venv +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "benchmarks/historical-validator-results" +PROTOCOL = ROOT / "paper/historical-validator-protocol.md" + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def write(path, value): + path.write_text(json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n") + + +def probe(sample): + """Only public payloads enter the third-party validator: no expected labels.""" + if sample["kind"] in ("enum", "refs"): + from jsonschema import Draft7Validator, RefResolver + + options = {} + if sample["kind"] == "refs": + options["resolver"] = RefResolver("", sample["schema"], store=sample["store"]) + errors = list(Draft7Validator(sample["schema"], **options).iter_errors(sample["value"])) + return { + "accepted": not errors, + "complete": True, + "violations": sorted({str(e.validator) + "@$" for e in errors}), + "messages": [e.message for e in errors], + "error": None, + } + from marshmallow import ValidationError, fields + + field = fields.Url(schemes={"file"}) if sample["kind"] == "file" else fields.Email() + try: + field.deserialize(sample["value"]) + except ValidationError as exc: + return { + "accepted": False, + "complete": True, + "violations": ["field@$"], + "messages": [str(exc)], + "error": None, + } + return {"accepted": True, "complete": True, "violations": [], "messages": [], "error": None} + + +def child(package): + # This process has only the pinned historical package and its dependencies. + samples = json.load(sys.stdin) + results = [] + for sample in samples: + try: + results.append(probe(sample)) + except Exception as exc: + results.append({"error": type(exc).__name__ + ": " + str(exc)}) + print( + json.dumps( + { + "python": platform.python_version(), + "package_version": importlib.metadata.version(package), + "results": results, + }, + sort_keys=True, + ) + ) + + +def freeze(): + target = OUT / "prepared" + target.mkdir(exist_ok=False) + shutil.copyfile(OUT / "frozen-inputs.json", target / "inputs.json") + shutil.copyfile(PROTOCOL, target / "protocol.md") + shutil.copyfile(__file__, target / "runner.py.txt") + write( + target / "manifest.json", + {name: sha(target / name) for name in ("inputs.json", "protocol.md", "runner.py.txt")}, + ) + + +def load_prepared(): + target = OUT / "prepared" + manifest = json.loads((target / "manifest.json").read_text()) + if set(manifest) != {"inputs.json", "protocol.md", "runner.py.txt"}: + raise ValueError("Unexpected prepared files") + for name, expected in manifest.items(): + if sha(target / name) != expected: + raise ValueError("Prepared content changed: " + name) + if sha(Path(__file__)) != manifest["runner.py.txt"] or sha(PROTOCOL) != manifest["protocol.md"]: + raise ValueError("Code or protocol differs from frozen source") + return json.loads((target / "inputs.json").read_text()) + + +def collect(work, output, lock_from=None): + data = load_prepared() + work.mkdir(parents=True, exist_ok=True) + output.mkdir(parents=True, exist_ok=False) + records = {} + for candidate in data["candidates"]: + if candidate["decision"] != "include": + continue + pairs = data["authored_inputs"][candidate["id"]] + samples = [pair[side] for pair in pairs for side in ("baseline", "variant")] + records[candidate["id"]] = {} + for version in (candidate["before"], candidate["after"]): + name = candidate["package"] + "-" + version + env = work / name + if env.exists(): + raise ValueError("Use a fresh environment directory: " + str(env)) + venv.EnvBuilder(with_pip=True, symlinks=True).create(env) + python = str(env / "bin/python") + pip_report = env / "install.json" + install_args = [candidate["package"] + "==" + version] + if lock_from: + install_args = ["-r", str(lock_from / (name + ".lock.txt"))] + install = subprocess.run( + [ + python, + "-m", + "pip", + "--disable-pip-version-check", + "install", + "--report", + str(pip_report), + *install_args, + ], + capture_output=True, + text=True, + timeout=180, + ) + record = {"install_returncode": install.returncode} + if install.returncode: + record["setup_error"] = install.stderr + records[candidate["id"]][version] = record + continue + installed = json.loads(pip_report.read_text()) + record["artifacts"] = [ + { + "name": item["metadata"]["name"], + "version": item["metadata"]["version"], + "url": item["download_info"]["url"], + "hashes": item["download_info"]["archive_info"].get("hashes", {}), + } + for item in installed["install"] + ] + lock = subprocess.check_output([python, "-m", "pip", "freeze"], text=True) + (output / (name + ".lock.txt")).write_text(lock) + record["lock_sha256"] = sha(output / (name + ".lock.txt")) + proc = subprocess.run( + [ + python, + "-I", + str(Path(__file__).resolve()), + "child", + "--package", + candidate["package"], + ], + input=json.dumps(samples), + capture_output=True, + text=True, + timeout=30, + ) + record.update( + {"returncode": proc.returncode, "stdout": proc.stdout, "stderr": proc.stderr} + ) + if proc.returncode == 0: + record["payload"] = json.loads(proc.stdout) + records[candidate["id"]][version] = record + print(name, "captured", flush=True) + write(output / "captured.json", records) + write( + output / "manifest.json", + { + "prepared_manifest_sha256": sha(OUT / "prepared/manifest.json"), + "files": {p.name: sha(p) for p in sorted(output.iterdir()) if p.is_file()}, + }, + ) + + +def replay(captured, output): + from narrative_contracts.model import digest + from narrative_contracts.mutations import Relation, Validity + from narrative_contracts.validator_audit import AuditCase, Obligation, Verdict, audit_validator + + data = load_prepared() + manifest = json.loads((captured / "manifest.json").read_text()) + if manifest["prepared_manifest_sha256"] != sha(OUT / "prepared/manifest.json"): + raise ValueError("Capture does not match preparation") + if "captured.json" not in manifest["files"] or any( + Path(n).name != n for n in manifest["files"] + ): + raise ValueError("Invalid capture manifest") + for name, expected in manifest["files"].items(): + if sha(captured / name) != expected: + raise ValueError("Capture changed: " + name) + records = json.loads((captured / "captured.json").read_text()) + reports, results = {}, [] + included = [c for c in data["candidates"] if c["decision"] == "include"] + if set(records) != {c["id"] for c in included}: + raise ValueError("Missing or extra candidate capture") + for candidate in included: + pairs = data["authored_inputs"][candidate["id"]] + versions = records[candidate["id"]] + outcomes = {} + for version in (candidate["before"], candidate["after"]): + record = versions[version] + if record["install_returncode"] or record["returncode"]: + outcomes[version] = {"execution": "unavailable"} + continue + payload = record["payload"] + if payload != json.loads(record["stdout"]) or payload["package_version"] != version: + raise ValueError("Captured version or raw output mismatch") + if len(payload["results"]) != 2 * len(pairs): + raise ValueError("Incomplete paired capture") + lookup = {} + for pair_index, pair in enumerate(pairs): + for side_index, side in enumerate(("baseline", "variant")): + key = digest(pair[side]) + observed = payload["results"][pair_index * 2 + side_index] + if key in lookup and lookup[key] != observed: + raise ValueError("Inconsistent repeated baseline capture") + lookup[key] = observed + + def adapter(sample): + record = lookup[digest(sample)] + if record["error"]: + raise RuntimeError(record["error"]) + return Verdict( + record["accepted"], + tuple(record["violations"]), + record["complete"], + tuple(record["messages"]), + ) + + cases = tuple( + AuditCase( + p["id"], + candidate["id"], + p["baseline"], + p["variant"], + Relation(p["relation"]), + tuple(p["expected"]), + Validity.VALID, + "Authored minimal reproduction of " + candidate["source"], + ) + for p in pairs + ) + report = audit_validator( + adapter, + cases, + obligations=(Obligation(candidate["id"], candidate["behavior"]),), + validator_id=f"{candidate['package']}/{version}", + ) + reports[candidate["id"] + "/" + version] = report.to_dict() + outcomes[version] = {c.id: c.outcome for c in report.cases} + trigger, control = pairs + before, after = outcomes[candidate["before"]], outcomes[candidate["after"]] + desired = "detected" if trigger["relation"] == "violation" else "preserved" + reproduced = ( + before.get(trigger["id"]) in ("survived", "regressed", "error") + and after.get(trigger["id"]) == desired + and before.get(control["id"]) == after.get(control["id"]) == "preserved" + ) + results.append( + { + "id": candidate["id"], + "source": candidate["source"], + "before_version": candidate["before"], + "after_version": candidate["after"], + "outcomes": outcomes, + "fix_reproduced": reproduced, + } + ) + summary = { + "schema_version": 1, + "screened": len(data["candidates"]), + "included": len(included), + "excluded": len(data["candidates"]) - len(included), + "fixes_reproduced": sum(r["fix_reproduced"] for r in results), + "results": results, + } + output.mkdir(parents=True, exist_ok=False) + write(output / "summary.json", summary) + write(output / "audit-reports.json", reports) + print(json.dumps(summary, indent=2)) + return summary + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("freeze", "collect", "replay", "child")) + parser.add_argument("--network", action="store_true") + parser.add_argument("--work", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument("--captured", type=Path) + parser.add_argument("--lock-from", type=Path) + parser.add_argument("--package") + args = parser.parse_args() + if args.action == "child": + child(args.package) + elif args.action == "freeze": + freeze() + elif args.action == "collect": + if not args.network or args.work is None or args.output is None: + parser.error("collect requires --network, --work and --output") + collect(args.work, args.output, args.lock_from) + else: + if args.captured is None or args.output is None: + parser.error("replay requires --captured and --output") + replay(args.captured, args.output) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/historical-validator-results/compatibility-followup/requirements/jsonschema-3.0.1.lock.txt b/benchmarks/historical-validator-results/compatibility-followup/requirements/jsonschema-3.0.1.lock.txt new file mode 100644 index 0000000..841bf59 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup/requirements/jsonschema-3.0.1.lock.txt @@ -0,0 +1,6 @@ +attrs==26.1.0 +jsonschema==3.0.1 +pyrsistent==0.20.0 +setuptools==84.0.0 +six==1.17.0 +setuptools==70.3.0 diff --git a/benchmarks/historical-validator-results/compatibility-followup/requirements/jsonschema-3.0.2.lock.txt b/benchmarks/historical-validator-results/compatibility-followup/requirements/jsonschema-3.0.2.lock.txt new file mode 100644 index 0000000..c295ec5 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup/requirements/jsonschema-3.0.2.lock.txt @@ -0,0 +1,6 @@ +attrs==26.1.0 +jsonschema==3.0.2 +pyrsistent==0.20.0 +setuptools==84.0.0 +six==1.17.0 +setuptools==70.3.0 diff --git a/benchmarks/historical-validator-results/compatibility-followup/requirements/jsonschema-4.17.1.lock.txt b/benchmarks/historical-validator-results/compatibility-followup/requirements/jsonschema-4.17.1.lock.txt new file mode 100644 index 0000000..eac7b5b --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup/requirements/jsonschema-4.17.1.lock.txt @@ -0,0 +1,3 @@ +attrs==26.1.0 +jsonschema==4.17.1 +pyrsistent==0.20.0 diff --git a/benchmarks/historical-validator-results/compatibility-followup/requirements/jsonschema-4.17.3.lock.txt b/benchmarks/historical-validator-results/compatibility-followup/requirements/jsonschema-4.17.3.lock.txt new file mode 100644 index 0000000..184ab11 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup/requirements/jsonschema-4.17.3.lock.txt @@ -0,0 +1,3 @@ +attrs==26.1.0 +jsonschema==4.17.3 +pyrsistent==0.20.0 diff --git a/benchmarks/historical-validator-results/compatibility-followup/requirements/marshmallow-4.2.0.lock.txt b/benchmarks/historical-validator-results/compatibility-followup/requirements/marshmallow-4.2.0.lock.txt new file mode 100644 index 0000000..410f96d --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup/requirements/marshmallow-4.2.0.lock.txt @@ -0,0 +1 @@ +marshmallow==4.2.0 diff --git a/benchmarks/historical-validator-results/compatibility-followup/requirements/marshmallow-4.2.1.lock.txt b/benchmarks/historical-validator-results/compatibility-followup/requirements/marshmallow-4.2.1.lock.txt new file mode 100644 index 0000000..8592200 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup/requirements/marshmallow-4.2.1.lock.txt @@ -0,0 +1 @@ +marshmallow==4.2.1 diff --git a/benchmarks/historical-validator-results/compatibility-followup/requirements/marshmallow-4.2.3.lock.txt b/benchmarks/historical-validator-results/compatibility-followup/requirements/marshmallow-4.2.3.lock.txt new file mode 100644 index 0000000..f15c45d --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup/requirements/marshmallow-4.2.3.lock.txt @@ -0,0 +1 @@ +marshmallow==4.2.3 diff --git a/benchmarks/historical-validator-results/compatibility-followup/requirements/marshmallow-4.2.4.lock.txt b/benchmarks/historical-validator-results/compatibility-followup/requirements/marshmallow-4.2.4.lock.txt new file mode 100644 index 0000000..90e1042 --- /dev/null +++ b/benchmarks/historical-validator-results/compatibility-followup/requirements/marshmallow-4.2.4.lock.txt @@ -0,0 +1 @@ +marshmallow==4.2.4 diff --git a/benchmarks/historical-validator-results/evaluation/audit-reports.json b/benchmarks/historical-validator-results/evaluation/audit-reports.json new file mode 100644 index 0000000..c9e977b --- /dev/null +++ b/benchmarks/historical-validator-results/evaluation/audit-reports.json @@ -0,0 +1,644 @@ +{ + "jsonschema-bool-ref-cache/4.17.3": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "23d1edd41d6aeba96e7014ab4f960b6025b66424e01c5f6445f9e687d47c0c0b", + "expected": [], + "id": "cached-boolean-valid", + "obligation": "jsonschema-bool-ref-cache", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/python-jsonschema/jsonschema/issues/1018", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "637d75abb2919f760e0cca7887cfffe980cd2fabdb0ea3cbce6fdddcda31f01c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "23d1edd41d6aeba96e7014ab4f960b6025b66424e01c5f6445f9e687d47c0c0b", + "expected": [], + "id": "cached-object-control", + "obligation": "jsonschema-bool-ref-cache", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/python-jsonschema/jsonschema/issues/1018", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "a1f217c76d39d422f165a35ee37ddd15f340ccbf3babf622ac450c8cf9909bdb" + } + ], + "corpus_digest": "51a25513137f3ebef8a66e4f4f413dee971902077d4e04ed8d8d25e126bace62", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Validator instantiation accepts cached boolean schemas", + "id": "jsonschema-bool-ref-cache", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 2 + }, + "detected_faults": 0, + "detection_score": null, + "eligible_controls": 2, + "eligible_faults": 0, + "obligations": [ + { + "assessment": "controls_only_no_faults", + "case_ids": [ + "cached-boolean-valid", + "cached-object-control" + ], + "counts": { + "preserved": 2 + }, + "description": "Validator instantiation accepts cached boolean schemas", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "jsonschema-bool-ref-cache", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 2, + "total": 2 + }, + "validator_id": "jsonschema/4.17.3" + }, + "jsonschema-enum-bool-int/3.0.1": { + "cases": [ + { + "baseline": null, + "baseline_digest": "7a3567c53252eb64648dedc0bd2efa892f4d90c8d43ca7a3e4c95ec095426acd", + "expected": [ + "enum@$" + ], + "id": "enum-int-fault", + "obligation": "jsonschema-enum-bool-int", + "outcome": "baseline_failed", + "provenance": "Authored minimal reproduction of https://github.com/python-jsonschema/jsonschema/issues/575", + "reason": "baseline_error: RuntimeError: ModuleNotFoundError: No module named 'pkg_resources'", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "85caeae8034324bdbc9eb23223c792bcf7a1f2730f4837aec95750e807539ffe" + }, + { + "baseline": null, + "baseline_digest": "7a3567c53252eb64648dedc0bd2efa892f4d90c8d43ca7a3e4c95ec095426acd", + "expected": [], + "id": "enum-valid-control", + "obligation": "jsonschema-enum-bool-int", + "outcome": "baseline_failed", + "provenance": "Authored minimal reproduction of https://github.com/python-jsonschema/jsonschema/issues/575", + "reason": "baseline_error: RuntimeError: ModuleNotFoundError: No module named 'pkg_resources'", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "2446758f1b7798b41baa464d7c0d72d91815b392d2f797b7afd30ab1f5fba147" + } + ], + "corpus_digest": "d918c96e79ae9e403b38755f3e55e55787ed7bec5e66923471224eb534a8b12b", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "JSON Schema enum/const distinguish 0 from false", + "id": "jsonschema-enum-bool-int", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "baseline_failed": 2 + }, + "detected_faults": 0, + "detection_score": null, + "eligible_controls": 0, + "eligible_faults": 0, + "obligations": [ + { + "assessment": "incomplete_evidence", + "case_ids": [ + "enum-int-fault", + "enum-valid-control" + ], + "counts": { + "baseline_failed": 2 + }, + "description": "JSON Schema enum/const distinguish 0 from false", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": true, + "has_known_gaps": false, + "id": "jsonschema-enum-bool-int", + "scope": "contract" + } + ], + "preservation_rate": null, + "preserved_controls": 0, + "total": 2 + }, + "validator_id": "jsonschema/3.0.1" + }, + "jsonschema-enum-bool-int/3.0.2": { + "cases": [ + { + "baseline": null, + "baseline_digest": "7a3567c53252eb64648dedc0bd2efa892f4d90c8d43ca7a3e4c95ec095426acd", + "expected": [ + "enum@$" + ], + "id": "enum-int-fault", + "obligation": "jsonschema-enum-bool-int", + "outcome": "baseline_failed", + "provenance": "Authored minimal reproduction of https://github.com/python-jsonschema/jsonschema/issues/575", + "reason": "baseline_error: RuntimeError: ModuleNotFoundError: No module named 'pkg_resources'", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "85caeae8034324bdbc9eb23223c792bcf7a1f2730f4837aec95750e807539ffe" + }, + { + "baseline": null, + "baseline_digest": "7a3567c53252eb64648dedc0bd2efa892f4d90c8d43ca7a3e4c95ec095426acd", + "expected": [], + "id": "enum-valid-control", + "obligation": "jsonschema-enum-bool-int", + "outcome": "baseline_failed", + "provenance": "Authored minimal reproduction of https://github.com/python-jsonschema/jsonschema/issues/575", + "reason": "baseline_error: RuntimeError: ModuleNotFoundError: No module named 'pkg_resources'", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "2446758f1b7798b41baa464d7c0d72d91815b392d2f797b7afd30ab1f5fba147" + } + ], + "corpus_digest": "d918c96e79ae9e403b38755f3e55e55787ed7bec5e66923471224eb534a8b12b", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "JSON Schema enum/const distinguish 0 from false", + "id": "jsonschema-enum-bool-int", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "baseline_failed": 2 + }, + "detected_faults": 0, + "detection_score": null, + "eligible_controls": 0, + "eligible_faults": 0, + "obligations": [ + { + "assessment": "incomplete_evidence", + "case_ids": [ + "enum-int-fault", + "enum-valid-control" + ], + "counts": { + "baseline_failed": 2 + }, + "description": "JSON Schema enum/const distinguish 0 from false", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": true, + "has_known_gaps": false, + "id": "jsonschema-enum-bool-int", + "scope": "contract" + } + ], + "preservation_rate": null, + "preserved_controls": 0, + "total": 2 + }, + "validator_id": "jsonschema/3.0.2" + }, + "marshmallow-file-url/4.2.0": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "8e0181a9a7ffb31fcbba12bac11a8e779ffb29d858ae6f80e2497df03de25d6a", + "expected": [], + "id": "uppercase-scheme-valid", + "obligation": "marshmallow-file-url", + "outcome": "regressed", + "provenance": "Authored minimal reproduction of https://github.com/marshmallow-code/marshmallow/issues/2891", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [ + "['Not a valid URL.']" + ], + "violations": [ + "field@$" + ] + }, + "variant_digest": "1c11eec5aa6a9d3631ee25994ad82fe370e5ebf0d0f1896c35db84ec1cbaeb31" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "8e0181a9a7ffb31fcbba12bac11a8e779ffb29d858ae6f80e2497df03de25d6a", + "expected": [], + "id": "lowercase-control", + "obligation": "marshmallow-file-url", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/marshmallow-code/marshmallow/issues/2891", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "fc2859890cc70e6029c2a2e07130c519ee56b0b8305960a2a6319888e012b5b3" + } + ], + "corpus_digest": "8372dc7ce05eca31612829b3af45404209289e23a319853266a2fd943026b844", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "URL validator handles uppercase FILE scheme", + "id": "marshmallow-file-url", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 1, + "regressed": 1 + }, + "detected_faults": 0, + "detection_score": null, + "eligible_controls": 2, + "eligible_faults": 0, + "obligations": [ + { + "assessment": "known_gaps", + "case_ids": [ + "uppercase-scheme-valid", + "lowercase-control" + ], + "counts": { + "preserved": 1, + "regressed": 1 + }, + "description": "URL validator handles uppercase FILE scheme", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": true, + "id": "marshmallow-file-url", + "scope": "contract" + } + ], + "preservation_rate": 0.5, + "preserved_controls": 1, + "total": 2 + }, + "validator_id": "marshmallow/4.2.0" + }, + "marshmallow-file-url/4.2.1": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "8e0181a9a7ffb31fcbba12bac11a8e779ffb29d858ae6f80e2497df03de25d6a", + "expected": [], + "id": "uppercase-scheme-valid", + "obligation": "marshmallow-file-url", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/marshmallow-code/marshmallow/issues/2891", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "1c11eec5aa6a9d3631ee25994ad82fe370e5ebf0d0f1896c35db84ec1cbaeb31" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "8e0181a9a7ffb31fcbba12bac11a8e779ffb29d858ae6f80e2497df03de25d6a", + "expected": [], + "id": "lowercase-control", + "obligation": "marshmallow-file-url", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/marshmallow-code/marshmallow/issues/2891", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "fc2859890cc70e6029c2a2e07130c519ee56b0b8305960a2a6319888e012b5b3" + } + ], + "corpus_digest": "8372dc7ce05eca31612829b3af45404209289e23a319853266a2fd943026b844", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "URL validator handles uppercase FILE scheme", + "id": "marshmallow-file-url", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 2 + }, + "detected_faults": 0, + "detection_score": null, + "eligible_controls": 2, + "eligible_faults": 0, + "obligations": [ + { + "assessment": "controls_only_no_faults", + "case_ids": [ + "uppercase-scheme-valid", + "lowercase-control" + ], + "counts": { + "preserved": 2 + }, + "description": "URL validator handles uppercase FILE scheme", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "marshmallow-file-url", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 2, + "total": 2 + }, + "validator_id": "marshmallow/4.2.1" + }, + "marshmallow-idn-email/4.2.3": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "e102831f1946c821ebba3d7d3e4eb088d810b857cf1169c07f5e9d16d06733e0", + "expected": [], + "id": "unicode-tld-valid", + "obligation": "marshmallow-idn-email", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/marshmallow-code/marshmallow/issues/2936", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "2d95c74f71d180a0d5ac62a81fafb19471a9d04abede546ea08e380c5789dc24" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "e102831f1946c821ebba3d7d3e4eb088d810b857cf1169c07f5e9d16d06733e0", + "expected": [], + "id": "ascii-control", + "obligation": "marshmallow-idn-email", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/marshmallow-code/marshmallow/issues/2936", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "01fd15a9b9c17e7048fc00711f58e979c1a0007be6d1e330f5f117bf706b3525" + } + ], + "corpus_digest": "be1c84141ec7eb9991b7399fdf0a9c0c209bfb654d2a24024f0a0cb14c258f33", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Email validator accepts valid internationalized domain names", + "id": "marshmallow-idn-email", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 2 + }, + "detected_faults": 0, + "detection_score": null, + "eligible_controls": 2, + "eligible_faults": 0, + "obligations": [ + { + "assessment": "controls_only_no_faults", + "case_ids": [ + "unicode-tld-valid", + "ascii-control" + ], + "counts": { + "preserved": 2 + }, + "description": "Email validator accepts valid internationalized domain names", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "marshmallow-idn-email", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 2, + "total": 2 + }, + "validator_id": "marshmallow/4.2.3" + }, + "marshmallow-idn-email/4.2.4": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "e102831f1946c821ebba3d7d3e4eb088d810b857cf1169c07f5e9d16d06733e0", + "expected": [], + "id": "unicode-tld-valid", + "obligation": "marshmallow-idn-email", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/marshmallow-code/marshmallow/issues/2936", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "2d95c74f71d180a0d5ac62a81fafb19471a9d04abede546ea08e380c5789dc24" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "e102831f1946c821ebba3d7d3e4eb088d810b857cf1169c07f5e9d16d06733e0", + "expected": [], + "id": "ascii-control", + "obligation": "marshmallow-idn-email", + "outcome": "preserved", + "provenance": "Authored minimal reproduction of https://github.com/marshmallow-code/marshmallow/issues/2936", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "01fd15a9b9c17e7048fc00711f58e979c1a0007be6d1e330f5f117bf706b3525" + } + ], + "corpus_digest": "be1c84141ec7eb9991b7399fdf0a9c0c209bfb654d2a24024f0a0cb14c258f33", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Email validator accepts valid internationalized domain names", + "id": "marshmallow-idn-email", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 2 + }, + "detected_faults": 0, + "detection_score": null, + "eligible_controls": 2, + "eligible_faults": 0, + "obligations": [ + { + "assessment": "controls_only_no_faults", + "case_ids": [ + "unicode-tld-valid", + "ascii-control" + ], + "counts": { + "preserved": 2 + }, + "description": "Email validator accepts valid internationalized domain names", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "marshmallow-idn-email", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 2, + "total": 2 + }, + "validator_id": "marshmallow/4.2.4" + } +} diff --git a/benchmarks/historical-validator-results/evaluation/summary.json b/benchmarks/historical-validator-results/evaluation/summary.json new file mode 100644 index 0000000..20df820 --- /dev/null +++ b/benchmarks/historical-validator-results/evaluation/summary.json @@ -0,0 +1,76 @@ +{ + "excluded": 6, + "fixes_reproduced": 1, + "included": 4, + "results": [ + { + "after_version": "3.0.2", + "before_version": "3.0.1", + "fix_reproduced": false, + "id": "jsonschema-enum-bool-int", + "outcomes": { + "3.0.1": { + "enum-int-fault": "baseline_failed", + "enum-valid-control": "baseline_failed" + }, + "3.0.2": { + "enum-int-fault": "baseline_failed", + "enum-valid-control": "baseline_failed" + } + }, + "source": "https://github.com/python-jsonschema/jsonschema/issues/575" + }, + { + "after_version": "4.17.3", + "before_version": "4.17.2", + "fix_reproduced": false, + "id": "jsonschema-bool-ref-cache", + "outcomes": { + "4.17.2": { + "execution": "unavailable" + }, + "4.17.3": { + "cached-boolean-valid": "preserved", + "cached-object-control": "preserved" + } + }, + "source": "https://github.com/python-jsonschema/jsonschema/issues/1018" + }, + { + "after_version": "4.2.1", + "before_version": "4.2.0", + "fix_reproduced": true, + "id": "marshmallow-file-url", + "outcomes": { + "4.2.0": { + "lowercase-control": "preserved", + "uppercase-scheme-valid": "regressed" + }, + "4.2.1": { + "lowercase-control": "preserved", + "uppercase-scheme-valid": "preserved" + } + }, + "source": "https://github.com/marshmallow-code/marshmallow/issues/2891" + }, + { + "after_version": "4.2.4", + "before_version": "4.2.3", + "fix_reproduced": false, + "id": "marshmallow-idn-email", + "outcomes": { + "4.2.3": { + "ascii-control": "preserved", + "unicode-tld-valid": "preserved" + }, + "4.2.4": { + "ascii-control": "preserved", + "unicode-tld-valid": "preserved" + } + }, + "source": "https://github.com/marshmallow-code/marshmallow/issues/2936" + } + ], + "schema_version": 1, + "screened": 10 +} diff --git a/benchmarks/historical-validator-results/frozen-inputs.json b/benchmarks/historical-validator-results/frozen-inputs.json new file mode 100644 index 0000000..94ec7f1 --- /dev/null +++ b/benchmarks/historical-validator-results/frozen-inputs.json @@ -0,0 +1,292 @@ +{ + "authored_inputs": { + "jsonschema-bool-ref-cache": [ + { + "baseline": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:ok": { + "type": "object" + } + }, + "value": {} + }, + "expected": [], + "id": "cached-boolean-valid", + "relation": "preserve", + "variant": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:any": true, + "urn:study:ok": { + "type": "object" + } + }, + "value": {} + } + }, + { + "baseline": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:ok": { + "type": "object" + } + }, + "value": {} + }, + "expected": [], + "id": "cached-object-control", + "relation": "preserve", + "variant": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:ok": { + "type": "object" + }, + "urn:study:other": { + "type": "array" + } + }, + "value": {} + } + } + ], + "jsonschema-enum-bool-int": [ + { + "baseline": { + "kind": "enum", + "schema": { + "enum": [ + false + ] + }, + "value": false + }, + "expected": [ + "enum@$" + ], + "id": "enum-int-fault", + "relation": "violation", + "variant": { + "kind": "enum", + "schema": { + "enum": [ + false + ] + }, + "value": 0 + } + }, + { + "baseline": { + "kind": "enum", + "schema": { + "enum": [ + false + ] + }, + "value": false + }, + "expected": [], + "id": "enum-valid-control", + "relation": "preserve", + "variant": { + "kind": "enum", + "schema": { + "enum": [ + false, + true + ] + }, + "value": false + } + } + ], + "marshmallow-file-url": [ + { + "baseline": { + "kind": "file", + "value": "file:///tmp/a" + }, + "expected": [], + "id": "uppercase-scheme-valid", + "relation": "preserve", + "variant": { + "kind": "file", + "value": "FILE:///tmp/a" + } + }, + { + "baseline": { + "kind": "file", + "value": "file:///tmp/a" + }, + "expected": [], + "id": "lowercase-control", + "relation": "preserve", + "variant": { + "kind": "file", + "value": "file:///tmp/b" + } + } + ], + "marshmallow-idn-email": [ + { + "baseline": { + "kind": "email", + "value": "user@example.com" + }, + "expected": [], + "id": "unicode-tld-valid", + "relation": "preserve", + "variant": { + "kind": "email", + "value": "user@пример.рф" + } + }, + { + "baseline": { + "kind": "email", + "value": "user@example.com" + }, + "expected": [], + "id": "ascii-control", + "relation": "preserve", + "variant": { + "kind": "email", + "value": "user2@example.com" + } + } + ] + }, + "candidates": [ + { + "after": "3.0.2", + "before": "3.0.1", + "behavior": "JSON Schema enum/const distinguish 0 from false", + "decision": "include", + "id": "jsonschema-enum-bool-int", + "package": "jsonschema", + "reason": "Release notes explicitly call this a bug fix and identify before/after releases.", + "release_source": "https://github.com/python-jsonschema/jsonschema/blob/51cd75e399c760e5aa3adce600dcce1385756ab0/CHANGELOG.rst", + "source": "https://github.com/python-jsonschema/jsonschema/issues/575", + "source_id": "issue #575" + }, + { + "after": "4.17.3", + "before": "4.17.2", + "behavior": "Validator instantiation accepts cached boolean schemas", + "decision": "include", + "id": "jsonschema-bool-ref-cache", + "package": "jsonschema", + "reason": "Release notes explicitly call this a bug fix with a fixed release.", + "release_source": "https://github.com/python-jsonschema/jsonschema/blob/51cd75e399c760e5aa3adce600dcce1385756ab0/CHANGELOG.rst", + "source": "https://github.com/python-jsonschema/jsonschema/issues/1018", + "source_id": "issue #1018" + }, + { + "after": "4.2.1", + "before": "4.2.0", + "behavior": "URL validator handles uppercase FILE scheme", + "decision": "include", + "id": "marshmallow-file-url", + "package": "marshmallow", + "reason": "Release notes identify a validation fix and adjacent releases.", + "release_source": "https://github.com/marshmallow-code/marshmallow/blob/7f0792bd7a06f72e393ec866ac1e89e1502ccfe1/CHANGELOG.rst", + "source": "https://github.com/marshmallow-code/marshmallow/issues/2891", + "source_id": "issue #2891" + }, + { + "after": "4.2.4", + "before": "4.2.3", + "behavior": "Email validator accepts valid internationalized domain names", + "decision": "include", + "id": "marshmallow-idn-email", + "package": "marshmallow", + "reason": "Release notes identify a validator behavior fix and adjacent releases.", + "release_source": "https://github.com/marshmallow-code/marshmallow/blob/7f0792bd7a06f72e393ec866ac1e89e1502ccfe1/CHANGELOG.rst", + "source": "https://github.com/marshmallow-code/marshmallow/issues/2936", + "source_id": "issue #2936" + }, + { + "after": null, + "before": null, + "behavior": "format checker is not enabled by default", + "decision": "exclude", + "id": "jsonschema-format-default", + "package": "jsonschema", + "reason": "Documented configuration/default behavior; not a library bug under the frozen rule.", + "source": "https://github.com/python-jsonschema/jsonschema/issues/945", + "source_id": "issue #945" + }, + { + "after": null, + "before": null, + "behavior": "email format checker is incomplete", + "decision": "exclude", + "id": "jsonschema-email-completeness", + "package": "jsonschema", + "reason": "Optional format-check completeness dispute; no documented before/after fix established in screening.", + "source": "https://github.com/python-jsonschema/jsonschema/issues/403", + "source_id": "issue #403" + }, + { + "after": null, + "before": null, + "behavior": "built-in generic forward references", + "decision": "exclude", + "id": "pydantic-forward-ref-generics", + "package": "pydantic", + "reason": "Model/type resolution rather than value validation; no fixed before/after release established in screening.", + "source": "https://github.com/pydantic/pydantic/issues/6130", + "source_id": "issue #6130" + }, + { + "after": null, + "before": null, + "behavior": "IPv4Address default JSON schema warning", + "decision": "exclude", + "id": "pydantic-ipv4-default", + "package": "pydantic", + "reason": "Serialization/schema warning, with no documented before/after validation fix.", + "source": "https://github.com/pydantic/pydantic/issues/8942", + "source_id": "issue #8942" + }, + { + "after": null, + "before": null, + "behavior": "default/missing parameter transition", + "decision": "exclude", + "id": "marshmallow-required-default", + "package": "marshmallow", + "reason": "Intentional API/deprecation transition and configuration semantics, not a validator defect.", + "source": "https://github.com/marshmallow-code/marshmallow/blob/dev/CHANGELOG.rst", + "source_id": "issue #1742" + }, + { + "after": null, + "before": null, + "behavior": "ModuleType serialization fails with serialize_as_any", + "decision": "exclude", + "id": "pydantic-serialize-any-regression", + "package": "pydantic", + "reason": "Serialization-only report; outside this study. No validation fix release was established.", + "source": "https://github.com/pydantic/pydantic/issues/12348", + "source_id": "issue #12348" + } + ], + "preflight_revision": 2 +} diff --git a/benchmarks/historical-validator-results/frozen-inputs.sha256 b/benchmarks/historical-validator-results/frozen-inputs.sha256 new file mode 100644 index 0000000..5aaa643 --- /dev/null +++ b/benchmarks/historical-validator-results/frozen-inputs.sha256 @@ -0,0 +1 @@ +b1dd2e43d625ca01e9ff588b59582931fed57f563ef4f178b8ec341be1941478 frozen-inputs.json diff --git a/benchmarks/historical-validator-results/preflight-v1/frozen-inputs.json b/benchmarks/historical-validator-results/preflight-v1/frozen-inputs.json new file mode 100644 index 0000000..01f8dfd --- /dev/null +++ b/benchmarks/historical-validator-results/preflight-v1/frozen-inputs.json @@ -0,0 +1,191 @@ +{ + "authored_inputs": { + "jsonschema-bool-ref-cache": [ + { + "id": "boolean-ref-control", + "invalid": false, + "schema": { + "$defs": { + "ok": true + }, + "$ref": "#/$defs/ok" + }, + "value": {} + }, + { + "id": "boolean-ref-fault", + "invalid": true, + "schema": { + "$defs": { + "no": false + }, + "$ref": "#/$defs/no" + }, + "value": {} + } + ], + "jsonschema-enum-bool-int": [ + { + "id": "enum-bool-control", + "invalid": false, + "schema": { + "enum": [ + false + ] + }, + "value": false + }, + { + "id": "enum-int-variant", + "invalid": true, + "schema": { + "enum": [ + false + ] + }, + "value": 0 + } + ], + "marshmallow-file-url": [ + { + "id": "file-url-control", + "invalid": false, + "value": "file:///tmp/a" + }, + { + "id": "file-url-variant", + "invalid": true, + "value": "FILE:///tmp/a" + } + ], + "marshmallow-idn-email": [ + { + "id": "idn-email-control", + "invalid": false, + "value": "user@example.com" + }, + { + "id": "idn-email-variant", + "invalid": true, + "value": "user@b\u00fccher.de" + } + ] + }, + "candidates": [ + { + "after": "3.0.2", + "before": "3.0.1", + "behavior": "JSON Schema enum/const distinguish 0 from false", + "decision": "include", + "id": "jsonschema-enum-bool-int", + "package": "jsonschema", + "reason": "Release notes explicitly call this a bug fix and identify before/after releases.", + "source": "https://github.com/python-jsonschema/jsonschema/blob/main/CHANGELOG.rst", + "source_id": "issue #575" + }, + { + "after": "4.17.3", + "before": "4.17.2", + "behavior": "Validator instantiation accepts cached boolean schemas", + "decision": "include", + "id": "jsonschema-bool-ref-cache", + "package": "jsonschema", + "reason": "Release notes explicitly call this a bug fix with a fixed release.", + "source": "https://github.com/python-jsonschema/jsonschema/blob/main/CHANGELOG.rst", + "source_id": "issue #1018" + }, + { + "after": "4.2.1", + "before": "4.2.0", + "behavior": "URL validator handles uppercase FILE scheme", + "decision": "include", + "id": "marshmallow-file-url", + "package": "marshmallow", + "reason": "Release notes identify a validation fix and adjacent releases.", + "source": "https://github.com/marshmallow-code/marshmallow/blob/dev/CHANGELOG.rst", + "source_id": "issue #2891" + }, + { + "after": "4.2.4", + "before": "4.2.3", + "behavior": "Email validator accepts valid internationalized domain names", + "decision": "include", + "id": "marshmallow-idn-email", + "package": "marshmallow", + "reason": "Release notes identify a validator behavior fix and adjacent releases.", + "source": "https://github.com/marshmallow-code/marshmallow/blob/dev/CHANGELOG.rst", + "source_id": "issues #2821/#2936" + }, + { + "after": "4.6.0", + "before": "4.5.1", + "behavior": "format checker is not enabled by default", + "decision": "exclude", + "id": "jsonschema-format-default", + "package": "jsonschema", + "reason": "Documented configuration/default behavior; not a library bug under the frozen rule.", + "source": "https://github.com/python-jsonschema/jsonschema/issues/945", + "source_id": "issue #945" + }, + { + "after": "3.0.1", + "before": "3.0.0", + "behavior": "email format checker is incomplete", + "decision": "exclude", + "id": "jsonschema-email-completeness", + "package": "jsonschema", + "reason": "Issue describes an intentionally limited optional format implementation, not a fixed acceptance contract.", + "source": "https://github.com/python-jsonschema/jsonschema/issues/403", + "source_id": "issue #403" + }, + { + "after": "2.3.0", + "before": "2.2.1", + "behavior": "built-in generic forward references", + "decision": "exclude", + "id": "pydantic-forward-ref-generics", + "package": "pydantic", + "reason": "Issue was marked unconfirmed and the behavior is typing resolution rather than a validator acceptance fix.", + "source": "https://github.com/pydantic/pydantic/issues/6130", + "source_id": "issue #6130" + }, + { + "after": "2.7.0", + "before": "2.6.2", + "behavior": "IPv4Address default JSON schema warning", + "decision": "exclude", + "id": "pydantic-ipv4-default", + "package": "pydantic", + "reason": "Serialization/schema warning, with no documented before/after validation fix.", + "source": "https://github.com/pydantic/pydantic/issues/8942", + "source_id": "issue #8942" + }, + { + "after": "3.13.0", + "before": "3.12.0", + "behavior": "default/missing parameter transition", + "decision": "exclude", + "id": "marshmallow-required-default", + "package": "marshmallow", + "reason": "Intentional API/deprecation transition and configuration semantics, not a validator defect.", + "source": "https://github.com/marshmallow-code/marshmallow/blob/dev/CHANGELOG.rst", + "source_id": "issue #1742" + }, + { + "after": "2.12.0", + "before": "2.11.0", + "behavior": "serialize_as_any IPv4 serialization", + "decision": "exclude", + "id": "pydantic-serialize-any-regression", + "package": "pydantic", + "reason": "Serialization regression; not a validation acceptance contract and no fixed after release in the primary record.", + "source": "https://github.com/pydantic/pydantic/issues/12348", + "source_id": "issue #12348" + } + ], + "protocol_sha256": "de180ac023177e9a72e38fa17f5e5cd036844533d1eacdc9f2d4bedc7a7ab1d0", + "source_hashes": { + "benchmarks/historical_validator_study.py": "69037aec28acf14af2c28c2ec8a35e1daf21a901369f7893e4ef7421860a075d", + "paper/historical-validator-protocol.md": "de180ac023177e9a72e38fa17f5e5cd036844533d1eacdc9f2d4bedc7a7ab1d0" + } +} diff --git a/benchmarks/historical-validator-results/preflight-v1/protocol.md b/benchmarks/historical-validator-results/preflight-v1/protocol.md new file mode 100644 index 0000000..0b3d149 --- /dev/null +++ b/benchmarks/historical-validator-results/preflight-v1/protocol.md @@ -0,0 +1,34 @@ +# Historical validator study protocol + +Status: frozen 2026-09-29 (America/Argentina/Buenos_Aires). + +This is a retrospective reproduction study, not an independent preregistration. +The unit is a documented validator defect with a public primary issue, pull +request, advisory, or release note naming the fix. A candidate is eligible only +when the upstream record describes an observable validation or validation-adjacent +behavior change and identifies both a before and after release (or commits). +Configuration omissions, documented defaults, feature requests, type-checking +only changes, and regressions without a later fix are excluded. + +Before any reproduction, the candidate ledger, minimal inputs, expected behavior, +adapter source, and this protocol are hashed. Reproductions use fresh virtual +environments below `work/`; no package is installed into the study checkout or +global interpreter. Network access is explicit (`collect --network`). The offline +replay consumes only the captured normalized verdicts and validates their hashes. + +Each included candidate has one valid baseline control and one invalid or newly +accepted variant. The control must remain accepted in both versions. The expected +finding is authored from the primary record and kept separate from adapter logic. +`audit_validator` outcomes are reported verbatim; errors, unsupported cases, and +failed baselines remain in the ledger and do not count as fixes. This protocol +does not tune library rules and makes no claim that the harness outperforms pytest. + +The frozen target is 8--12 candidates, with at least three reproduced fixes when +the available Python/runtime/package constraints permit. Fewer successful cases +are reported honestly. For every candidate we retain package/version/runtime, +source URL and ID, rationale, input digest, adapter hash, raw before/after output, +normalized audit report, and environment `pip freeze` hash. + +Primary sources are upstream GitHub issues, pull requests, release notes, or +security advisories. Minimal reproducers are authored for this study and do not +copy upstream test files. diff --git a/benchmarks/historical-validator-results/preflight-v1/runner.py.txt b/benchmarks/historical-validator-results/preflight-v1/runner.py.txt new file mode 100644 index 0000000..ef30871 --- /dev/null +++ b/benchmarks/historical-validator-results/preflight-v1/runner.py.txt @@ -0,0 +1,425 @@ +"""Frozen historical validator ledger and isolated reproduction/replay tool. + +``freeze`` writes authored inputs and the candidate ledger. ``setup`` creates +per-version virtual environments below ``work/``. ``collect --network`` runs +the minimal cases in those environments. ``replay`` is offline and checks the +captured normalized verdicts with :func:`audit_validator`. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import os +import platform +import subprocess +import venv +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Callable + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "benchmarks" / "historical-validator-results" +WORK = ROOT.parent / "work" / "historical-validator-study" +PROTOCOL = ROOT / "paper" / "historical-validator-protocol.md" +SCRIPT = Path(__file__).resolve() + + +@dataclass(frozen=True) +class Candidate: + id: str + package: str + before: str + after: str + source: str + source_id: str + behavior: str + decision: str + reason: str + + +CANDIDATES = ( + Candidate( + "jsonschema-enum-bool-int", + "jsonschema", + "3.0.1", + "3.0.2", + "https://github.com/python-jsonschema/jsonschema/blob/main/CHANGELOG.rst", + "issue #575", + "JSON Schema enum/const distinguish 0 from false", + "include", + "Release notes explicitly call this a bug fix and identify before/after releases.", + ), + Candidate( + "jsonschema-bool-ref-cache", + "jsonschema", + "4.17.2", + "4.17.3", + "https://github.com/python-jsonschema/jsonschema/blob/main/CHANGELOG.rst", + "issue #1018", + "Validator instantiation accepts cached boolean schemas", + "include", + "Release notes explicitly call this a bug fix with a fixed release.", + ), + Candidate( + "marshmallow-file-url", + "marshmallow", + "4.2.0", + "4.2.1", + "https://github.com/marshmallow-code/marshmallow/blob/dev/CHANGELOG.rst", + "issue #2891", + "URL validator handles uppercase FILE scheme", + "include", + "Release notes identify a validation fix and adjacent releases.", + ), + Candidate( + "marshmallow-idn-email", + "marshmallow", + "4.2.3", + "4.2.4", + "https://github.com/marshmallow-code/marshmallow/blob/dev/CHANGELOG.rst", + "issues #2821/#2936", + "Email validator accepts valid internationalized domain names", + "include", + "Release notes identify a validator behavior fix and adjacent releases.", + ), + Candidate( + "jsonschema-format-default", + "jsonschema", + "4.5.1", + "4.6.0", + "https://github.com/python-jsonschema/jsonschema/issues/945", + "issue #945", + "format checker is not enabled by default", + "exclude", + "Documented configuration/default behavior; not a library bug under the frozen rule.", + ), + Candidate( + "jsonschema-email-completeness", + "jsonschema", + "3.0.0", + "3.0.1", + "https://github.com/python-jsonschema/jsonschema/issues/403", + "issue #403", + "email format checker is incomplete", + "exclude", + "Issue describes an intentionally limited optional format implementation, not a fixed acceptance contract.", + ), + Candidate( + "pydantic-forward-ref-generics", + "pydantic", + "2.2.1", + "2.3.0", + "https://github.com/pydantic/pydantic/issues/6130", + "issue #6130", + "built-in generic forward references", + "exclude", + "Issue was marked unconfirmed and the behavior is typing resolution rather than a validator acceptance fix.", + ), + Candidate( + "pydantic-ipv4-default", + "pydantic", + "2.6.2", + "2.7.0", + "https://github.com/pydantic/pydantic/issues/8942", + "issue #8942", + "IPv4Address default JSON schema warning", + "exclude", + "Serialization/schema warning, with no documented before/after validation fix.", + ), + Candidate( + "marshmallow-required-default", + "marshmallow", + "3.12.0", + "3.13.0", + "https://github.com/marshmallow-code/marshmallow/blob/dev/CHANGELOG.rst", + "issue #1742", + "default/missing parameter transition", + "exclude", + "Intentional API/deprecation transition and configuration semantics, not a validator defect.", + ), + Candidate( + "pydantic-serialize-any-regression", + "pydantic", + "2.11.0", + "2.12.0", + "https://github.com/pydantic/pydantic/issues/12348", + "issue #12348", + "serialize_as_any IPv4 serialization", + "exclude", + "Serialization regression; not a validation acceptance contract and no fixed after release in the primary record.", + ), +) + + +def digest(value: Any) -> str: + return hashlib.sha256( + json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + + +def source_hashes() -> dict[str, str]: + return { + str(p.relative_to(ROOT)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in (PROTOCOL, SCRIPT) + } + + +def authored_cases(candidate_id: str) -> list[dict[str, Any]]: + if candidate_id == "jsonschema-enum-bool-int": + return [ + { + "id": "enum-bool-control", + "schema": {"enum": [False]}, + "value": False, + "invalid": False, + }, + {"id": "enum-int-variant", "schema": {"enum": [False]}, "value": 0, "invalid": True}, + ] + if candidate_id == "jsonschema-bool-ref-cache": + return [ + { + "id": "boolean-ref-control", + "schema": {"$ref": "#/$defs/ok", "$defs": {"ok": True}}, + "value": {}, + "invalid": False, + }, + { + "id": "boolean-ref-fault", + "schema": {"$ref": "#/$defs/no", "$defs": {"no": False}}, + "value": {}, + "invalid": True, + }, + ] + if candidate_id == "marshmallow-file-url": + return [ + {"id": "file-url-control", "value": "file:///tmp/a", "invalid": False}, + {"id": "file-url-variant", "value": "FILE:///tmp/a", "invalid": True}, + ] + if candidate_id == "marshmallow-idn-email": + return [ + {"id": "idn-email-control", "value": "user@example.com", "invalid": False}, + {"id": "idn-email-variant", "value": "user@bücher.de", "invalid": True}, + ] + return [] + + +def _jsonschema_adapter(sample: dict[str, Any]) -> Any: + from jsonschema import Draft7Validator + + from narrative_contracts.validator_audit import Verdict + + try: + errors = list(Draft7Validator(sample["schema"]).iter_errors(sample["value"])) + return Verdict(accepted=not errors, violations=("schema@/value",) if errors else ()) + except Exception as exc: + raise RuntimeError(str(exc)) from exc + + +def _marshmallow_adapter(sample: dict[str, Any]) -> Any: + from marshmallow import ValidationError, fields + + from narrative_contracts.validator_audit import Verdict + + field = fields.Url() if "url" in sample["id"] else fields.Email() + try: + field.deserialize(sample["value"]) + except ValidationError: + return Verdict(accepted=False, violations=("schema@/value",)) + return Verdict(accepted=True) + + +def _cases(candidate_id: str): + from narrative_contracts.mutations import Relation, Validity + from narrative_contracts.validator_audit import AuditCase, Obligation + + cases = authored_cases(candidate_id) + result = [] + for item in cases: + result.append( + AuditCase( + id=item["id"], + obligation="historical-validator", + baseline=cases[0], + variant=item, + relation=Relation.VIOLATION if item["invalid"] else Relation.PRESERVE, + expected=("schema@/value",) if item["invalid"] else (), + validity=Validity.VALID, + provenance="Minimal authored control/variant from the cited primary release record.", + ) + ) + return tuple(result), ( + Obligation( + "historical-validator", + "The validator must distinguish the documented valid and invalid values.", + ), + ) + + +def _run(candidate_id: str) -> dict[str, Any]: + from narrative_contracts.validator_audit import audit_validator + + candidate = next(c for c in CANDIDATES if c.id == candidate_id) + if candidate.package == "jsonschema": + adapter: Callable[[dict[str, Any]], Any] = _jsonschema_adapter + else: + adapter = _marshmallow_adapter + cases, obligations = _cases(candidate_id) + report = audit_validator( + adapter, cases, obligations=obligations, validator_id=f"{candidate.package}/{candidate_id}" + ) + return { + "candidate": asdict(candidate), + "runtime": platform.python_version(), + "package_version": importlib.metadata.version(candidate.package), + "adapter_hash": source_hashes()[str(SCRIPT.relative_to(ROOT))], + "raw_cases": [asdict(c) for c in cases], + "report": report.to_dict(), + } + + +def freeze() -> None: + OUT.mkdir(parents=True, exist_ok=True) + payload = { + "protocol_sha256": hashlib.sha256(PROTOCOL.read_bytes()).hexdigest(), + "source_hashes": source_hashes(), + "candidates": [asdict(c) for c in CANDIDATES], + "authored_inputs": { + c.id: authored_cases(c.id) for c in CANDIDATES if c.decision == "include" + }, + } + (OUT / "frozen-inputs.json").write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + (OUT / "frozen-inputs.sha256").write_text( + hashlib.sha256((OUT / "frozen-inputs.json").read_bytes()).hexdigest() + + " frozen-inputs.json\n" + ) + + +def setup() -> None: + freeze() + WORK.mkdir(parents=True, exist_ok=True) + for c in CANDIDATES: + if c.decision != "include": + continue + for version in (c.before, c.after): + env = WORK / f"{c.package}-{version}" + if not (env / "bin" / "python").exists(): + venv.EnvBuilder(with_pip=True, clear=False).create(env) + subprocess.run( + [ + str(env / "bin" / "python"), + "-m", + "pip", + "install", + "--disable-pip-version-check", + f"{c.package}=={version}", + ], + check=True, + ) + freeze_file = env / "pip-freeze.txt" + freeze_file.write_text( + subprocess.check_output( + [str(env / "bin" / "python"), "-m", "pip", "freeze"], text=True + ) + ) + + +def collect() -> None: + freeze() + OUT.mkdir(parents=True, exist_ok=True) + records = [] + for c in CANDIDATES: + if c.decision != "include": + continue + versions = {} + for version in (c.before, c.after): + env_python = WORK / f"{c.package}-{version}" / "bin" / "python" + if not env_python.exists(): + raise SystemExit(f"missing isolated environment {env_python}; run setup first") + env = dict(os.environ, PYTHONPATH=str(ROOT / "src")) + proc = subprocess.run( + [str(env_python), str(SCRIPT), "--run-case", c.id], + env=env, + capture_output=True, + text=True, + ) + versions[version] = { + "returncode": proc.returncode, + "stdout": proc.stdout, + "stderr": proc.stderr, + } + records.append( + { + "candidate": asdict(c), + "versions": versions, + "inputs_sha256": hashlib.sha256( + (OUT / "frozen-inputs.json").read_bytes() + ).hexdigest(), + } + ) + (OUT / "captured-verdicts.json").write_text( + json.dumps( + { + "protocol_sha256": hashlib.sha256(PROTOCOL.read_bytes()).hexdigest(), + "records": records, + }, + indent=2, + sort_keys=True, + ) + + "\n" + ) + + +def replay() -> dict[str, Any]: + frozen = json.loads((OUT / "frozen-inputs.json").read_text()) + if frozen["protocol_sha256"] != hashlib.sha256(PROTOCOL.read_bytes()).hexdigest(): + raise AssertionError("protocol hash mismatch") + captured_path = OUT / "captured-verdicts.json" + if not captured_path.exists(): + return {"status": "not_collected", "candidates": []} + captured = json.loads(captured_path.read_text()) + if captured["protocol_sha256"] != frozen["protocol_sha256"]: + raise AssertionError("captured protocol hash mismatch") + return { + "status": "replayed", + "candidates": [ + { + "id": r["candidate"]["id"], + "before": r["versions"][r["candidate"]["before"]]["returncode"], + "after": r["versions"][r["candidate"]["after"]]["returncode"], + } + for r in captured["records"] + ], + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "command", nargs="?", choices=("freeze", "setup", "collect", "replay"), default="replay" + ) + parser.add_argument( + "--network", action="store_true", help="explicitly permit package downloads" + ) + parser.add_argument("--run-case") + args = parser.parse_args() + if args.run_case: + print(json.dumps(_run(args.run_case), sort_keys=True, default=str)) + elif args.command == "freeze": + freeze() + elif args.command == "setup": + if not args.network: + raise SystemExit("setup downloads packages; pass --network explicitly") + setup() + elif args.command == "collect": + if not args.network: + raise SystemExit("collect requires isolated packages; pass --network explicitly") + collect() + else: + print(json.dumps(replay(), indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/historical-validator-results/prepared-v2-bootstrap/inputs.json b/benchmarks/historical-validator-results/prepared-v2-bootstrap/inputs.json new file mode 100644 index 0000000..94ec7f1 --- /dev/null +++ b/benchmarks/historical-validator-results/prepared-v2-bootstrap/inputs.json @@ -0,0 +1,292 @@ +{ + "authored_inputs": { + "jsonschema-bool-ref-cache": [ + { + "baseline": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:ok": { + "type": "object" + } + }, + "value": {} + }, + "expected": [], + "id": "cached-boolean-valid", + "relation": "preserve", + "variant": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:any": true, + "urn:study:ok": { + "type": "object" + } + }, + "value": {} + } + }, + { + "baseline": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:ok": { + "type": "object" + } + }, + "value": {} + }, + "expected": [], + "id": "cached-object-control", + "relation": "preserve", + "variant": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:ok": { + "type": "object" + }, + "urn:study:other": { + "type": "array" + } + }, + "value": {} + } + } + ], + "jsonschema-enum-bool-int": [ + { + "baseline": { + "kind": "enum", + "schema": { + "enum": [ + false + ] + }, + "value": false + }, + "expected": [ + "enum@$" + ], + "id": "enum-int-fault", + "relation": "violation", + "variant": { + "kind": "enum", + "schema": { + "enum": [ + false + ] + }, + "value": 0 + } + }, + { + "baseline": { + "kind": "enum", + "schema": { + "enum": [ + false + ] + }, + "value": false + }, + "expected": [], + "id": "enum-valid-control", + "relation": "preserve", + "variant": { + "kind": "enum", + "schema": { + "enum": [ + false, + true + ] + }, + "value": false + } + } + ], + "marshmallow-file-url": [ + { + "baseline": { + "kind": "file", + "value": "file:///tmp/a" + }, + "expected": [], + "id": "uppercase-scheme-valid", + "relation": "preserve", + "variant": { + "kind": "file", + "value": "FILE:///tmp/a" + } + }, + { + "baseline": { + "kind": "file", + "value": "file:///tmp/a" + }, + "expected": [], + "id": "lowercase-control", + "relation": "preserve", + "variant": { + "kind": "file", + "value": "file:///tmp/b" + } + } + ], + "marshmallow-idn-email": [ + { + "baseline": { + "kind": "email", + "value": "user@example.com" + }, + "expected": [], + "id": "unicode-tld-valid", + "relation": "preserve", + "variant": { + "kind": "email", + "value": "user@пример.рф" + } + }, + { + "baseline": { + "kind": "email", + "value": "user@example.com" + }, + "expected": [], + "id": "ascii-control", + "relation": "preserve", + "variant": { + "kind": "email", + "value": "user2@example.com" + } + } + ] + }, + "candidates": [ + { + "after": "3.0.2", + "before": "3.0.1", + "behavior": "JSON Schema enum/const distinguish 0 from false", + "decision": "include", + "id": "jsonschema-enum-bool-int", + "package": "jsonschema", + "reason": "Release notes explicitly call this a bug fix and identify before/after releases.", + "release_source": "https://github.com/python-jsonschema/jsonschema/blob/51cd75e399c760e5aa3adce600dcce1385756ab0/CHANGELOG.rst", + "source": "https://github.com/python-jsonschema/jsonschema/issues/575", + "source_id": "issue #575" + }, + { + "after": "4.17.3", + "before": "4.17.2", + "behavior": "Validator instantiation accepts cached boolean schemas", + "decision": "include", + "id": "jsonschema-bool-ref-cache", + "package": "jsonschema", + "reason": "Release notes explicitly call this a bug fix with a fixed release.", + "release_source": "https://github.com/python-jsonschema/jsonschema/blob/51cd75e399c760e5aa3adce600dcce1385756ab0/CHANGELOG.rst", + "source": "https://github.com/python-jsonschema/jsonschema/issues/1018", + "source_id": "issue #1018" + }, + { + "after": "4.2.1", + "before": "4.2.0", + "behavior": "URL validator handles uppercase FILE scheme", + "decision": "include", + "id": "marshmallow-file-url", + "package": "marshmallow", + "reason": "Release notes identify a validation fix and adjacent releases.", + "release_source": "https://github.com/marshmallow-code/marshmallow/blob/7f0792bd7a06f72e393ec866ac1e89e1502ccfe1/CHANGELOG.rst", + "source": "https://github.com/marshmallow-code/marshmallow/issues/2891", + "source_id": "issue #2891" + }, + { + "after": "4.2.4", + "before": "4.2.3", + "behavior": "Email validator accepts valid internationalized domain names", + "decision": "include", + "id": "marshmallow-idn-email", + "package": "marshmallow", + "reason": "Release notes identify a validator behavior fix and adjacent releases.", + "release_source": "https://github.com/marshmallow-code/marshmallow/blob/7f0792bd7a06f72e393ec866ac1e89e1502ccfe1/CHANGELOG.rst", + "source": "https://github.com/marshmallow-code/marshmallow/issues/2936", + "source_id": "issue #2936" + }, + { + "after": null, + "before": null, + "behavior": "format checker is not enabled by default", + "decision": "exclude", + "id": "jsonschema-format-default", + "package": "jsonschema", + "reason": "Documented configuration/default behavior; not a library bug under the frozen rule.", + "source": "https://github.com/python-jsonschema/jsonschema/issues/945", + "source_id": "issue #945" + }, + { + "after": null, + "before": null, + "behavior": "email format checker is incomplete", + "decision": "exclude", + "id": "jsonschema-email-completeness", + "package": "jsonschema", + "reason": "Optional format-check completeness dispute; no documented before/after fix established in screening.", + "source": "https://github.com/python-jsonschema/jsonschema/issues/403", + "source_id": "issue #403" + }, + { + "after": null, + "before": null, + "behavior": "built-in generic forward references", + "decision": "exclude", + "id": "pydantic-forward-ref-generics", + "package": "pydantic", + "reason": "Model/type resolution rather than value validation; no fixed before/after release established in screening.", + "source": "https://github.com/pydantic/pydantic/issues/6130", + "source_id": "issue #6130" + }, + { + "after": null, + "before": null, + "behavior": "IPv4Address default JSON schema warning", + "decision": "exclude", + "id": "pydantic-ipv4-default", + "package": "pydantic", + "reason": "Serialization/schema warning, with no documented before/after validation fix.", + "source": "https://github.com/pydantic/pydantic/issues/8942", + "source_id": "issue #8942" + }, + { + "after": null, + "before": null, + "behavior": "default/missing parameter transition", + "decision": "exclude", + "id": "marshmallow-required-default", + "package": "marshmallow", + "reason": "Intentional API/deprecation transition and configuration semantics, not a validator defect.", + "source": "https://github.com/marshmallow-code/marshmallow/blob/dev/CHANGELOG.rst", + "source_id": "issue #1742" + }, + { + "after": null, + "before": null, + "behavior": "ModuleType serialization fails with serialize_as_any", + "decision": "exclude", + "id": "pydantic-serialize-any-regression", + "package": "pydantic", + "reason": "Serialization-only report; outside this study. No validation fix release was established.", + "source": "https://github.com/pydantic/pydantic/issues/12348", + "source_id": "issue #12348" + } + ], + "preflight_revision": 2 +} diff --git a/benchmarks/historical-validator-results/prepared-v2-bootstrap/manifest.json b/benchmarks/historical-validator-results/prepared-v2-bootstrap/manifest.json new file mode 100644 index 0000000..b39f60b --- /dev/null +++ b/benchmarks/historical-validator-results/prepared-v2-bootstrap/manifest.json @@ -0,0 +1,5 @@ +{ + "inputs.json": "b1dd2e43d625ca01e9ff588b59582931fed57f563ef4f178b8ec341be1941478", + "protocol.md": "21158fced5f031c5ea67a47dda2574b032eaf09bb37fd1bd45457899ecf97e09", + "runner.py.txt": "c194b1a04598c31ab6a0f43c40dfcc8b389ef8183f78c74686aa8ad633d3857e" +} diff --git a/benchmarks/historical-validator-results/prepared-v2-bootstrap/protocol.md b/benchmarks/historical-validator-results/prepared-v2-bootstrap/protocol.md new file mode 100644 index 0000000..39ffd46 --- /dev/null +++ b/benchmarks/historical-validator-results/prepared-v2-bootstrap/protocol.md @@ -0,0 +1,71 @@ +# Historical validator study protocol + +Status: version 2 prepared 2026-09-29 (America/Argentina/Buenos_Aires), before execution. + +This is a retrospective reproduction study, not an independent preregistration. +The unit is a documented validator defect with a public primary issue, pull +request, advisory, or release note naming the fix. A candidate is eligible only +when the upstream record describes an observable validation or validation-adjacent +behavior change and identifies both a before and after release (or commits). +Configuration omissions, documented defaults, feature requests, type-checking +only changes, and regressions without a later fix are excluded. + +Before any reproduction, the candidate ledger, minimal inputs, expected behavior, +adapter source, and this protocol are hashed. Reproductions use fresh virtual +environments below `work/`; no package is installed into the study checkout or +global interpreter. Network access is explicit (`collect --network`). The offline +replay consumes only the captured normalized verdicts and validates their hashes. + +Each included candidate has one valid baseline control and one invalid or newly +accepted variant. The control must remain accepted in both versions. The expected +finding is authored from the primary record and kept separate from adapter logic. +`audit_validator` outcomes are reported verbatim; errors, unsupported cases, and +failed baselines remain in the ledger and do not count as fixes. This protocol +does not tune library rules and makes no claim that the harness outperforms pytest. + +The frozen target is 8--12 candidates, with at least three reproduced fixes when +the available Python/runtime/package constraints permit. Fewer successful cases +are reported honestly. For every candidate we retain package/version/runtime, +source URL and ID, rationale, input digest, adapter hash, raw before/after output, +normalized audit report, and environment `pip freeze` hash. + +Primary sources are upstream GitHub issues, pull requests, release notes, or +security advisories. Minimal reproducers are authored for this study and do not +copy upstream test files. + +## Preflight corrections and selection limitations + +Version 1 is retained under `preflight-v1/` and in Git commit `5035606`. +No package reproduction was executed with that preparation. Before collection, +review of the original upstream reports found: uppercase file schemes and IDN +email addresses are valid variations, not faults; the file example needs explicit +`schemes={"file"}`; the boolean-ref issue requires a cached boolean schema in a +RefResolver store rather than a plain local reference; and the IDN case should +exercise a Unicode TLD. Those inputs/adapters and relations are corrected in v2. +Unsupported release guesses for excluded candidates are replaced with null. +The original serialization description for Pydantic #12348 was also corrected +after reading its actual ModuleType reproducer. These are pre-execution corrections, +not findings about the third-party libraries. Original flawed inputs are not evidence. + +Every included issue now links to its upstream report and a commit-pinned changelog +establishing the fixed version. The ten candidates were a purposive, bounded +screening sample, not an exhaustive or randomly sampled review of issues. +Exclusion means outside this experiment or insufficient documented evidence; it +does not mean an upstream report is invalid. No population discovery rate follows. + +Each included issue has two actual changed pairs: the issue-triggering variant +and a valid preservation control. The enum issue is a violation; the other three +are preservation issues, including a valid input triggering an exception. +Replay compares the full recorded normalized results against the same a3 audit; +exceptions remain errors rather than validation rejections. Baseline verdicts +and child failures are retained. The summary calls a fix reproduced only when +the before-version misbehaves on the trigger, the after-version satisfies it, +and the control is preserved in both versions. + +Freeze writes the prepared ledger and source/protocol hashes once. Setup never +rewrites it. Collection verifies prepared hashes, uses isolated version-specific +environments, and records exact installed dependency versions plus downloaded +artifact hashes from pip's installation report. Re-collection installs those +recorded dependency versions when supplied via `--lock-from`. Offline replay +does not reinstall packages and is distinguished from fresh package execution. +Hashes detect accidental changes; mutable manifests are not tamper-proof evidence. diff --git a/benchmarks/historical-validator-results/prepared-v2-bootstrap/runner.py.txt b/benchmarks/historical-validator-results/prepared-v2-bootstrap/runner.py.txt new file mode 100644 index 0000000..193cf88 --- /dev/null +++ b/benchmarks/historical-validator-results/prepared-v2-bootstrap/runner.py.txt @@ -0,0 +1,327 @@ +"""Version-pinned historical captures; isolated collection and verified offline replay.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import platform +import shutil +import subprocess +import sys +import venv +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "benchmarks/historical-validator-results" +PROTOCOL = ROOT / "paper/historical-validator-protocol.md" + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def write(path, value): + path.write_text(json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n") + + +def probe(sample): + """Only public payloads enter the third-party validator: no expected labels.""" + if sample["kind"] in ("enum", "refs"): + from jsonschema import Draft7Validator, RefResolver + + options = {} + if sample["kind"] == "refs": + options["resolver"] = RefResolver("", sample["schema"], store=sample["store"]) + errors = list(Draft7Validator(sample["schema"], **options).iter_errors(sample["value"])) + return { + "accepted": not errors, + "complete": True, + "violations": sorted({str(e.validator) + "@$" for e in errors}), + "messages": [e.message for e in errors], + "error": None, + } + from marshmallow import ValidationError, fields + + field = fields.Url(schemes={"file"}) if sample["kind"] == "file" else fields.Email() + try: + field.deserialize(sample["value"]) + except ValidationError as exc: + return { + "accepted": False, + "complete": True, + "violations": ["field@$"], + "messages": [str(exc)], + "error": None, + } + return {"accepted": True, "complete": True, "violations": [], "messages": [], "error": None} + + +def child(package): + # This process has only the pinned historical package and its dependencies. + samples = json.load(sys.stdin) + results = [] + for sample in samples: + try: + results.append(probe(sample)) + except Exception as exc: + results.append({"error": type(exc).__name__ + ": " + str(exc)}) + print( + json.dumps( + { + "python": platform.python_version(), + "package_version": importlib.metadata.version(package), + "results": results, + }, + sort_keys=True, + ) + ) + + +def freeze(): + target = OUT / "prepared" + target.mkdir(exist_ok=False) + shutil.copyfile(OUT / "frozen-inputs.json", target / "inputs.json") + shutil.copyfile(PROTOCOL, target / "protocol.md") + shutil.copyfile(__file__, target / "runner.py.txt") + write( + target / "manifest.json", + {name: sha(target / name) for name in ("inputs.json", "protocol.md", "runner.py.txt")}, + ) + + +def load_prepared(): + target = OUT / "prepared" + manifest = json.loads((target / "manifest.json").read_text()) + if set(manifest) != {"inputs.json", "protocol.md", "runner.py.txt"}: + raise ValueError("Unexpected prepared files") + for name, expected in manifest.items(): + if sha(target / name) != expected: + raise ValueError("Prepared content changed: " + name) + if sha(Path(__file__)) != manifest["runner.py.txt"] or sha(PROTOCOL) != manifest["protocol.md"]: + raise ValueError("Code or protocol differs from frozen source") + return json.loads((target / "inputs.json").read_text()) + + +def collect(work, output, lock_from=None): + data = load_prepared() + work.mkdir(parents=True, exist_ok=True) + output.mkdir(parents=True, exist_ok=False) + records = {} + for candidate in data["candidates"]: + if candidate["decision"] != "include": + continue + pairs = data["authored_inputs"][candidate["id"]] + samples = [pair[side] for pair in pairs for side in ("baseline", "variant")] + records[candidate["id"]] = {} + for version in (candidate["before"], candidate["after"]): + name = candidate["package"] + "-" + version + env = work / name + if env.exists(): + raise ValueError("Use a fresh environment directory: " + str(env)) + venv.EnvBuilder(with_pip=True).create(env) + python = str(env / "bin/python") + pip_report = env / "install.json" + install_args = [candidate["package"] + "==" + version] + if lock_from: + install_args = ["-r", str(lock_from / (name + ".lock.txt"))] + install = subprocess.run( + [ + python, + "-m", + "pip", + "--disable-pip-version-check", + "install", + "--report", + str(pip_report), + *install_args, + ], + capture_output=True, + text=True, + timeout=180, + ) + record = {"install_returncode": install.returncode} + if install.returncode: + record["setup_error"] = install.stderr + records[candidate["id"]][version] = record + continue + installed = json.loads(pip_report.read_text()) + record["artifacts"] = [ + { + "name": item["metadata"]["name"], + "version": item["metadata"]["version"], + "url": item["download_info"]["url"], + "hashes": item["download_info"]["archive_info"].get("hashes", {}), + } + for item in installed["install"] + ] + lock = subprocess.check_output([python, "-m", "pip", "freeze"], text=True) + (output / (name + ".lock.txt")).write_text(lock) + record["lock_sha256"] = sha(output / (name + ".lock.txt")) + proc = subprocess.run( + [ + python, + "-I", + str(Path(__file__).resolve()), + "child", + "--package", + candidate["package"], + ], + input=json.dumps(samples), + capture_output=True, + text=True, + timeout=30, + ) + record.update( + {"returncode": proc.returncode, "stdout": proc.stdout, "stderr": proc.stderr} + ) + if proc.returncode == 0: + record["payload"] = json.loads(proc.stdout) + records[candidate["id"]][version] = record + print(name, "captured", flush=True) + write(output / "captured.json", records) + write( + output / "manifest.json", + { + "prepared_manifest_sha256": sha(OUT / "prepared/manifest.json"), + "files": {p.name: sha(p) for p in sorted(output.iterdir()) if p.is_file()}, + }, + ) + + +def replay(captured, output): + from narrative_contracts.model import digest + from narrative_contracts.mutations import Relation, Validity + from narrative_contracts.validator_audit import AuditCase, Obligation, Verdict, audit_validator + + data = load_prepared() + manifest = json.loads((captured / "manifest.json").read_text()) + if manifest["prepared_manifest_sha256"] != sha(OUT / "prepared/manifest.json"): + raise ValueError("Capture does not match preparation") + if "captured.json" not in manifest["files"] or any( + Path(n).name != n for n in manifest["files"] + ): + raise ValueError("Invalid capture manifest") + for name, expected in manifest["files"].items(): + if sha(captured / name) != expected: + raise ValueError("Capture changed: " + name) + records = json.loads((captured / "captured.json").read_text()) + reports, results = {}, [] + included = [c for c in data["candidates"] if c["decision"] == "include"] + if set(records) != {c["id"] for c in included}: + raise ValueError("Missing or extra candidate capture") + for candidate in included: + pairs = data["authored_inputs"][candidate["id"]] + versions = records[candidate["id"]] + outcomes = {} + for version in (candidate["before"], candidate["after"]): + record = versions[version] + if record["install_returncode"] or record["returncode"]: + outcomes[version] = {"execution": "unavailable"} + continue + payload = record["payload"] + if payload != json.loads(record["stdout"]) or payload["package_version"] != version: + raise ValueError("Captured version or raw output mismatch") + if len(payload["results"]) != 2 * len(pairs): + raise ValueError("Incomplete paired capture") + lookup = {} + for pair_index, pair in enumerate(pairs): + for side_index, side in enumerate(("baseline", "variant")): + key = digest(pair[side]) + observed = payload["results"][pair_index * 2 + side_index] + if key in lookup and lookup[key] != observed: + raise ValueError("Inconsistent repeated baseline capture") + lookup[key] = observed + + def adapter(sample): + record = lookup[digest(sample)] + if record["error"]: + raise RuntimeError(record["error"]) + return Verdict( + record["accepted"], + tuple(record["violations"]), + record["complete"], + tuple(record["messages"]), + ) + + cases = tuple( + AuditCase( + p["id"], + candidate["id"], + p["baseline"], + p["variant"], + Relation(p["relation"]), + tuple(p["expected"]), + Validity.VALID, + "Authored minimal reproduction of " + candidate["source"], + ) + for p in pairs + ) + report = audit_validator( + adapter, + cases, + obligations=(Obligation(candidate["id"], candidate["behavior"]),), + validator_id=f"{candidate['package']}/{version}", + ) + reports[candidate["id"] + "/" + version] = report.to_dict() + outcomes[version] = {c.id: c.outcome for c in report.cases} + trigger, control = pairs + before, after = outcomes[candidate["before"]], outcomes[candidate["after"]] + desired = "detected" if trigger["relation"] == "violation" else "preserved" + reproduced = ( + before.get(trigger["id"]) in ("survived", "regressed", "error") + and after.get(trigger["id"]) == desired + and before.get(control["id"]) == after.get(control["id"]) == "preserved" + ) + results.append( + { + "id": candidate["id"], + "source": candidate["source"], + "before_version": candidate["before"], + "after_version": candidate["after"], + "outcomes": outcomes, + "fix_reproduced": reproduced, + } + ) + summary = { + "schema_version": 1, + "screened": len(data["candidates"]), + "included": len(included), + "excluded": len(data["candidates"]) - len(included), + "fixes_reproduced": sum(r["fix_reproduced"] for r in results), + "results": results, + } + output.mkdir(parents=True, exist_ok=False) + write(output / "summary.json", summary) + write(output / "audit-reports.json", reports) + print(json.dumps(summary, indent=2)) + return summary + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("freeze", "collect", "replay", "child")) + parser.add_argument("--network", action="store_true") + parser.add_argument("--work", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument("--captured", type=Path) + parser.add_argument("--lock-from", type=Path) + parser.add_argument("--package") + args = parser.parse_args() + if args.action == "child": + child(args.package) + elif args.action == "freeze": + freeze() + elif args.action == "collect": + if not args.network or args.work is None or args.output is None: + parser.error("collect requires --network, --work and --output") + collect(args.work, args.output, args.lock_from) + else: + if args.captured is None or args.output is None: + parser.error("replay requires --captured and --output") + replay(args.captured, args.output) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/historical-validator-results/prepared/inputs.json b/benchmarks/historical-validator-results/prepared/inputs.json new file mode 100644 index 0000000..94ec7f1 --- /dev/null +++ b/benchmarks/historical-validator-results/prepared/inputs.json @@ -0,0 +1,292 @@ +{ + "authored_inputs": { + "jsonschema-bool-ref-cache": [ + { + "baseline": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:ok": { + "type": "object" + } + }, + "value": {} + }, + "expected": [], + "id": "cached-boolean-valid", + "relation": "preserve", + "variant": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:any": true, + "urn:study:ok": { + "type": "object" + } + }, + "value": {} + } + }, + { + "baseline": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:ok": { + "type": "object" + } + }, + "value": {} + }, + "expected": [], + "id": "cached-object-control", + "relation": "preserve", + "variant": { + "kind": "refs", + "schema": { + "$ref": "urn:study:ok" + }, + "store": { + "urn:study:ok": { + "type": "object" + }, + "urn:study:other": { + "type": "array" + } + }, + "value": {} + } + } + ], + "jsonschema-enum-bool-int": [ + { + "baseline": { + "kind": "enum", + "schema": { + "enum": [ + false + ] + }, + "value": false + }, + "expected": [ + "enum@$" + ], + "id": "enum-int-fault", + "relation": "violation", + "variant": { + "kind": "enum", + "schema": { + "enum": [ + false + ] + }, + "value": 0 + } + }, + { + "baseline": { + "kind": "enum", + "schema": { + "enum": [ + false + ] + }, + "value": false + }, + "expected": [], + "id": "enum-valid-control", + "relation": "preserve", + "variant": { + "kind": "enum", + "schema": { + "enum": [ + false, + true + ] + }, + "value": false + } + } + ], + "marshmallow-file-url": [ + { + "baseline": { + "kind": "file", + "value": "file:///tmp/a" + }, + "expected": [], + "id": "uppercase-scheme-valid", + "relation": "preserve", + "variant": { + "kind": "file", + "value": "FILE:///tmp/a" + } + }, + { + "baseline": { + "kind": "file", + "value": "file:///tmp/a" + }, + "expected": [], + "id": "lowercase-control", + "relation": "preserve", + "variant": { + "kind": "file", + "value": "file:///tmp/b" + } + } + ], + "marshmallow-idn-email": [ + { + "baseline": { + "kind": "email", + "value": "user@example.com" + }, + "expected": [], + "id": "unicode-tld-valid", + "relation": "preserve", + "variant": { + "kind": "email", + "value": "user@пример.рф" + } + }, + { + "baseline": { + "kind": "email", + "value": "user@example.com" + }, + "expected": [], + "id": "ascii-control", + "relation": "preserve", + "variant": { + "kind": "email", + "value": "user2@example.com" + } + } + ] + }, + "candidates": [ + { + "after": "3.0.2", + "before": "3.0.1", + "behavior": "JSON Schema enum/const distinguish 0 from false", + "decision": "include", + "id": "jsonschema-enum-bool-int", + "package": "jsonschema", + "reason": "Release notes explicitly call this a bug fix and identify before/after releases.", + "release_source": "https://github.com/python-jsonschema/jsonschema/blob/51cd75e399c760e5aa3adce600dcce1385756ab0/CHANGELOG.rst", + "source": "https://github.com/python-jsonschema/jsonschema/issues/575", + "source_id": "issue #575" + }, + { + "after": "4.17.3", + "before": "4.17.2", + "behavior": "Validator instantiation accepts cached boolean schemas", + "decision": "include", + "id": "jsonschema-bool-ref-cache", + "package": "jsonschema", + "reason": "Release notes explicitly call this a bug fix with a fixed release.", + "release_source": "https://github.com/python-jsonschema/jsonschema/blob/51cd75e399c760e5aa3adce600dcce1385756ab0/CHANGELOG.rst", + "source": "https://github.com/python-jsonschema/jsonschema/issues/1018", + "source_id": "issue #1018" + }, + { + "after": "4.2.1", + "before": "4.2.0", + "behavior": "URL validator handles uppercase FILE scheme", + "decision": "include", + "id": "marshmallow-file-url", + "package": "marshmallow", + "reason": "Release notes identify a validation fix and adjacent releases.", + "release_source": "https://github.com/marshmallow-code/marshmallow/blob/7f0792bd7a06f72e393ec866ac1e89e1502ccfe1/CHANGELOG.rst", + "source": "https://github.com/marshmallow-code/marshmallow/issues/2891", + "source_id": "issue #2891" + }, + { + "after": "4.2.4", + "before": "4.2.3", + "behavior": "Email validator accepts valid internationalized domain names", + "decision": "include", + "id": "marshmallow-idn-email", + "package": "marshmallow", + "reason": "Release notes identify a validator behavior fix and adjacent releases.", + "release_source": "https://github.com/marshmallow-code/marshmallow/blob/7f0792bd7a06f72e393ec866ac1e89e1502ccfe1/CHANGELOG.rst", + "source": "https://github.com/marshmallow-code/marshmallow/issues/2936", + "source_id": "issue #2936" + }, + { + "after": null, + "before": null, + "behavior": "format checker is not enabled by default", + "decision": "exclude", + "id": "jsonschema-format-default", + "package": "jsonschema", + "reason": "Documented configuration/default behavior; not a library bug under the frozen rule.", + "source": "https://github.com/python-jsonschema/jsonschema/issues/945", + "source_id": "issue #945" + }, + { + "after": null, + "before": null, + "behavior": "email format checker is incomplete", + "decision": "exclude", + "id": "jsonschema-email-completeness", + "package": "jsonschema", + "reason": "Optional format-check completeness dispute; no documented before/after fix established in screening.", + "source": "https://github.com/python-jsonschema/jsonschema/issues/403", + "source_id": "issue #403" + }, + { + "after": null, + "before": null, + "behavior": "built-in generic forward references", + "decision": "exclude", + "id": "pydantic-forward-ref-generics", + "package": "pydantic", + "reason": "Model/type resolution rather than value validation; no fixed before/after release established in screening.", + "source": "https://github.com/pydantic/pydantic/issues/6130", + "source_id": "issue #6130" + }, + { + "after": null, + "before": null, + "behavior": "IPv4Address default JSON schema warning", + "decision": "exclude", + "id": "pydantic-ipv4-default", + "package": "pydantic", + "reason": "Serialization/schema warning, with no documented before/after validation fix.", + "source": "https://github.com/pydantic/pydantic/issues/8942", + "source_id": "issue #8942" + }, + { + "after": null, + "before": null, + "behavior": "default/missing parameter transition", + "decision": "exclude", + "id": "marshmallow-required-default", + "package": "marshmallow", + "reason": "Intentional API/deprecation transition and configuration semantics, not a validator defect.", + "source": "https://github.com/marshmallow-code/marshmallow/blob/dev/CHANGELOG.rst", + "source_id": "issue #1742" + }, + { + "after": null, + "before": null, + "behavior": "ModuleType serialization fails with serialize_as_any", + "decision": "exclude", + "id": "pydantic-serialize-any-regression", + "package": "pydantic", + "reason": "Serialization-only report; outside this study. No validation fix release was established.", + "source": "https://github.com/pydantic/pydantic/issues/12348", + "source_id": "issue #12348" + } + ], + "preflight_revision": 2 +} diff --git a/benchmarks/historical-validator-results/prepared/manifest.json b/benchmarks/historical-validator-results/prepared/manifest.json new file mode 100644 index 0000000..5460753 --- /dev/null +++ b/benchmarks/historical-validator-results/prepared/manifest.json @@ -0,0 +1,5 @@ +{ + "inputs.json": "b1dd2e43d625ca01e9ff588b59582931fed57f563ef4f178b8ec341be1941478", + "protocol.md": "e24aa2a4b34d9184e615f79939d99f403f356ba90332b70fc767722b22b43df3", + "runner.py.txt": "cab4f5433556cf3c4448084ed716da2daa80eefdab1125d47fec264e83ca71df" +} diff --git a/benchmarks/historical-validator-results/prepared/protocol.md b/benchmarks/historical-validator-results/prepared/protocol.md new file mode 100644 index 0000000..3bb1a4a --- /dev/null +++ b/benchmarks/historical-validator-results/prepared/protocol.md @@ -0,0 +1,80 @@ +# Historical validator study protocol + +Status: version 2 prepared 2026-09-29 (America/Argentina/Buenos_Aires), before execution. + +This is a retrospective reproduction study, not an independent preregistration. +The unit is a documented validator defect with a public primary issue, pull +request, advisory, or release note naming the fix. A candidate is eligible only +when the upstream record describes an observable validation or validation-adjacent +behavior change and identifies both a before and after release (or commits). +Configuration omissions, documented defaults, feature requests, type-checking +only changes, and regressions without a later fix are excluded. + +Before any reproduction, the candidate ledger, minimal inputs, expected behavior, +adapter source, and this protocol are hashed. Reproductions use fresh virtual +environments below `work/`; no package is installed into the study checkout or +global interpreter. Network access is explicit (`collect --network`). The offline +replay consumes only the captured normalized verdicts and validates their hashes. + +Each included candidate has one valid baseline control and one invalid or newly +accepted variant. The control must remain accepted in both versions. The expected +finding is authored from the primary record and kept separate from adapter logic. +`audit_validator` outcomes are reported verbatim; errors, unsupported cases, and +failed baselines remain in the ledger and do not count as fixes. This protocol +does not tune library rules and makes no claim that the harness outperforms pytest. + +The frozen target is 8--12 candidates, with at least three reproduced fixes when +the available Python/runtime/package constraints permit. Fewer successful cases +are reported honestly. For every candidate we retain package/version/runtime, +source URL and ID, rationale, input digest, adapter hash, raw before/after output, +normalized audit report, and environment `pip freeze` hash. + +Primary sources are upstream GitHub issues, pull requests, release notes, or +security advisories. Minimal reproducers are authored for this study and do not +copy upstream test files. + +## Preflight corrections and selection limitations + +Version 1 is retained under `preflight-v1/` and in Git commit `5035606`. +No package reproduction was executed with that preparation. Before collection, +review of the original upstream reports found: uppercase file schemes and IDN +email addresses are valid variations, not faults; the file example needs explicit +`schemes={"file"}`; the boolean-ref issue requires a cached boolean schema in a +RefResolver store rather than a plain local reference; and the IDN case should +exercise a Unicode TLD. Those inputs/adapters and relations are corrected in v2. +Unsupported release guesses for excluded candidates are replaced with null. +The original serialization description for Pydantic #12348 was also corrected +after reading its actual ModuleType reproducer. These are pre-execution corrections, +not findings about the third-party libraries. Original flawed inputs are not evidence. + +Every included issue now links to its upstream report and a commit-pinned changelog +establishing the fixed version. The ten candidates were a purposive, bounded +screening sample, not an exhaustive or randomly sampled review of issues. +Exclusion means outside this experiment or insufficient documented evidence; it +does not mean an upstream report is invalid. No population discovery rate follows. + +Each included issue has two actual changed pairs: the issue-triggering variant +and a valid preservation control. The enum issue is a violation; the other three +are preservation issues, including a valid input triggering an exception. +Replay compares the full recorded normalized results against the same a3 audit; +exceptions remain errors rather than validation rejections. Baseline verdicts +and child failures are retained. The summary calls a fix reproduced only when +the before-version misbehaves on the trigger, the after-version satisfies it, +and the control is preserved in both versions. + +Freeze writes the prepared ledger and source/protocol hashes once. Setup never +rewrites it. Collection verifies prepared hashes, uses isolated version-specific +environments, and records exact installed dependency versions plus downloaded +artifact hashes from pip's installation report. Re-collection installs those +recorded dependency versions when supplied via `--lock-from`. Offline replay +does not reinstall packages and is distinguished from fresh package execution. +Hashes detect accidental changes; mutable manifests are not tamper-proof evidence. + +## Bootstrap-only execution deviation + +Two initial collection attempts stopped while creating the first virtual environment, +before installing a historical package or executing any sample. This macOS Python +distribution aborts when its executable is copied by EnvBuilder; the collector now +uses symlinks, matching the working command-line venv behavior. The source/protocol +were re-frozen before sample execution. The preceding prepared snapshot is retained +as `prepared-v2-bootstrap/`; case selection, labels and adapter logic are unchanged. diff --git a/benchmarks/historical-validator-results/prepared/runner.py.txt b/benchmarks/historical-validator-results/prepared/runner.py.txt new file mode 100644 index 0000000..8017b3a --- /dev/null +++ b/benchmarks/historical-validator-results/prepared/runner.py.txt @@ -0,0 +1,327 @@ +"""Version-pinned historical captures; isolated collection and verified offline replay.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import platform +import shutil +import subprocess +import sys +import venv +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "benchmarks/historical-validator-results" +PROTOCOL = ROOT / "paper/historical-validator-protocol.md" + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def write(path, value): + path.write_text(json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n") + + +def probe(sample): + """Only public payloads enter the third-party validator: no expected labels.""" + if sample["kind"] in ("enum", "refs"): + from jsonschema import Draft7Validator, RefResolver + + options = {} + if sample["kind"] == "refs": + options["resolver"] = RefResolver("", sample["schema"], store=sample["store"]) + errors = list(Draft7Validator(sample["schema"], **options).iter_errors(sample["value"])) + return { + "accepted": not errors, + "complete": True, + "violations": sorted({str(e.validator) + "@$" for e in errors}), + "messages": [e.message for e in errors], + "error": None, + } + from marshmallow import ValidationError, fields + + field = fields.Url(schemes={"file"}) if sample["kind"] == "file" else fields.Email() + try: + field.deserialize(sample["value"]) + except ValidationError as exc: + return { + "accepted": False, + "complete": True, + "violations": ["field@$"], + "messages": [str(exc)], + "error": None, + } + return {"accepted": True, "complete": True, "violations": [], "messages": [], "error": None} + + +def child(package): + # This process has only the pinned historical package and its dependencies. + samples = json.load(sys.stdin) + results = [] + for sample in samples: + try: + results.append(probe(sample)) + except Exception as exc: + results.append({"error": type(exc).__name__ + ": " + str(exc)}) + print( + json.dumps( + { + "python": platform.python_version(), + "package_version": importlib.metadata.version(package), + "results": results, + }, + sort_keys=True, + ) + ) + + +def freeze(): + target = OUT / "prepared" + target.mkdir(exist_ok=False) + shutil.copyfile(OUT / "frozen-inputs.json", target / "inputs.json") + shutil.copyfile(PROTOCOL, target / "protocol.md") + shutil.copyfile(__file__, target / "runner.py.txt") + write( + target / "manifest.json", + {name: sha(target / name) for name in ("inputs.json", "protocol.md", "runner.py.txt")}, + ) + + +def load_prepared(): + target = OUT / "prepared" + manifest = json.loads((target / "manifest.json").read_text()) + if set(manifest) != {"inputs.json", "protocol.md", "runner.py.txt"}: + raise ValueError("Unexpected prepared files") + for name, expected in manifest.items(): + if sha(target / name) != expected: + raise ValueError("Prepared content changed: " + name) + if sha(Path(__file__)) != manifest["runner.py.txt"] or sha(PROTOCOL) != manifest["protocol.md"]: + raise ValueError("Code or protocol differs from frozen source") + return json.loads((target / "inputs.json").read_text()) + + +def collect(work, output, lock_from=None): + data = load_prepared() + work.mkdir(parents=True, exist_ok=True) + output.mkdir(parents=True, exist_ok=False) + records = {} + for candidate in data["candidates"]: + if candidate["decision"] != "include": + continue + pairs = data["authored_inputs"][candidate["id"]] + samples = [pair[side] for pair in pairs for side in ("baseline", "variant")] + records[candidate["id"]] = {} + for version in (candidate["before"], candidate["after"]): + name = candidate["package"] + "-" + version + env = work / name + if env.exists(): + raise ValueError("Use a fresh environment directory: " + str(env)) + venv.EnvBuilder(with_pip=True, symlinks=True).create(env) + python = str(env / "bin/python") + pip_report = env / "install.json" + install_args = [candidate["package"] + "==" + version] + if lock_from: + install_args = ["-r", str(lock_from / (name + ".lock.txt"))] + install = subprocess.run( + [ + python, + "-m", + "pip", + "--disable-pip-version-check", + "install", + "--report", + str(pip_report), + *install_args, + ], + capture_output=True, + text=True, + timeout=180, + ) + record = {"install_returncode": install.returncode} + if install.returncode: + record["setup_error"] = install.stderr + records[candidate["id"]][version] = record + continue + installed = json.loads(pip_report.read_text()) + record["artifacts"] = [ + { + "name": item["metadata"]["name"], + "version": item["metadata"]["version"], + "url": item["download_info"]["url"], + "hashes": item["download_info"]["archive_info"].get("hashes", {}), + } + for item in installed["install"] + ] + lock = subprocess.check_output([python, "-m", "pip", "freeze"], text=True) + (output / (name + ".lock.txt")).write_text(lock) + record["lock_sha256"] = sha(output / (name + ".lock.txt")) + proc = subprocess.run( + [ + python, + "-I", + str(Path(__file__).resolve()), + "child", + "--package", + candidate["package"], + ], + input=json.dumps(samples), + capture_output=True, + text=True, + timeout=30, + ) + record.update( + {"returncode": proc.returncode, "stdout": proc.stdout, "stderr": proc.stderr} + ) + if proc.returncode == 0: + record["payload"] = json.loads(proc.stdout) + records[candidate["id"]][version] = record + print(name, "captured", flush=True) + write(output / "captured.json", records) + write( + output / "manifest.json", + { + "prepared_manifest_sha256": sha(OUT / "prepared/manifest.json"), + "files": {p.name: sha(p) for p in sorted(output.iterdir()) if p.is_file()}, + }, + ) + + +def replay(captured, output): + from narrative_contracts.model import digest + from narrative_contracts.mutations import Relation, Validity + from narrative_contracts.validator_audit import AuditCase, Obligation, Verdict, audit_validator + + data = load_prepared() + manifest = json.loads((captured / "manifest.json").read_text()) + if manifest["prepared_manifest_sha256"] != sha(OUT / "prepared/manifest.json"): + raise ValueError("Capture does not match preparation") + if "captured.json" not in manifest["files"] or any( + Path(n).name != n for n in manifest["files"] + ): + raise ValueError("Invalid capture manifest") + for name, expected in manifest["files"].items(): + if sha(captured / name) != expected: + raise ValueError("Capture changed: " + name) + records = json.loads((captured / "captured.json").read_text()) + reports, results = {}, [] + included = [c for c in data["candidates"] if c["decision"] == "include"] + if set(records) != {c["id"] for c in included}: + raise ValueError("Missing or extra candidate capture") + for candidate in included: + pairs = data["authored_inputs"][candidate["id"]] + versions = records[candidate["id"]] + outcomes = {} + for version in (candidate["before"], candidate["after"]): + record = versions[version] + if record["install_returncode"] or record["returncode"]: + outcomes[version] = {"execution": "unavailable"} + continue + payload = record["payload"] + if payload != json.loads(record["stdout"]) or payload["package_version"] != version: + raise ValueError("Captured version or raw output mismatch") + if len(payload["results"]) != 2 * len(pairs): + raise ValueError("Incomplete paired capture") + lookup = {} + for pair_index, pair in enumerate(pairs): + for side_index, side in enumerate(("baseline", "variant")): + key = digest(pair[side]) + observed = payload["results"][pair_index * 2 + side_index] + if key in lookup and lookup[key] != observed: + raise ValueError("Inconsistent repeated baseline capture") + lookup[key] = observed + + def adapter(sample): + record = lookup[digest(sample)] + if record["error"]: + raise RuntimeError(record["error"]) + return Verdict( + record["accepted"], + tuple(record["violations"]), + record["complete"], + tuple(record["messages"]), + ) + + cases = tuple( + AuditCase( + p["id"], + candidate["id"], + p["baseline"], + p["variant"], + Relation(p["relation"]), + tuple(p["expected"]), + Validity.VALID, + "Authored minimal reproduction of " + candidate["source"], + ) + for p in pairs + ) + report = audit_validator( + adapter, + cases, + obligations=(Obligation(candidate["id"], candidate["behavior"]),), + validator_id=f"{candidate['package']}/{version}", + ) + reports[candidate["id"] + "/" + version] = report.to_dict() + outcomes[version] = {c.id: c.outcome for c in report.cases} + trigger, control = pairs + before, after = outcomes[candidate["before"]], outcomes[candidate["after"]] + desired = "detected" if trigger["relation"] == "violation" else "preserved" + reproduced = ( + before.get(trigger["id"]) in ("survived", "regressed", "error") + and after.get(trigger["id"]) == desired + and before.get(control["id"]) == after.get(control["id"]) == "preserved" + ) + results.append( + { + "id": candidate["id"], + "source": candidate["source"], + "before_version": candidate["before"], + "after_version": candidate["after"], + "outcomes": outcomes, + "fix_reproduced": reproduced, + } + ) + summary = { + "schema_version": 1, + "screened": len(data["candidates"]), + "included": len(included), + "excluded": len(data["candidates"]) - len(included), + "fixes_reproduced": sum(r["fix_reproduced"] for r in results), + "results": results, + } + output.mkdir(parents=True, exist_ok=False) + write(output / "summary.json", summary) + write(output / "audit-reports.json", reports) + print(json.dumps(summary, indent=2)) + return summary + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("freeze", "collect", "replay", "child")) + parser.add_argument("--network", action="store_true") + parser.add_argument("--work", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument("--captured", type=Path) + parser.add_argument("--lock-from", type=Path) + parser.add_argument("--package") + args = parser.parse_args() + if args.action == "child": + child(args.package) + elif args.action == "freeze": + freeze() + elif args.action == "collect": + if not args.network or args.work is None or args.output is None: + parser.error("collect requires --network, --work and --output") + collect(args.work, args.output, args.lock_from) + else: + if args.captured is None or args.output is None: + parser.error("replay requires --captured and --output") + replay(args.captured, args.output) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/historical_followup.py b/benchmarks/historical_followup.py new file mode 100644 index 0000000..a63503a --- /dev/null +++ b/benchmarks/historical_followup.py @@ -0,0 +1,102 @@ +"""Explicit compatibility follow-up; preserves the original historical captures.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BASE = ROOT / "benchmarks/historical-validator-results" +OUT = BASE / "compatibility-followup-v2" +spec = importlib.util.spec_from_file_location( + "historical_base", ROOT / "benchmarks/historical_validator_study.py" +) +base = importlib.util.module_from_spec(spec) +spec.loader.exec_module(base) +base.OUT = OUT +base.PROTOCOL = ROOT / "paper/historical-followup-protocol.md" + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def prepare(): + OUT.mkdir(exist_ok=False) + requirements = OUT / "requirements" + requirements.mkdir() + data = json.loads((BASE / "prepared/inputs.json").read_text()) + for c in data["candidates"]: + if c["id"] == "jsonschema-bool-ref-cache": + c["before"] = "4.17.1" + c["reason"] += ( + " Follow-up uses available 4.17.1; initial 4.17.2 could not be installed." + ) + if c["decision"] != "include": + continue + for version in (c["before"], c["after"]): + name = c["package"] + "-" + version + ".lock.txt" + original = BASE / "captured" / name + if version == "4.17.1": + content = ( + (BASE / "captured/jsonschema-4.17.3.lock.txt") + .read_text() + .replace("jsonschema==4.17.3", "jsonschema==4.17.1") + ) + else: + content = original.read_text() + if c["package"] == "jsonschema" and version.startswith("3."): + content = ( + "\n".join( + line for line in content.splitlines() if not line.startswith("setuptools==") + ) + + "\nsetuptools==70.3.0\n" + ) + (requirements / name).write_text(content) + data["followup_driver_sha256"] = sha(Path(__file__)) + data["requirements_sha256"] = {p.name: sha(p) for p in requirements.iterdir()} + data["parent_capture_manifest_sha256"] = sha(BASE / "captured/manifest.json") + (OUT / "frozen-inputs.json").write_text( + json.dumps(data, indent=2, sort_keys=True, ensure_ascii=False) + "\n" + ) + base.freeze() + + +def verify(): + data = base.load_prepared() + if sha(Path(__file__)) != data["followup_driver_sha256"]: + raise ValueError("Follow-up driver changed") + for name, expected in data["requirements_sha256"].items(): + if Path(name).name != name or sha(OUT / "requirements" / name) != expected: + raise ValueError("Follow-up requirements changed") + if sha(BASE / "captured/manifest.json") != data["parent_capture_manifest_sha256"]: + raise ValueError("Parent capture identity changed") + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("action", choices=("prepare", "collect", "replay")) + p.add_argument("--network", action="store_true") + p.add_argument("--work", type=Path) + p.add_argument("--output", type=Path) + p.add_argument("--captured", type=Path) + args = p.parse_args() + if args.action == "prepare": + prepare() + return + verify() + if args.action == "collect": + if not args.network or args.work is None or args.output is None: + p.error("collect requires --network, --work and --output") + base.collect(args.work, args.output, OUT / "requirements") + else: + if args.captured is None or args.output is None: + p.error("replay requires --captured and --output") + base.replay(args.captured, args.output) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/historical_validator_study.py b/benchmarks/historical_validator_study.py new file mode 100644 index 0000000..8017b3a --- /dev/null +++ b/benchmarks/historical_validator_study.py @@ -0,0 +1,327 @@ +"""Version-pinned historical captures; isolated collection and verified offline replay.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import platform +import shutil +import subprocess +import sys +import venv +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "benchmarks/historical-validator-results" +PROTOCOL = ROOT / "paper/historical-validator-protocol.md" + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def write(path, value): + path.write_text(json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n") + + +def probe(sample): + """Only public payloads enter the third-party validator: no expected labels.""" + if sample["kind"] in ("enum", "refs"): + from jsonschema import Draft7Validator, RefResolver + + options = {} + if sample["kind"] == "refs": + options["resolver"] = RefResolver("", sample["schema"], store=sample["store"]) + errors = list(Draft7Validator(sample["schema"], **options).iter_errors(sample["value"])) + return { + "accepted": not errors, + "complete": True, + "violations": sorted({str(e.validator) + "@$" for e in errors}), + "messages": [e.message for e in errors], + "error": None, + } + from marshmallow import ValidationError, fields + + field = fields.Url(schemes={"file"}) if sample["kind"] == "file" else fields.Email() + try: + field.deserialize(sample["value"]) + except ValidationError as exc: + return { + "accepted": False, + "complete": True, + "violations": ["field@$"], + "messages": [str(exc)], + "error": None, + } + return {"accepted": True, "complete": True, "violations": [], "messages": [], "error": None} + + +def child(package): + # This process has only the pinned historical package and its dependencies. + samples = json.load(sys.stdin) + results = [] + for sample in samples: + try: + results.append(probe(sample)) + except Exception as exc: + results.append({"error": type(exc).__name__ + ": " + str(exc)}) + print( + json.dumps( + { + "python": platform.python_version(), + "package_version": importlib.metadata.version(package), + "results": results, + }, + sort_keys=True, + ) + ) + + +def freeze(): + target = OUT / "prepared" + target.mkdir(exist_ok=False) + shutil.copyfile(OUT / "frozen-inputs.json", target / "inputs.json") + shutil.copyfile(PROTOCOL, target / "protocol.md") + shutil.copyfile(__file__, target / "runner.py.txt") + write( + target / "manifest.json", + {name: sha(target / name) for name in ("inputs.json", "protocol.md", "runner.py.txt")}, + ) + + +def load_prepared(): + target = OUT / "prepared" + manifest = json.loads((target / "manifest.json").read_text()) + if set(manifest) != {"inputs.json", "protocol.md", "runner.py.txt"}: + raise ValueError("Unexpected prepared files") + for name, expected in manifest.items(): + if sha(target / name) != expected: + raise ValueError("Prepared content changed: " + name) + if sha(Path(__file__)) != manifest["runner.py.txt"] or sha(PROTOCOL) != manifest["protocol.md"]: + raise ValueError("Code or protocol differs from frozen source") + return json.loads((target / "inputs.json").read_text()) + + +def collect(work, output, lock_from=None): + data = load_prepared() + work.mkdir(parents=True, exist_ok=True) + output.mkdir(parents=True, exist_ok=False) + records = {} + for candidate in data["candidates"]: + if candidate["decision"] != "include": + continue + pairs = data["authored_inputs"][candidate["id"]] + samples = [pair[side] for pair in pairs for side in ("baseline", "variant")] + records[candidate["id"]] = {} + for version in (candidate["before"], candidate["after"]): + name = candidate["package"] + "-" + version + env = work / name + if env.exists(): + raise ValueError("Use a fresh environment directory: " + str(env)) + venv.EnvBuilder(with_pip=True, symlinks=True).create(env) + python = str(env / "bin/python") + pip_report = env / "install.json" + install_args = [candidate["package"] + "==" + version] + if lock_from: + install_args = ["-r", str(lock_from / (name + ".lock.txt"))] + install = subprocess.run( + [ + python, + "-m", + "pip", + "--disable-pip-version-check", + "install", + "--report", + str(pip_report), + *install_args, + ], + capture_output=True, + text=True, + timeout=180, + ) + record = {"install_returncode": install.returncode} + if install.returncode: + record["setup_error"] = install.stderr + records[candidate["id"]][version] = record + continue + installed = json.loads(pip_report.read_text()) + record["artifacts"] = [ + { + "name": item["metadata"]["name"], + "version": item["metadata"]["version"], + "url": item["download_info"]["url"], + "hashes": item["download_info"]["archive_info"].get("hashes", {}), + } + for item in installed["install"] + ] + lock = subprocess.check_output([python, "-m", "pip", "freeze"], text=True) + (output / (name + ".lock.txt")).write_text(lock) + record["lock_sha256"] = sha(output / (name + ".lock.txt")) + proc = subprocess.run( + [ + python, + "-I", + str(Path(__file__).resolve()), + "child", + "--package", + candidate["package"], + ], + input=json.dumps(samples), + capture_output=True, + text=True, + timeout=30, + ) + record.update( + {"returncode": proc.returncode, "stdout": proc.stdout, "stderr": proc.stderr} + ) + if proc.returncode == 0: + record["payload"] = json.loads(proc.stdout) + records[candidate["id"]][version] = record + print(name, "captured", flush=True) + write(output / "captured.json", records) + write( + output / "manifest.json", + { + "prepared_manifest_sha256": sha(OUT / "prepared/manifest.json"), + "files": {p.name: sha(p) for p in sorted(output.iterdir()) if p.is_file()}, + }, + ) + + +def replay(captured, output): + from narrative_contracts.model import digest + from narrative_contracts.mutations import Relation, Validity + from narrative_contracts.validator_audit import AuditCase, Obligation, Verdict, audit_validator + + data = load_prepared() + manifest = json.loads((captured / "manifest.json").read_text()) + if manifest["prepared_manifest_sha256"] != sha(OUT / "prepared/manifest.json"): + raise ValueError("Capture does not match preparation") + if "captured.json" not in manifest["files"] or any( + Path(n).name != n for n in manifest["files"] + ): + raise ValueError("Invalid capture manifest") + for name, expected in manifest["files"].items(): + if sha(captured / name) != expected: + raise ValueError("Capture changed: " + name) + records = json.loads((captured / "captured.json").read_text()) + reports, results = {}, [] + included = [c for c in data["candidates"] if c["decision"] == "include"] + if set(records) != {c["id"] for c in included}: + raise ValueError("Missing or extra candidate capture") + for candidate in included: + pairs = data["authored_inputs"][candidate["id"]] + versions = records[candidate["id"]] + outcomes = {} + for version in (candidate["before"], candidate["after"]): + record = versions[version] + if record["install_returncode"] or record["returncode"]: + outcomes[version] = {"execution": "unavailable"} + continue + payload = record["payload"] + if payload != json.loads(record["stdout"]) or payload["package_version"] != version: + raise ValueError("Captured version or raw output mismatch") + if len(payload["results"]) != 2 * len(pairs): + raise ValueError("Incomplete paired capture") + lookup = {} + for pair_index, pair in enumerate(pairs): + for side_index, side in enumerate(("baseline", "variant")): + key = digest(pair[side]) + observed = payload["results"][pair_index * 2 + side_index] + if key in lookup and lookup[key] != observed: + raise ValueError("Inconsistent repeated baseline capture") + lookup[key] = observed + + def adapter(sample): + record = lookup[digest(sample)] + if record["error"]: + raise RuntimeError(record["error"]) + return Verdict( + record["accepted"], + tuple(record["violations"]), + record["complete"], + tuple(record["messages"]), + ) + + cases = tuple( + AuditCase( + p["id"], + candidate["id"], + p["baseline"], + p["variant"], + Relation(p["relation"]), + tuple(p["expected"]), + Validity.VALID, + "Authored minimal reproduction of " + candidate["source"], + ) + for p in pairs + ) + report = audit_validator( + adapter, + cases, + obligations=(Obligation(candidate["id"], candidate["behavior"]),), + validator_id=f"{candidate['package']}/{version}", + ) + reports[candidate["id"] + "/" + version] = report.to_dict() + outcomes[version] = {c.id: c.outcome for c in report.cases} + trigger, control = pairs + before, after = outcomes[candidate["before"]], outcomes[candidate["after"]] + desired = "detected" if trigger["relation"] == "violation" else "preserved" + reproduced = ( + before.get(trigger["id"]) in ("survived", "regressed", "error") + and after.get(trigger["id"]) == desired + and before.get(control["id"]) == after.get(control["id"]) == "preserved" + ) + results.append( + { + "id": candidate["id"], + "source": candidate["source"], + "before_version": candidate["before"], + "after_version": candidate["after"], + "outcomes": outcomes, + "fix_reproduced": reproduced, + } + ) + summary = { + "schema_version": 1, + "screened": len(data["candidates"]), + "included": len(included), + "excluded": len(data["candidates"]) - len(included), + "fixes_reproduced": sum(r["fix_reproduced"] for r in results), + "results": results, + } + output.mkdir(parents=True, exist_ok=False) + write(output / "summary.json", summary) + write(output / "audit-reports.json", reports) + print(json.dumps(summary, indent=2)) + return summary + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("freeze", "collect", "replay", "child")) + parser.add_argument("--network", action="store_true") + parser.add_argument("--work", type=Path) + parser.add_argument("--output", type=Path) + parser.add_argument("--captured", type=Path) + parser.add_argument("--lock-from", type=Path) + parser.add_argument("--package") + args = parser.parse_args() + if args.action == "child": + child(args.package) + elif args.action == "freeze": + freeze() + elif args.action == "collect": + if not args.network or args.work is None or args.output is None: + parser.error("collect requires --network, --work and --output") + collect(args.work, args.output, args.lock_from) + else: + if args.captured is None or args.output is None: + parser.error("replay requires --captured and --output") + replay(args.captured, args.output) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/study_replay.py b/benchmarks/study_replay.py new file mode 100644 index 0000000..ac52786 --- /dev/null +++ b/benchmarks/study_replay.py @@ -0,0 +1,120 @@ +"""Replay immutable a3 study drivers against MateProbe without changing their sources. + +The CLI runs in its own process. Tests use the same scoped import bridge. Archived +reports stay unchanged; comparison permits only explicit library-version metadata. +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import sys +from contextlib import contextmanager +from copy import deepcopy +from pathlib import Path + +import mateprobe +import mateprobe.model +import mateprobe.mutations +import mateprobe.validator_audit + +ROOT = Path(__file__).resolve().parents[1] +DRIVERS = {"validator_study", "historical_validator_study", "historical_followup"} + + +@contextmanager +def legacy_imports(): + aliases = { + "narrative_contracts": mateprobe, + "narrative_contracts.model": mateprobe.model, + "narrative_contracts.mutations": mateprobe.mutations, + "narrative_contracts.validator_audit": mateprobe.validator_audit, + } + if any( + name in sys.modules and sys.modules[name] is not module for name, module in aliases.items() + ): + raise RuntimeError("Use an environment without an already imported legacy library") + existing = {name: sys.modules[name] for name in aliases if name in sys.modules} + try: + sys.modules.update(aliases) + yield + finally: + for name in aliases: + if name in existing: + sys.modules[name] = existing[name] + else: + sys.modules.pop(name, None) + + +def load_runner(name): + if name not in DRIVERS: + raise ValueError("Unknown frozen study driver") + spec = importlib.util.spec_from_file_location(name, ROOT / "benchmarks" / (name + ".py")) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def compare_outputs(study, frozen, replay): + names = ("summary.json", "audit-reports.json") + if study == "controlled": + names += ("comparisons.json", "observations.json") + for name in names: + expected = json.loads((frozen / name).read_text()) + actual = json.loads((replay / name).read_text()) + normalized = deepcopy(actual) + if name == "audit-reports.json": + if expected.keys() != normalized.keys(): + raise ValueError("Replay report membership changed") + metadata = [(expected[key], normalized[key]) for key in expected] + elif name == "summary.json" and study == "controlled": + metadata = [(expected, normalized)] + else: + metadata = [] + for old, new in metadata: + if ( + old["library_version"] != "0.1.0a3" + or new["library_version"] != mateprobe.__version__ + ): + raise ValueError("Unexpected replay library version") + new["library_version"] = old["library_version"] + if expected != normalized: + raise ValueError(f"Study replay differs beyond library version: {name}") + + +def replay_all(output): + output.mkdir(parents=True, exist_ok=False) + with legacy_imports(): + controlled = load_runner("validator_study") + frozen = ROOT / "benchmarks/validator-study" + controlled.run(frozen / "prepared", output / "controlled", frozen / "evaluation") + compare_outputs("controlled", frozen / "evaluation", output / "controlled") + historical = load_runner("historical_validator_study") + historical.replay(historical.OUT / "captured", output / "historical") + compare_outputs("historical", historical.OUT / "evaluation", output / "historical") + followup = load_runner("historical_followup") + followup.verify() + followup.base.replay(followup.OUT / "captured", output / "followup") + compare_outputs("followup", followup.OUT / "evaluation", output / "followup") + (output / "compatibility.json").write_text( + json.dumps( + { + "status": "passed", + "frozen_library_version": "0.1.0a3", + "replay_library_version": mateprobe.__version__, + "studies": ["controlled", "historical", "followup"], + "allowed_difference": "library_version in reports and controlled summary only", + "scope": "Reclassification of frozen observations; not new third-party executions.", + }, + indent=2, + ) + + "\n" + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True, help="New output directory") + replay_all(parser.parse_args().output.resolve()) diff --git a/benchmarks/validator-study/README.md b/benchmarks/validator-study/README.md new file mode 100644 index 0000000..22d4274 --- /dev/null +++ b/benchmarks/validator-study/README.md @@ -0,0 +1,14 @@ +# Controlled audit evidence + +`prepared/` fixes 144 pairs, protocol and source before execution. `evaluation/` +contains the observed verdicts and 1,152 classifications across eight authored +profiles, complete library reports, and stable summaries. The 24 groups share +policy templates; these are not independent natural-output labels. + +Detailed ordinary assertions agree with the audit on every classification. Boolean +rejection misses diagnostic attribution; removing valid controls hides regressions. +See [interpretation and commands](../../docs/validator-study.md). Offline replay +and fresh controlled-validator execution are both covered by pytest. + +Freeze commits: protocol `5de4a06`, prepared corpus/source `97005d6`. Hashes protect +against accidental mismatches; the manifests themselves are not immutable trust roots. diff --git a/benchmarks/validator-study/evaluation/audit-reports.json b/benchmarks/validator-study/evaluation/audit-reports.json new file mode 100644 index 0000000..2a06193 --- /dev/null +++ b/benchmarks/validator-study/evaluation/audit-reports.json @@ -0,0 +1,31688 @@ +{ + "refund/accept-all": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "completion_status" + ], + "id": "refund-0-0", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "ccd37bbbe861aac51fb3417c8875bc16a79ad8b1b15650bfd3bd93db012c3e1a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "completion_status" + ], + "id": "refund-0-1", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "7d6365821a41313b32aeb27bcd64165be674007e585cc2017f8e9062cc50132a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "completion_status" + ], + "id": "refund-0-2", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "eb15248acc9779f09f4aee4e03341db85a5ee766d91be39efe6502380d99062a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "receipt_request" + ], + "id": "refund-0-3", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "3949f9298e5f5649e74f83a1d8e62d96c5b64edff8f9378df92b8085684382d4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [], + "id": "refund-0-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "bcddd8511b297270f4047551ac9e6640c6771c2ef31edce71189059ebdf5ec57" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [], + "id": "refund-0-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "4e408c96e8c13018a9b1bdfc04b38ff2c68a8a3122f83af9a253565b0538d3fc" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "completion_status" + ], + "id": "refund-1-0", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "6a9d900f69ef9075dbce2d7d04a8709ae7ff9029bc447b683e283f89f303731a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "completion_status" + ], + "id": "refund-1-1", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "74c2c49200a3a03c4c59aea9beae887cb3ccee6c2a0a2ba9ac3f2c6f1d5768ec" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "completion_status" + ], + "id": "refund-1-2", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "c3c21448ab0285666f840c72ce3191a9eb3b6d02f2445327a44f80df8361aadd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "receipt_request" + ], + "id": "refund-1-3", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "bb92aa002539565fd017c122a240a57d93749b7eb3ed8328c812984da8858137" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [], + "id": "refund-1-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f2e8ccfca6cfc8d81233ad79e38f131662442f7725b727aa3c80ca285ddc9e7c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [], + "id": "refund-1-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "31344474c0e8afd5345be5cd26a211de7a50a32d0e37bf34f94d7d6ccec738eb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "completion_status" + ], + "id": "refund-2-0", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "54a66c0bb0365e4877b306eae4d66407a32ef33d3890461a23be376147fac726" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "completion_status" + ], + "id": "refund-2-1", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "32194e2df8816051ebb680eedd4e6c99f439f712454960574bfcacb0c22e508f" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "completion_status" + ], + "id": "refund-2-2", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "134e295e6b5d7cda0f4c2a06f4334f5204a84edd051d54a1a78abc3531dba904" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "receipt_request" + ], + "id": "refund-2-3", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "2fc4b15d440d00482e36a499432d6d72c43a55ff64441550d9f02e3ac0ad8d26" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [], + "id": "refund-2-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "7e019a9656e1cae92ebb143b60725822f3712f715297f9394a0f79046f1af1d2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [], + "id": "refund-2-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "a38bc25f3cb16d0b3c704a0b022794e7521c4658f52b6565316bb4658c14dd1c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "completion_status" + ], + "id": "refund-3-0", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "e7647198864504f2e9e12aacd644c263255cef198d13f952982f4c3e15aea2bd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "completion_status" + ], + "id": "refund-3-1", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "76cb0f8403a05996f178ad64163962622a145090a6f4bb07b1b55a3fd03094fa" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "completion_status" + ], + "id": "refund-3-2", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "ed6aed85348bf39def43220afc8ffd43cb41fc50dfb3cbc4a8e5ce45b81db5ac" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "receipt_request" + ], + "id": "refund-3-3", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "e52d125362843d3d44f14e3668ebe62089654de977049b3dcad5352518000dfd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [], + "id": "refund-3-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "d51b5977bf6366c7e7e9aefaa56db51aef306321b28a25ed75ee55537536240d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [], + "id": "refund-3-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0c2efddae9167df5d220b997e46801498da20707d4af962b6cbdd7e3e967e4b0" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "completion_status" + ], + "id": "refund-4-0", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "1d75a8187e9866f155a596451a99b0ef662fc8bd47468cec8132232c6927284d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "completion_status" + ], + "id": "refund-4-1", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "42c352cde2acf7abea3b16efb2b4eeaf8e64a1fb0550ef0593169add982fe25d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "completion_status" + ], + "id": "refund-4-2", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "39ae7f239158badba8909e1d2647a5e538f1ecbaa275777742f8a5dcefead5e6" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "receipt_request" + ], + "id": "refund-4-3", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "280757cbd993d059788757ec2b3089139d8c77fe3c5c532cc838bcbd647df35c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [], + "id": "refund-4-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5000f458a8323a9c05c1dc32358e0e1475606a8dbfc3dc09030ac4791c1a1b18" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [], + "id": "refund-4-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "75d40912f0d6f3b9d3ddf46ff1995665fc4abba879060050405d594bf53c4201" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "completion_status" + ], + "id": "refund-5-0", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "96b6a87a001bd42d0df33547bac4961c1d13612525accb66637b28d4a515c9ab" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "completion_status" + ], + "id": "refund-5-1", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "6c97be4ee3d68ec0b3bc39d85274b56f5d5cec7a2c6c105853a0eeb287da6149" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "completion_status" + ], + "id": "refund-5-2", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "ac589b53d44f747cb664fcd573e5cb584521c654d209fc192b848ac4d1ae7378" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "receipt_request" + ], + "id": "refund-5-3", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "e4263b7597fbee212263ed32946494cc65ca01235b3500f7976496dcafa90740" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [], + "id": "refund-5-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "a02cd1667f690a76af1b1719d8d18fcd4868e7750bd73264c1848cdf246d31e1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [], + "id": "refund-5-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5f6a0e34ff576a0ebc29af01225a2a7098920857c85199231bfcce4b3640ffb5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "completion_status" + ], + "id": "refund-6-0", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "31350a4758ede02f95dec893447b133494a2aacbf3d4c716f0cc16768d34c934" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "completion_status" + ], + "id": "refund-6-1", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "00ae197ceab0f1a201f556841cb42de0ee5f0e19c76f01477668b26b0d40154d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "completion_status" + ], + "id": "refund-6-2", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "e24387dae93302876c669696b12600f751aec9d39a605c6b285de4460ce1048a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "receipt_request" + ], + "id": "refund-6-3", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "ad00a18a0839d4dc6a7387b6e0cc112257922597feeeea56242bbe3d6bc22be9" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [], + "id": "refund-6-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "11c7484e3401b23a072b73587705d50fcb50229566e11e2f3da54f87c4c57ccd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [], + "id": "refund-6-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "131d4656c82f6ce43a8e50a3e6ec46bcd41e5742afb6b1fb087da19e20d75c72" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "completion_status" + ], + "id": "refund-7-0", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "8c48ec41dd6d04dabc02ff2c69afff046389d4b74d66036cfeb392a3311408da" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "completion_status" + ], + "id": "refund-7-1", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "59776e3cb3b7d053fdc756923213d545c00a861ba319018c169932ec06059715" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "completion_status" + ], + "id": "refund-7-2", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "ecf3771059b7d4db065532b54e2be9b7d0ce69c2c9c7b5d1758b7853c6b98af4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "receipt_request" + ], + "id": "refund-7-3", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "be5a55ceefe275822757e621b44986822d174567af2eb9af68a57802ed45c15a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [], + "id": "refund-7-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0167ec43674a8508cc13432dd4c7a3129f5fc2656d54fe36c8ebd24bc4eade46" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [], + "id": "refund-7-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b4e93f8e27c26696b6260c481d2dcffa7d5ed711ac8562f20566c6576b92d51f" + } + ], + "corpus_digest": "57e293e2ced394347abd60be88a272cf04f7857bc07a906241534db4cd76156b", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Authored refund policy.", + "id": "refund", + "scope": "contract" + }, + { + "description": "An inventoried untested obligation.", + "id": "refund-untested", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 16, + "survived": 32 + }, + "detected_faults": 0, + "detection_score": 0.0, + "eligible_controls": 16, + "eligible_faults": 32, + "obligations": [ + { + "assessment": "known_gaps", + "case_ids": [ + "refund-0-0", + "refund-0-1", + "refund-0-2", + "refund-0-3", + "refund-0-4", + "refund-0-5", + "refund-1-0", + "refund-1-1", + "refund-1-2", + "refund-1-3", + "refund-1-4", + "refund-1-5", + "refund-2-0", + "refund-2-1", + "refund-2-2", + "refund-2-3", + "refund-2-4", + "refund-2-5", + "refund-3-0", + "refund-3-1", + "refund-3-2", + "refund-3-3", + "refund-3-4", + "refund-3-5", + "refund-4-0", + "refund-4-1", + "refund-4-2", + "refund-4-3", + "refund-4-4", + "refund-4-5", + "refund-5-0", + "refund-5-1", + "refund-5-2", + "refund-5-3", + "refund-5-4", + "refund-5-5", + "refund-6-0", + "refund-6-1", + "refund-6-2", + "refund-6-3", + "refund-6-4", + "refund-6-5", + "refund-7-0", + "refund-7-1", + "refund-7-2", + "refund-7-3", + "refund-7-4", + "refund-7-5" + ], + "counts": { + "preserved": 16, + "survived": 32 + }, + "description": "Authored refund policy.", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": true, + "has_incomplete_evidence": false, + "has_known_gaps": true, + "id": "refund", + "scope": "contract" + }, + { + "assessment": "untested", + "case_ids": [], + "counts": {}, + "description": "An inventoried untested obligation.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "refund-untested", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 16, + "total": 48 + }, + "validator_id": "controlled/refund/accept-all/1" + }, + "refund/correct": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "completion_status" + ], + "id": "refund-0-0", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "ccd37bbbe861aac51fb3417c8875bc16a79ad8b1b15650bfd3bd93db012c3e1a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "completion_status" + ], + "id": "refund-0-1", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "7d6365821a41313b32aeb27bcd64165be674007e585cc2017f8e9062cc50132a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "completion_status" + ], + "id": "refund-0-2", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "eb15248acc9779f09f4aee4e03341db85a5ee766d91be39efe6502380d99062a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "receipt_request" + ], + "id": "refund-0-3", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "receipt_request" + ] + }, + "variant_digest": "3949f9298e5f5649e74f83a1d8e62d96c5b64edff8f9378df92b8085684382d4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [], + "id": "refund-0-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "bcddd8511b297270f4047551ac9e6640c6771c2ef31edce71189059ebdf5ec57" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [], + "id": "refund-0-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "4e408c96e8c13018a9b1bdfc04b38ff2c68a8a3122f83af9a253565b0538d3fc" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "completion_status" + ], + "id": "refund-1-0", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "6a9d900f69ef9075dbce2d7d04a8709ae7ff9029bc447b683e283f89f303731a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "completion_status" + ], + "id": "refund-1-1", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "74c2c49200a3a03c4c59aea9beae887cb3ccee6c2a0a2ba9ac3f2c6f1d5768ec" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "completion_status" + ], + "id": "refund-1-2", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "c3c21448ab0285666f840c72ce3191a9eb3b6d02f2445327a44f80df8361aadd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "receipt_request" + ], + "id": "refund-1-3", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "receipt_request" + ] + }, + "variant_digest": "bb92aa002539565fd017c122a240a57d93749b7eb3ed8328c812984da8858137" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [], + "id": "refund-1-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f2e8ccfca6cfc8d81233ad79e38f131662442f7725b727aa3c80ca285ddc9e7c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [], + "id": "refund-1-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "31344474c0e8afd5345be5cd26a211de7a50a32d0e37bf34f94d7d6ccec738eb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "completion_status" + ], + "id": "refund-2-0", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "54a66c0bb0365e4877b306eae4d66407a32ef33d3890461a23be376147fac726" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "completion_status" + ], + "id": "refund-2-1", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "32194e2df8816051ebb680eedd4e6c99f439f712454960574bfcacb0c22e508f" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "completion_status" + ], + "id": "refund-2-2", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "134e295e6b5d7cda0f4c2a06f4334f5204a84edd051d54a1a78abc3531dba904" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "receipt_request" + ], + "id": "refund-2-3", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "receipt_request" + ] + }, + "variant_digest": "2fc4b15d440d00482e36a499432d6d72c43a55ff64441550d9f02e3ac0ad8d26" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [], + "id": "refund-2-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "7e019a9656e1cae92ebb143b60725822f3712f715297f9394a0f79046f1af1d2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [], + "id": "refund-2-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "a38bc25f3cb16d0b3c704a0b022794e7521c4658f52b6565316bb4658c14dd1c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "completion_status" + ], + "id": "refund-3-0", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "e7647198864504f2e9e12aacd644c263255cef198d13f952982f4c3e15aea2bd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "completion_status" + ], + "id": "refund-3-1", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "76cb0f8403a05996f178ad64163962622a145090a6f4bb07b1b55a3fd03094fa" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "completion_status" + ], + "id": "refund-3-2", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "ed6aed85348bf39def43220afc8ffd43cb41fc50dfb3cbc4a8e5ce45b81db5ac" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "receipt_request" + ], + "id": "refund-3-3", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "receipt_request" + ] + }, + "variant_digest": "e52d125362843d3d44f14e3668ebe62089654de977049b3dcad5352518000dfd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [], + "id": "refund-3-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "d51b5977bf6366c7e7e9aefaa56db51aef306321b28a25ed75ee55537536240d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [], + "id": "refund-3-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0c2efddae9167df5d220b997e46801498da20707d4af962b6cbdd7e3e967e4b0" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "completion_status" + ], + "id": "refund-4-0", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "1d75a8187e9866f155a596451a99b0ef662fc8bd47468cec8132232c6927284d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "completion_status" + ], + "id": "refund-4-1", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "42c352cde2acf7abea3b16efb2b4eeaf8e64a1fb0550ef0593169add982fe25d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "completion_status" + ], + "id": "refund-4-2", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "39ae7f239158badba8909e1d2647a5e538f1ecbaa275777742f8a5dcefead5e6" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "receipt_request" + ], + "id": "refund-4-3", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "receipt_request" + ] + }, + "variant_digest": "280757cbd993d059788757ec2b3089139d8c77fe3c5c532cc838bcbd647df35c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [], + "id": "refund-4-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5000f458a8323a9c05c1dc32358e0e1475606a8dbfc3dc09030ac4791c1a1b18" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [], + "id": "refund-4-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "75d40912f0d6f3b9d3ddf46ff1995665fc4abba879060050405d594bf53c4201" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "completion_status" + ], + "id": "refund-5-0", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "96b6a87a001bd42d0df33547bac4961c1d13612525accb66637b28d4a515c9ab" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "completion_status" + ], + "id": "refund-5-1", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "6c97be4ee3d68ec0b3bc39d85274b56f5d5cec7a2c6c105853a0eeb287da6149" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "completion_status" + ], + "id": "refund-5-2", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "ac589b53d44f747cb664fcd573e5cb584521c654d209fc192b848ac4d1ae7378" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "receipt_request" + ], + "id": "refund-5-3", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "receipt_request" + ] + }, + "variant_digest": "e4263b7597fbee212263ed32946494cc65ca01235b3500f7976496dcafa90740" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [], + "id": "refund-5-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "a02cd1667f690a76af1b1719d8d18fcd4868e7750bd73264c1848cdf246d31e1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [], + "id": "refund-5-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5f6a0e34ff576a0ebc29af01225a2a7098920857c85199231bfcce4b3640ffb5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "completion_status" + ], + "id": "refund-6-0", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "31350a4758ede02f95dec893447b133494a2aacbf3d4c716f0cc16768d34c934" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "completion_status" + ], + "id": "refund-6-1", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "00ae197ceab0f1a201f556841cb42de0ee5f0e19c76f01477668b26b0d40154d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "completion_status" + ], + "id": "refund-6-2", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "e24387dae93302876c669696b12600f751aec9d39a605c6b285de4460ce1048a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "receipt_request" + ], + "id": "refund-6-3", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "receipt_request" + ] + }, + "variant_digest": "ad00a18a0839d4dc6a7387b6e0cc112257922597feeeea56242bbe3d6bc22be9" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [], + "id": "refund-6-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "11c7484e3401b23a072b73587705d50fcb50229566e11e2f3da54f87c4c57ccd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [], + "id": "refund-6-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "131d4656c82f6ce43a8e50a3e6ec46bcd41e5742afb6b1fb087da19e20d75c72" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "completion_status" + ], + "id": "refund-7-0", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "8c48ec41dd6d04dabc02ff2c69afff046389d4b74d66036cfeb392a3311408da" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "completion_status" + ], + "id": "refund-7-1", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "59776e3cb3b7d053fdc756923213d545c00a861ba319018c169932ec06059715" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "completion_status" + ], + "id": "refund-7-2", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "ecf3771059b7d4db065532b54e2be9b7d0ce69c2c9c7b5d1758b7853c6b98af4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "receipt_request" + ], + "id": "refund-7-3", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "receipt_request" + ] + }, + "variant_digest": "be5a55ceefe275822757e621b44986822d174567af2eb9af68a57802ed45c15a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [], + "id": "refund-7-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0167ec43674a8508cc13432dd4c7a3129f5fc2656d54fe36c8ebd24bc4eade46" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [], + "id": "refund-7-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b4e93f8e27c26696b6260c481d2dcffa7d5ed711ac8562f20566c6576b92d51f" + } + ], + "corpus_digest": "57e293e2ced394347abd60be88a272cf04f7857bc07a906241534db4cd76156b", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Authored refund policy.", + "id": "refund", + "scope": "contract" + }, + { + "description": "An inventoried untested obligation.", + "id": "refund-untested", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "detected": 32, + "preserved": 16 + }, + "detected_faults": 32, + "detection_score": 1.0, + "eligible_controls": 16, + "eligible_faults": 32, + "obligations": [ + { + "assessment": "no_failures_observed", + "case_ids": [ + "refund-0-0", + "refund-0-1", + "refund-0-2", + "refund-0-3", + "refund-0-4", + "refund-0-5", + "refund-1-0", + "refund-1-1", + "refund-1-2", + "refund-1-3", + "refund-1-4", + "refund-1-5", + "refund-2-0", + "refund-2-1", + "refund-2-2", + "refund-2-3", + "refund-2-4", + "refund-2-5", + "refund-3-0", + "refund-3-1", + "refund-3-2", + "refund-3-3", + "refund-3-4", + "refund-3-5", + "refund-4-0", + "refund-4-1", + "refund-4-2", + "refund-4-3", + "refund-4-4", + "refund-4-5", + "refund-5-0", + "refund-5-1", + "refund-5-2", + "refund-5-3", + "refund-5-4", + "refund-5-5", + "refund-6-0", + "refund-6-1", + "refund-6-2", + "refund-6-3", + "refund-6-4", + "refund-6-5", + "refund-7-0", + "refund-7-1", + "refund-7-2", + "refund-7-3", + "refund-7-4", + "refund-7-5" + ], + "counts": { + "detected": 32, + "preserved": 16 + }, + "description": "Authored refund policy.", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": true, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "refund", + "scope": "contract" + }, + { + "assessment": "untested", + "case_ids": [], + "counts": {}, + "description": "An inventoried untested obligation.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "refund-untested", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 16, + "total": 48 + }, + "validator_id": "controlled/refund/correct/1" + }, + "refund/crash-on-fault": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "completion_status" + ], + "id": "refund-0-0", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "ccd37bbbe861aac51fb3417c8875bc16a79ad8b1b15650bfd3bd93db012c3e1a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "completion_status" + ], + "id": "refund-0-1", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "7d6365821a41313b32aeb27bcd64165be674007e585cc2017f8e9062cc50132a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "completion_status" + ], + "id": "refund-0-2", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "eb15248acc9779f09f4aee4e03341db85a5ee766d91be39efe6502380d99062a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "receipt_request" + ], + "id": "refund-0-3", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "3949f9298e5f5649e74f83a1d8e62d96c5b64edff8f9378df92b8085684382d4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [], + "id": "refund-0-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "bcddd8511b297270f4047551ac9e6640c6771c2ef31edce71189059ebdf5ec57" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [], + "id": "refund-0-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "4e408c96e8c13018a9b1bdfc04b38ff2c68a8a3122f83af9a253565b0538d3fc" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "completion_status" + ], + "id": "refund-1-0", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "6a9d900f69ef9075dbce2d7d04a8709ae7ff9029bc447b683e283f89f303731a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "completion_status" + ], + "id": "refund-1-1", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "74c2c49200a3a03c4c59aea9beae887cb3ccee6c2a0a2ba9ac3f2c6f1d5768ec" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "completion_status" + ], + "id": "refund-1-2", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "c3c21448ab0285666f840c72ce3191a9eb3b6d02f2445327a44f80df8361aadd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "receipt_request" + ], + "id": "refund-1-3", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "bb92aa002539565fd017c122a240a57d93749b7eb3ed8328c812984da8858137" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [], + "id": "refund-1-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f2e8ccfca6cfc8d81233ad79e38f131662442f7725b727aa3c80ca285ddc9e7c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [], + "id": "refund-1-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "31344474c0e8afd5345be5cd26a211de7a50a32d0e37bf34f94d7d6ccec738eb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "completion_status" + ], + "id": "refund-2-0", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "54a66c0bb0365e4877b306eae4d66407a32ef33d3890461a23be376147fac726" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "completion_status" + ], + "id": "refund-2-1", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "32194e2df8816051ebb680eedd4e6c99f439f712454960574bfcacb0c22e508f" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "completion_status" + ], + "id": "refund-2-2", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "134e295e6b5d7cda0f4c2a06f4334f5204a84edd051d54a1a78abc3531dba904" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "receipt_request" + ], + "id": "refund-2-3", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "2fc4b15d440d00482e36a499432d6d72c43a55ff64441550d9f02e3ac0ad8d26" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [], + "id": "refund-2-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "7e019a9656e1cae92ebb143b60725822f3712f715297f9394a0f79046f1af1d2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [], + "id": "refund-2-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "a38bc25f3cb16d0b3c704a0b022794e7521c4658f52b6565316bb4658c14dd1c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "completion_status" + ], + "id": "refund-3-0", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "e7647198864504f2e9e12aacd644c263255cef198d13f952982f4c3e15aea2bd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "completion_status" + ], + "id": "refund-3-1", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "76cb0f8403a05996f178ad64163962622a145090a6f4bb07b1b55a3fd03094fa" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "completion_status" + ], + "id": "refund-3-2", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "ed6aed85348bf39def43220afc8ffd43cb41fc50dfb3cbc4a8e5ce45b81db5ac" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "receipt_request" + ], + "id": "refund-3-3", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "e52d125362843d3d44f14e3668ebe62089654de977049b3dcad5352518000dfd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [], + "id": "refund-3-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "d51b5977bf6366c7e7e9aefaa56db51aef306321b28a25ed75ee55537536240d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [], + "id": "refund-3-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0c2efddae9167df5d220b997e46801498da20707d4af962b6cbdd7e3e967e4b0" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "completion_status" + ], + "id": "refund-4-0", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "1d75a8187e9866f155a596451a99b0ef662fc8bd47468cec8132232c6927284d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "completion_status" + ], + "id": "refund-4-1", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "42c352cde2acf7abea3b16efb2b4eeaf8e64a1fb0550ef0593169add982fe25d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "completion_status" + ], + "id": "refund-4-2", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "39ae7f239158badba8909e1d2647a5e538f1ecbaa275777742f8a5dcefead5e6" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "receipt_request" + ], + "id": "refund-4-3", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "280757cbd993d059788757ec2b3089139d8c77fe3c5c532cc838bcbd647df35c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [], + "id": "refund-4-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5000f458a8323a9c05c1dc32358e0e1475606a8dbfc3dc09030ac4791c1a1b18" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [], + "id": "refund-4-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "75d40912f0d6f3b9d3ddf46ff1995665fc4abba879060050405d594bf53c4201" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "completion_status" + ], + "id": "refund-5-0", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "96b6a87a001bd42d0df33547bac4961c1d13612525accb66637b28d4a515c9ab" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "completion_status" + ], + "id": "refund-5-1", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "6c97be4ee3d68ec0b3bc39d85274b56f5d5cec7a2c6c105853a0eeb287da6149" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "completion_status" + ], + "id": "refund-5-2", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "ac589b53d44f747cb664fcd573e5cb584521c654d209fc192b848ac4d1ae7378" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "receipt_request" + ], + "id": "refund-5-3", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "e4263b7597fbee212263ed32946494cc65ca01235b3500f7976496dcafa90740" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [], + "id": "refund-5-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "a02cd1667f690a76af1b1719d8d18fcd4868e7750bd73264c1848cdf246d31e1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [], + "id": "refund-5-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5f6a0e34ff576a0ebc29af01225a2a7098920857c85199231bfcce4b3640ffb5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "completion_status" + ], + "id": "refund-6-0", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "31350a4758ede02f95dec893447b133494a2aacbf3d4c716f0cc16768d34c934" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "completion_status" + ], + "id": "refund-6-1", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "00ae197ceab0f1a201f556841cb42de0ee5f0e19c76f01477668b26b0d40154d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "completion_status" + ], + "id": "refund-6-2", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "e24387dae93302876c669696b12600f751aec9d39a605c6b285de4460ce1048a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "receipt_request" + ], + "id": "refund-6-3", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "ad00a18a0839d4dc6a7387b6e0cc112257922597feeeea56242bbe3d6bc22be9" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [], + "id": "refund-6-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "11c7484e3401b23a072b73587705d50fcb50229566e11e2f3da54f87c4c57ccd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [], + "id": "refund-6-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "131d4656c82f6ce43a8e50a3e6ec46bcd41e5742afb6b1fb087da19e20d75c72" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "completion_status" + ], + "id": "refund-7-0", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "8c48ec41dd6d04dabc02ff2c69afff046389d4b74d66036cfeb392a3311408da" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "completion_status" + ], + "id": "refund-7-1", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "59776e3cb3b7d053fdc756923213d545c00a861ba319018c169932ec06059715" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "completion_status" + ], + "id": "refund-7-2", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "ecf3771059b7d4db065532b54e2be9b7d0ce69c2c9c7b5d1758b7853c6b98af4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "receipt_request" + ], + "id": "refund-7-3", + "obligation": "refund", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "be5a55ceefe275822757e621b44986822d174567af2eb9af68a57802ed45c15a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [], + "id": "refund-7-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0167ec43674a8508cc13432dd4c7a3129f5fc2656d54fe36c8ebd24bc4eade46" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [], + "id": "refund-7-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b4e93f8e27c26696b6260c481d2dcffa7d5ed711ac8562f20566c6576b92d51f" + } + ], + "corpus_digest": "57e293e2ced394347abd60be88a272cf04f7857bc07a906241534db4cd76156b", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Authored refund policy.", + "id": "refund", + "scope": "contract" + }, + { + "description": "An inventoried untested obligation.", + "id": "refund-untested", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "error": 32, + "preserved": 16 + }, + "detected_faults": 0, + "detection_score": 0.0, + "eligible_controls": 16, + "eligible_faults": 32, + "obligations": [ + { + "assessment": "incomplete_evidence", + "case_ids": [ + "refund-0-0", + "refund-0-1", + "refund-0-2", + "refund-0-3", + "refund-0-4", + "refund-0-5", + "refund-1-0", + "refund-1-1", + "refund-1-2", + "refund-1-3", + "refund-1-4", + "refund-1-5", + "refund-2-0", + "refund-2-1", + "refund-2-2", + "refund-2-3", + "refund-2-4", + "refund-2-5", + "refund-3-0", + "refund-3-1", + "refund-3-2", + "refund-3-3", + "refund-3-4", + "refund-3-5", + "refund-4-0", + "refund-4-1", + "refund-4-2", + "refund-4-3", + "refund-4-4", + "refund-4-5", + "refund-5-0", + "refund-5-1", + "refund-5-2", + "refund-5-3", + "refund-5-4", + "refund-5-5", + "refund-6-0", + "refund-6-1", + "refund-6-2", + "refund-6-3", + "refund-6-4", + "refund-6-5", + "refund-7-0", + "refund-7-1", + "refund-7-2", + "refund-7-3", + "refund-7-4", + "refund-7-5" + ], + "counts": { + "error": 32, + "preserved": 16 + }, + "description": "Authored refund policy.", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": true, + "has_incomplete_evidence": true, + "has_known_gaps": false, + "id": "refund", + "scope": "contract" + }, + { + "assessment": "untested", + "case_ids": [], + "counts": {}, + "description": "An inventoried untested obligation.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "refund-untested", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 16, + "total": 48 + }, + "validator_id": "controlled/refund/crash-on-fault/1" + }, + "refund/omit-first-obligation": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "completion_status" + ], + "id": "refund-0-0", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "ccd37bbbe861aac51fb3417c8875bc16a79ad8b1b15650bfd3bd93db012c3e1a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "completion_status" + ], + "id": "refund-0-1", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "7d6365821a41313b32aeb27bcd64165be674007e585cc2017f8e9062cc50132a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "completion_status" + ], + "id": "refund-0-2", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "eb15248acc9779f09f4aee4e03341db85a5ee766d91be39efe6502380d99062a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "receipt_request" + ], + "id": "refund-0-3", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "receipt_request" + ] + }, + "variant_digest": "3949f9298e5f5649e74f83a1d8e62d96c5b64edff8f9378df92b8085684382d4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [], + "id": "refund-0-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "bcddd8511b297270f4047551ac9e6640c6771c2ef31edce71189059ebdf5ec57" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [], + "id": "refund-0-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "4e408c96e8c13018a9b1bdfc04b38ff2c68a8a3122f83af9a253565b0538d3fc" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "completion_status" + ], + "id": "refund-1-0", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "6a9d900f69ef9075dbce2d7d04a8709ae7ff9029bc447b683e283f89f303731a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "completion_status" + ], + "id": "refund-1-1", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "74c2c49200a3a03c4c59aea9beae887cb3ccee6c2a0a2ba9ac3f2c6f1d5768ec" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "completion_status" + ], + "id": "refund-1-2", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "c3c21448ab0285666f840c72ce3191a9eb3b6d02f2445327a44f80df8361aadd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "receipt_request" + ], + "id": "refund-1-3", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "receipt_request" + ] + }, + "variant_digest": "bb92aa002539565fd017c122a240a57d93749b7eb3ed8328c812984da8858137" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [], + "id": "refund-1-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f2e8ccfca6cfc8d81233ad79e38f131662442f7725b727aa3c80ca285ddc9e7c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [], + "id": "refund-1-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "31344474c0e8afd5345be5cd26a211de7a50a32d0e37bf34f94d7d6ccec738eb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "completion_status" + ], + "id": "refund-2-0", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "54a66c0bb0365e4877b306eae4d66407a32ef33d3890461a23be376147fac726" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "completion_status" + ], + "id": "refund-2-1", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "32194e2df8816051ebb680eedd4e6c99f439f712454960574bfcacb0c22e508f" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "completion_status" + ], + "id": "refund-2-2", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "134e295e6b5d7cda0f4c2a06f4334f5204a84edd051d54a1a78abc3531dba904" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "receipt_request" + ], + "id": "refund-2-3", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "receipt_request" + ] + }, + "variant_digest": "2fc4b15d440d00482e36a499432d6d72c43a55ff64441550d9f02e3ac0ad8d26" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [], + "id": "refund-2-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "7e019a9656e1cae92ebb143b60725822f3712f715297f9394a0f79046f1af1d2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [], + "id": "refund-2-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "a38bc25f3cb16d0b3c704a0b022794e7521c4658f52b6565316bb4658c14dd1c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "completion_status" + ], + "id": "refund-3-0", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "e7647198864504f2e9e12aacd644c263255cef198d13f952982f4c3e15aea2bd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "completion_status" + ], + "id": "refund-3-1", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "76cb0f8403a05996f178ad64163962622a145090a6f4bb07b1b55a3fd03094fa" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "completion_status" + ], + "id": "refund-3-2", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "ed6aed85348bf39def43220afc8ffd43cb41fc50dfb3cbc4a8e5ce45b81db5ac" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "receipt_request" + ], + "id": "refund-3-3", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "receipt_request" + ] + }, + "variant_digest": "e52d125362843d3d44f14e3668ebe62089654de977049b3dcad5352518000dfd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [], + "id": "refund-3-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "d51b5977bf6366c7e7e9aefaa56db51aef306321b28a25ed75ee55537536240d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [], + "id": "refund-3-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0c2efddae9167df5d220b997e46801498da20707d4af962b6cbdd7e3e967e4b0" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "completion_status" + ], + "id": "refund-4-0", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "1d75a8187e9866f155a596451a99b0ef662fc8bd47468cec8132232c6927284d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "completion_status" + ], + "id": "refund-4-1", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "42c352cde2acf7abea3b16efb2b4eeaf8e64a1fb0550ef0593169add982fe25d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "completion_status" + ], + "id": "refund-4-2", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "39ae7f239158badba8909e1d2647a5e538f1ecbaa275777742f8a5dcefead5e6" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "receipt_request" + ], + "id": "refund-4-3", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "receipt_request" + ] + }, + "variant_digest": "280757cbd993d059788757ec2b3089139d8c77fe3c5c532cc838bcbd647df35c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [], + "id": "refund-4-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5000f458a8323a9c05c1dc32358e0e1475606a8dbfc3dc09030ac4791c1a1b18" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [], + "id": "refund-4-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "75d40912f0d6f3b9d3ddf46ff1995665fc4abba879060050405d594bf53c4201" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "completion_status" + ], + "id": "refund-5-0", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "96b6a87a001bd42d0df33547bac4961c1d13612525accb66637b28d4a515c9ab" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "completion_status" + ], + "id": "refund-5-1", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "6c97be4ee3d68ec0b3bc39d85274b56f5d5cec7a2c6c105853a0eeb287da6149" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "completion_status" + ], + "id": "refund-5-2", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "ac589b53d44f747cb664fcd573e5cb584521c654d209fc192b848ac4d1ae7378" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "receipt_request" + ], + "id": "refund-5-3", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "receipt_request" + ] + }, + "variant_digest": "e4263b7597fbee212263ed32946494cc65ca01235b3500f7976496dcafa90740" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [], + "id": "refund-5-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "a02cd1667f690a76af1b1719d8d18fcd4868e7750bd73264c1848cdf246d31e1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [], + "id": "refund-5-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5f6a0e34ff576a0ebc29af01225a2a7098920857c85199231bfcce4b3640ffb5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "completion_status" + ], + "id": "refund-6-0", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "31350a4758ede02f95dec893447b133494a2aacbf3d4c716f0cc16768d34c934" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "completion_status" + ], + "id": "refund-6-1", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "00ae197ceab0f1a201f556841cb42de0ee5f0e19c76f01477668b26b0d40154d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "completion_status" + ], + "id": "refund-6-2", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "e24387dae93302876c669696b12600f751aec9d39a605c6b285de4460ce1048a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "receipt_request" + ], + "id": "refund-6-3", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "receipt_request" + ] + }, + "variant_digest": "ad00a18a0839d4dc6a7387b6e0cc112257922597feeeea56242bbe3d6bc22be9" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [], + "id": "refund-6-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "11c7484e3401b23a072b73587705d50fcb50229566e11e2f3da54f87c4c57ccd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [], + "id": "refund-6-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "131d4656c82f6ce43a8e50a3e6ec46bcd41e5742afb6b1fb087da19e20d75c72" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "completion_status" + ], + "id": "refund-7-0", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "8c48ec41dd6d04dabc02ff2c69afff046389d4b74d66036cfeb392a3311408da" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "completion_status" + ], + "id": "refund-7-1", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "59776e3cb3b7d053fdc756923213d545c00a861ba319018c169932ec06059715" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "completion_status" + ], + "id": "refund-7-2", + "obligation": "refund", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "ecf3771059b7d4db065532b54e2be9b7d0ce69c2c9c7b5d1758b7853c6b98af4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "receipt_request" + ], + "id": "refund-7-3", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "receipt_request" + ] + }, + "variant_digest": "be5a55ceefe275822757e621b44986822d174567af2eb9af68a57802ed45c15a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [], + "id": "refund-7-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0167ec43674a8508cc13432dd4c7a3129f5fc2656d54fe36c8ebd24bc4eade46" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [], + "id": "refund-7-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b4e93f8e27c26696b6260c481d2dcffa7d5ed711ac8562f20566c6576b92d51f" + } + ], + "corpus_digest": "57e293e2ced394347abd60be88a272cf04f7857bc07a906241534db4cd76156b", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Authored refund policy.", + "id": "refund", + "scope": "contract" + }, + { + "description": "An inventoried untested obligation.", + "id": "refund-untested", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "detected": 8, + "preserved": 16, + "survived": 24 + }, + "detected_faults": 8, + "detection_score": 0.25, + "eligible_controls": 16, + "eligible_faults": 32, + "obligations": [ + { + "assessment": "known_gaps", + "case_ids": [ + "refund-0-0", + "refund-0-1", + "refund-0-2", + "refund-0-3", + "refund-0-4", + "refund-0-5", + "refund-1-0", + "refund-1-1", + "refund-1-2", + "refund-1-3", + "refund-1-4", + "refund-1-5", + "refund-2-0", + "refund-2-1", + "refund-2-2", + "refund-2-3", + "refund-2-4", + "refund-2-5", + "refund-3-0", + "refund-3-1", + "refund-3-2", + "refund-3-3", + "refund-3-4", + "refund-3-5", + "refund-4-0", + "refund-4-1", + "refund-4-2", + "refund-4-3", + "refund-4-4", + "refund-4-5", + "refund-5-0", + "refund-5-1", + "refund-5-2", + "refund-5-3", + "refund-5-4", + "refund-5-5", + "refund-6-0", + "refund-6-1", + "refund-6-2", + "refund-6-3", + "refund-6-4", + "refund-6-5", + "refund-7-0", + "refund-7-1", + "refund-7-2", + "refund-7-3", + "refund-7-4", + "refund-7-5" + ], + "counts": { + "detected": 8, + "preserved": 16, + "survived": 24 + }, + "description": "Authored refund policy.", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": true, + "has_incomplete_evidence": false, + "has_known_gaps": true, + "id": "refund", + "scope": "contract" + }, + { + "assessment": "untested", + "case_ids": [], + "counts": {}, + "description": "An inventoried untested obligation.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "refund-untested", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 16, + "total": 48 + }, + "validator_id": "controlled/refund/omit-first-obligation/1" + }, + "refund/reject-all": { + "cases": [ + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "completion_status" + ], + "id": "refund-0-0", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "ccd37bbbe861aac51fb3417c8875bc16a79ad8b1b15650bfd3bd93db012c3e1a" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "completion_status" + ], + "id": "refund-0-1", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "7d6365821a41313b32aeb27bcd64165be674007e585cc2017f8e9062cc50132a" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "completion_status" + ], + "id": "refund-0-2", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "eb15248acc9779f09f4aee4e03341db85a5ee766d91be39efe6502380d99062a" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "receipt_request" + ], + "id": "refund-0-3", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "3949f9298e5f5649e74f83a1d8e62d96c5b64edff8f9378df92b8085684382d4" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [], + "id": "refund-0-4", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "bcddd8511b297270f4047551ac9e6640c6771c2ef31edce71189059ebdf5ec57" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [], + "id": "refund-0-5", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "4e408c96e8c13018a9b1bdfc04b38ff2c68a8a3122f83af9a253565b0538d3fc" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "completion_status" + ], + "id": "refund-1-0", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "6a9d900f69ef9075dbce2d7d04a8709ae7ff9029bc447b683e283f89f303731a" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "completion_status" + ], + "id": "refund-1-1", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "74c2c49200a3a03c4c59aea9beae887cb3ccee6c2a0a2ba9ac3f2c6f1d5768ec" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "completion_status" + ], + "id": "refund-1-2", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "c3c21448ab0285666f840c72ce3191a9eb3b6d02f2445327a44f80df8361aadd" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "receipt_request" + ], + "id": "refund-1-3", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "bb92aa002539565fd017c122a240a57d93749b7eb3ed8328c812984da8858137" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [], + "id": "refund-1-4", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "f2e8ccfca6cfc8d81233ad79e38f131662442f7725b727aa3c80ca285ddc9e7c" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [], + "id": "refund-1-5", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "31344474c0e8afd5345be5cd26a211de7a50a32d0e37bf34f94d7d6ccec738eb" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "completion_status" + ], + "id": "refund-2-0", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "54a66c0bb0365e4877b306eae4d66407a32ef33d3890461a23be376147fac726" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "completion_status" + ], + "id": "refund-2-1", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "32194e2df8816051ebb680eedd4e6c99f439f712454960574bfcacb0c22e508f" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "completion_status" + ], + "id": "refund-2-2", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "134e295e6b5d7cda0f4c2a06f4334f5204a84edd051d54a1a78abc3531dba904" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "receipt_request" + ], + "id": "refund-2-3", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "2fc4b15d440d00482e36a499432d6d72c43a55ff64441550d9f02e3ac0ad8d26" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [], + "id": "refund-2-4", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "7e019a9656e1cae92ebb143b60725822f3712f715297f9394a0f79046f1af1d2" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [], + "id": "refund-2-5", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "a38bc25f3cb16d0b3c704a0b022794e7521c4658f52b6565316bb4658c14dd1c" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "completion_status" + ], + "id": "refund-3-0", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "e7647198864504f2e9e12aacd644c263255cef198d13f952982f4c3e15aea2bd" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "completion_status" + ], + "id": "refund-3-1", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "76cb0f8403a05996f178ad64163962622a145090a6f4bb07b1b55a3fd03094fa" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "completion_status" + ], + "id": "refund-3-2", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "ed6aed85348bf39def43220afc8ffd43cb41fc50dfb3cbc4a8e5ce45b81db5ac" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "receipt_request" + ], + "id": "refund-3-3", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "e52d125362843d3d44f14e3668ebe62089654de977049b3dcad5352518000dfd" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [], + "id": "refund-3-4", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "d51b5977bf6366c7e7e9aefaa56db51aef306321b28a25ed75ee55537536240d" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [], + "id": "refund-3-5", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "0c2efddae9167df5d220b997e46801498da20707d4af962b6cbdd7e3e967e4b0" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "completion_status" + ], + "id": "refund-4-0", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "1d75a8187e9866f155a596451a99b0ef662fc8bd47468cec8132232c6927284d" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "completion_status" + ], + "id": "refund-4-1", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "42c352cde2acf7abea3b16efb2b4eeaf8e64a1fb0550ef0593169add982fe25d" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "completion_status" + ], + "id": "refund-4-2", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "39ae7f239158badba8909e1d2647a5e538f1ecbaa275777742f8a5dcefead5e6" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "receipt_request" + ], + "id": "refund-4-3", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "280757cbd993d059788757ec2b3089139d8c77fe3c5c532cc838bcbd647df35c" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [], + "id": "refund-4-4", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "5000f458a8323a9c05c1dc32358e0e1475606a8dbfc3dc09030ac4791c1a1b18" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [], + "id": "refund-4-5", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "75d40912f0d6f3b9d3ddf46ff1995665fc4abba879060050405d594bf53c4201" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "completion_status" + ], + "id": "refund-5-0", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "96b6a87a001bd42d0df33547bac4961c1d13612525accb66637b28d4a515c9ab" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "completion_status" + ], + "id": "refund-5-1", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "6c97be4ee3d68ec0b3bc39d85274b56f5d5cec7a2c6c105853a0eeb287da6149" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "completion_status" + ], + "id": "refund-5-2", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "ac589b53d44f747cb664fcd573e5cb584521c654d209fc192b848ac4d1ae7378" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "receipt_request" + ], + "id": "refund-5-3", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "e4263b7597fbee212263ed32946494cc65ca01235b3500f7976496dcafa90740" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [], + "id": "refund-5-4", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "a02cd1667f690a76af1b1719d8d18fcd4868e7750bd73264c1848cdf246d31e1" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [], + "id": "refund-5-5", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "5f6a0e34ff576a0ebc29af01225a2a7098920857c85199231bfcce4b3640ffb5" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "completion_status" + ], + "id": "refund-6-0", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "31350a4758ede02f95dec893447b133494a2aacbf3d4c716f0cc16768d34c934" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "completion_status" + ], + "id": "refund-6-1", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "00ae197ceab0f1a201f556841cb42de0ee5f0e19c76f01477668b26b0d40154d" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "completion_status" + ], + "id": "refund-6-2", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "e24387dae93302876c669696b12600f751aec9d39a605c6b285de4460ce1048a" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "receipt_request" + ], + "id": "refund-6-3", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "ad00a18a0839d4dc6a7387b6e0cc112257922597feeeea56242bbe3d6bc22be9" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [], + "id": "refund-6-4", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "11c7484e3401b23a072b73587705d50fcb50229566e11e2f3da54f87c4c57ccd" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [], + "id": "refund-6-5", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "131d4656c82f6ce43a8e50a3e6ec46bcd41e5742afb6b1fb087da19e20d75c72" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "completion_status" + ], + "id": "refund-7-0", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "8c48ec41dd6d04dabc02ff2c69afff046389d4b74d66036cfeb392a3311408da" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "completion_status" + ], + "id": "refund-7-1", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "59776e3cb3b7d053fdc756923213d545c00a861ba319018c169932ec06059715" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "completion_status" + ], + "id": "refund-7-2", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "ecf3771059b7d4db065532b54e2be9b7d0ce69c2c9c7b5d1758b7853c6b98af4" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "receipt_request" + ], + "id": "refund-7-3", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "be5a55ceefe275822757e621b44986822d174567af2eb9af68a57802ed45c15a" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [], + "id": "refund-7-4", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "0167ec43674a8508cc13432dd4c7a3129f5fc2656d54fe36c8ebd24bc4eade46" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [], + "id": "refund-7-5", + "obligation": "refund", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "b4e93f8e27c26696b6260c481d2dcffa7d5ed711ac8562f20566c6576b92d51f" + } + ], + "corpus_digest": "57e293e2ced394347abd60be88a272cf04f7857bc07a906241534db4cd76156b", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Authored refund policy.", + "id": "refund", + "scope": "contract" + }, + { + "description": "An inventoried untested obligation.", + "id": "refund-untested", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "baseline_failed": 48 + }, + "detected_faults": 0, + "detection_score": null, + "eligible_controls": 0, + "eligible_faults": 0, + "obligations": [ + { + "assessment": "incomplete_evidence", + "case_ids": [ + "refund-0-0", + "refund-0-1", + "refund-0-2", + "refund-0-3", + "refund-0-4", + "refund-0-5", + "refund-1-0", + "refund-1-1", + "refund-1-2", + "refund-1-3", + "refund-1-4", + "refund-1-5", + "refund-2-0", + "refund-2-1", + "refund-2-2", + "refund-2-3", + "refund-2-4", + "refund-2-5", + "refund-3-0", + "refund-3-1", + "refund-3-2", + "refund-3-3", + "refund-3-4", + "refund-3-5", + "refund-4-0", + "refund-4-1", + "refund-4-2", + "refund-4-3", + "refund-4-4", + "refund-4-5", + "refund-5-0", + "refund-5-1", + "refund-5-2", + "refund-5-3", + "refund-5-4", + "refund-5-5", + "refund-6-0", + "refund-6-1", + "refund-6-2", + "refund-6-3", + "refund-6-4", + "refund-6-5", + "refund-7-0", + "refund-7-1", + "refund-7-2", + "refund-7-3", + "refund-7-4", + "refund-7-5" + ], + "counts": { + "baseline_failed": 48 + }, + "description": "Authored refund policy.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": true, + "has_known_gaps": false, + "id": "refund", + "scope": "contract" + }, + { + "assessment": "untested", + "case_ids": [], + "counts": {}, + "description": "An inventoried untested obligation.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "refund-untested", + "scope": "contract" + } + ], + "preservation_rate": null, + "preserved_controls": 0, + "total": 48 + }, + "validator_id": "controlled/refund/reject-all/1" + }, + "refund/reject-valid-variation": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "completion_status" + ], + "id": "refund-0-0", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "ccd37bbbe861aac51fb3417c8875bc16a79ad8b1b15650bfd3bd93db012c3e1a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "completion_status" + ], + "id": "refund-0-1", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "7d6365821a41313b32aeb27bcd64165be674007e585cc2017f8e9062cc50132a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "completion_status" + ], + "id": "refund-0-2", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "eb15248acc9779f09f4aee4e03341db85a5ee766d91be39efe6502380d99062a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "receipt_request" + ], + "id": "refund-0-3", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "receipt_request" + ] + }, + "variant_digest": "3949f9298e5f5649e74f83a1d8e62d96c5b64edff8f9378df92b8085684382d4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [], + "id": "refund-0-4", + "obligation": "refund", + "outcome": "regressed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "wording" + ] + }, + "variant_digest": "bcddd8511b297270f4047551ac9e6640c6771c2ef31edce71189059ebdf5ec57" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [], + "id": "refund-0-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "4e408c96e8c13018a9b1bdfc04b38ff2c68a8a3122f83af9a253565b0538d3fc" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "completion_status" + ], + "id": "refund-1-0", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "6a9d900f69ef9075dbce2d7d04a8709ae7ff9029bc447b683e283f89f303731a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "completion_status" + ], + "id": "refund-1-1", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "74c2c49200a3a03c4c59aea9beae887cb3ccee6c2a0a2ba9ac3f2c6f1d5768ec" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "completion_status" + ], + "id": "refund-1-2", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "c3c21448ab0285666f840c72ce3191a9eb3b6d02f2445327a44f80df8361aadd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "receipt_request" + ], + "id": "refund-1-3", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "receipt_request" + ] + }, + "variant_digest": "bb92aa002539565fd017c122a240a57d93749b7eb3ed8328c812984da8858137" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [], + "id": "refund-1-4", + "obligation": "refund", + "outcome": "regressed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "wording" + ] + }, + "variant_digest": "f2e8ccfca6cfc8d81233ad79e38f131662442f7725b727aa3c80ca285ddc9e7c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [], + "id": "refund-1-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "31344474c0e8afd5345be5cd26a211de7a50a32d0e37bf34f94d7d6ccec738eb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "completion_status" + ], + "id": "refund-2-0", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "54a66c0bb0365e4877b306eae4d66407a32ef33d3890461a23be376147fac726" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "completion_status" + ], + "id": "refund-2-1", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "32194e2df8816051ebb680eedd4e6c99f439f712454960574bfcacb0c22e508f" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "completion_status" + ], + "id": "refund-2-2", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "134e295e6b5d7cda0f4c2a06f4334f5204a84edd051d54a1a78abc3531dba904" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "receipt_request" + ], + "id": "refund-2-3", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "receipt_request" + ] + }, + "variant_digest": "2fc4b15d440d00482e36a499432d6d72c43a55ff64441550d9f02e3ac0ad8d26" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [], + "id": "refund-2-4", + "obligation": "refund", + "outcome": "regressed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "wording" + ] + }, + "variant_digest": "7e019a9656e1cae92ebb143b60725822f3712f715297f9394a0f79046f1af1d2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [], + "id": "refund-2-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "a38bc25f3cb16d0b3c704a0b022794e7521c4658f52b6565316bb4658c14dd1c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "completion_status" + ], + "id": "refund-3-0", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "e7647198864504f2e9e12aacd644c263255cef198d13f952982f4c3e15aea2bd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "completion_status" + ], + "id": "refund-3-1", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "76cb0f8403a05996f178ad64163962622a145090a6f4bb07b1b55a3fd03094fa" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "completion_status" + ], + "id": "refund-3-2", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "ed6aed85348bf39def43220afc8ffd43cb41fc50dfb3cbc4a8e5ce45b81db5ac" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "receipt_request" + ], + "id": "refund-3-3", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "receipt_request" + ] + }, + "variant_digest": "e52d125362843d3d44f14e3668ebe62089654de977049b3dcad5352518000dfd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [], + "id": "refund-3-4", + "obligation": "refund", + "outcome": "regressed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "wording" + ] + }, + "variant_digest": "d51b5977bf6366c7e7e9aefaa56db51aef306321b28a25ed75ee55537536240d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [], + "id": "refund-3-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0c2efddae9167df5d220b997e46801498da20707d4af962b6cbdd7e3e967e4b0" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "completion_status" + ], + "id": "refund-4-0", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "1d75a8187e9866f155a596451a99b0ef662fc8bd47468cec8132232c6927284d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "completion_status" + ], + "id": "refund-4-1", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "42c352cde2acf7abea3b16efb2b4eeaf8e64a1fb0550ef0593169add982fe25d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "completion_status" + ], + "id": "refund-4-2", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "39ae7f239158badba8909e1d2647a5e538f1ecbaa275777742f8a5dcefead5e6" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "receipt_request" + ], + "id": "refund-4-3", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "receipt_request" + ] + }, + "variant_digest": "280757cbd993d059788757ec2b3089139d8c77fe3c5c532cc838bcbd647df35c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [], + "id": "refund-4-4", + "obligation": "refund", + "outcome": "regressed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "wording" + ] + }, + "variant_digest": "5000f458a8323a9c05c1dc32358e0e1475606a8dbfc3dc09030ac4791c1a1b18" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [], + "id": "refund-4-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "75d40912f0d6f3b9d3ddf46ff1995665fc4abba879060050405d594bf53c4201" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "completion_status" + ], + "id": "refund-5-0", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "96b6a87a001bd42d0df33547bac4961c1d13612525accb66637b28d4a515c9ab" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "completion_status" + ], + "id": "refund-5-1", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "6c97be4ee3d68ec0b3bc39d85274b56f5d5cec7a2c6c105853a0eeb287da6149" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "completion_status" + ], + "id": "refund-5-2", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "ac589b53d44f747cb664fcd573e5cb584521c654d209fc192b848ac4d1ae7378" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "receipt_request" + ], + "id": "refund-5-3", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "receipt_request" + ] + }, + "variant_digest": "e4263b7597fbee212263ed32946494cc65ca01235b3500f7976496dcafa90740" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [], + "id": "refund-5-4", + "obligation": "refund", + "outcome": "regressed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "wording" + ] + }, + "variant_digest": "a02cd1667f690a76af1b1719d8d18fcd4868e7750bd73264c1848cdf246d31e1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [], + "id": "refund-5-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5f6a0e34ff576a0ebc29af01225a2a7098920857c85199231bfcce4b3640ffb5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "completion_status" + ], + "id": "refund-6-0", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "31350a4758ede02f95dec893447b133494a2aacbf3d4c716f0cc16768d34c934" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "completion_status" + ], + "id": "refund-6-1", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "00ae197ceab0f1a201f556841cb42de0ee5f0e19c76f01477668b26b0d40154d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "completion_status" + ], + "id": "refund-6-2", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "e24387dae93302876c669696b12600f751aec9d39a605c6b285de4460ce1048a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "receipt_request" + ], + "id": "refund-6-3", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "receipt_request" + ] + }, + "variant_digest": "ad00a18a0839d4dc6a7387b6e0cc112257922597feeeea56242bbe3d6bc22be9" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [], + "id": "refund-6-4", + "obligation": "refund", + "outcome": "regressed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "wording" + ] + }, + "variant_digest": "11c7484e3401b23a072b73587705d50fcb50229566e11e2f3da54f87c4c57ccd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [], + "id": "refund-6-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "131d4656c82f6ce43a8e50a3e6ec46bcd41e5742afb6b1fb087da19e20d75c72" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "completion_status" + ], + "id": "refund-7-0", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "8c48ec41dd6d04dabc02ff2c69afff046389d4b74d66036cfeb392a3311408da" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "completion_status" + ], + "id": "refund-7-1", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "59776e3cb3b7d053fdc756923213d545c00a861ba319018c169932ec06059715" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "completion_status" + ], + "id": "refund-7-2", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "completion_status" + ] + }, + "variant_digest": "ecf3771059b7d4db065532b54e2be9b7d0ce69c2c9c7b5d1758b7853c6b98af4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "receipt_request" + ], + "id": "refund-7-3", + "obligation": "refund", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "receipt_request" + ] + }, + "variant_digest": "be5a55ceefe275822757e621b44986822d174567af2eb9af68a57802ed45c15a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [], + "id": "refund-7-4", + "obligation": "refund", + "outcome": "regressed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "wording" + ] + }, + "variant_digest": "0167ec43674a8508cc13432dd4c7a3129f5fc2656d54fe36c8ebd24bc4eade46" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [], + "id": "refund-7-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b4e93f8e27c26696b6260c481d2dcffa7d5ed711ac8562f20566c6576b92d51f" + } + ], + "corpus_digest": "57e293e2ced394347abd60be88a272cf04f7857bc07a906241534db4cd76156b", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Authored refund policy.", + "id": "refund", + "scope": "contract" + }, + { + "description": "An inventoried untested obligation.", + "id": "refund-untested", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "detected": 32, + "preserved": 8, + "regressed": 8 + }, + "detected_faults": 32, + "detection_score": 1.0, + "eligible_controls": 16, + "eligible_faults": 32, + "obligations": [ + { + "assessment": "known_gaps", + "case_ids": [ + "refund-0-0", + "refund-0-1", + "refund-0-2", + "refund-0-3", + "refund-0-4", + "refund-0-5", + "refund-1-0", + "refund-1-1", + "refund-1-2", + "refund-1-3", + "refund-1-4", + "refund-1-5", + "refund-2-0", + "refund-2-1", + "refund-2-2", + "refund-2-3", + "refund-2-4", + "refund-2-5", + "refund-3-0", + "refund-3-1", + "refund-3-2", + "refund-3-3", + "refund-3-4", + "refund-3-5", + "refund-4-0", + "refund-4-1", + "refund-4-2", + "refund-4-3", + "refund-4-4", + "refund-4-5", + "refund-5-0", + "refund-5-1", + "refund-5-2", + "refund-5-3", + "refund-5-4", + "refund-5-5", + "refund-6-0", + "refund-6-1", + "refund-6-2", + "refund-6-3", + "refund-6-4", + "refund-6-5", + "refund-7-0", + "refund-7-1", + "refund-7-2", + "refund-7-3", + "refund-7-4", + "refund-7-5" + ], + "counts": { + "detected": 32, + "preserved": 8, + "regressed": 8 + }, + "description": "Authored refund policy.", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": true, + "has_incomplete_evidence": false, + "has_known_gaps": true, + "id": "refund", + "scope": "contract" + }, + { + "assessment": "untested", + "case_ids": [], + "counts": {}, + "description": "An inventoried untested obligation.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "refund-untested", + "scope": "contract" + } + ], + "preservation_rate": 0.5, + "preserved_controls": 8, + "total": 48 + }, + "validator_id": "controlled/refund/reject-valid-variation/1" + }, + "refund/unknown-on-fault": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "completion_status" + ], + "id": "refund-0-0", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "ccd37bbbe861aac51fb3417c8875bc16a79ad8b1b15650bfd3bd93db012c3e1a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "completion_status" + ], + "id": "refund-0-1", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "7d6365821a41313b32aeb27bcd64165be674007e585cc2017f8e9062cc50132a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "completion_status" + ], + "id": "refund-0-2", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "eb15248acc9779f09f4aee4e03341db85a5ee766d91be39efe6502380d99062a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "receipt_request" + ], + "id": "refund-0-3", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "3949f9298e5f5649e74f83a1d8e62d96c5b64edff8f9378df92b8085684382d4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [], + "id": "refund-0-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "bcddd8511b297270f4047551ac9e6640c6771c2ef31edce71189059ebdf5ec57" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [], + "id": "refund-0-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "4e408c96e8c13018a9b1bdfc04b38ff2c68a8a3122f83af9a253565b0538d3fc" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "completion_status" + ], + "id": "refund-1-0", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "6a9d900f69ef9075dbce2d7d04a8709ae7ff9029bc447b683e283f89f303731a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "completion_status" + ], + "id": "refund-1-1", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "74c2c49200a3a03c4c59aea9beae887cb3ccee6c2a0a2ba9ac3f2c6f1d5768ec" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "completion_status" + ], + "id": "refund-1-2", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "c3c21448ab0285666f840c72ce3191a9eb3b6d02f2445327a44f80df8361aadd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "receipt_request" + ], + "id": "refund-1-3", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "bb92aa002539565fd017c122a240a57d93749b7eb3ed8328c812984da8858137" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [], + "id": "refund-1-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f2e8ccfca6cfc8d81233ad79e38f131662442f7725b727aa3c80ca285ddc9e7c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [], + "id": "refund-1-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "31344474c0e8afd5345be5cd26a211de7a50a32d0e37bf34f94d7d6ccec738eb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "completion_status" + ], + "id": "refund-2-0", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "54a66c0bb0365e4877b306eae4d66407a32ef33d3890461a23be376147fac726" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "completion_status" + ], + "id": "refund-2-1", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "32194e2df8816051ebb680eedd4e6c99f439f712454960574bfcacb0c22e508f" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "completion_status" + ], + "id": "refund-2-2", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "134e295e6b5d7cda0f4c2a06f4334f5204a84edd051d54a1a78abc3531dba904" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "receipt_request" + ], + "id": "refund-2-3", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "2fc4b15d440d00482e36a499432d6d72c43a55ff64441550d9f02e3ac0ad8d26" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [], + "id": "refund-2-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "7e019a9656e1cae92ebb143b60725822f3712f715297f9394a0f79046f1af1d2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [], + "id": "refund-2-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "a38bc25f3cb16d0b3c704a0b022794e7521c4658f52b6565316bb4658c14dd1c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "completion_status" + ], + "id": "refund-3-0", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "e7647198864504f2e9e12aacd644c263255cef198d13f952982f4c3e15aea2bd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "completion_status" + ], + "id": "refund-3-1", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "76cb0f8403a05996f178ad64163962622a145090a6f4bb07b1b55a3fd03094fa" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "completion_status" + ], + "id": "refund-3-2", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "ed6aed85348bf39def43220afc8ffd43cb41fc50dfb3cbc4a8e5ce45b81db5ac" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "receipt_request" + ], + "id": "refund-3-3", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "e52d125362843d3d44f14e3668ebe62089654de977049b3dcad5352518000dfd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [], + "id": "refund-3-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "d51b5977bf6366c7e7e9aefaa56db51aef306321b28a25ed75ee55537536240d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [], + "id": "refund-3-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0c2efddae9167df5d220b997e46801498da20707d4af962b6cbdd7e3e967e4b0" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "completion_status" + ], + "id": "refund-4-0", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "1d75a8187e9866f155a596451a99b0ef662fc8bd47468cec8132232c6927284d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "completion_status" + ], + "id": "refund-4-1", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "42c352cde2acf7abea3b16efb2b4eeaf8e64a1fb0550ef0593169add982fe25d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "completion_status" + ], + "id": "refund-4-2", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "39ae7f239158badba8909e1d2647a5e538f1ecbaa275777742f8a5dcefead5e6" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "receipt_request" + ], + "id": "refund-4-3", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "280757cbd993d059788757ec2b3089139d8c77fe3c5c532cc838bcbd647df35c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [], + "id": "refund-4-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5000f458a8323a9c05c1dc32358e0e1475606a8dbfc3dc09030ac4791c1a1b18" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [], + "id": "refund-4-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "75d40912f0d6f3b9d3ddf46ff1995665fc4abba879060050405d594bf53c4201" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "completion_status" + ], + "id": "refund-5-0", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "96b6a87a001bd42d0df33547bac4961c1d13612525accb66637b28d4a515c9ab" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "completion_status" + ], + "id": "refund-5-1", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "6c97be4ee3d68ec0b3bc39d85274b56f5d5cec7a2c6c105853a0eeb287da6149" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "completion_status" + ], + "id": "refund-5-2", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "ac589b53d44f747cb664fcd573e5cb584521c654d209fc192b848ac4d1ae7378" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "receipt_request" + ], + "id": "refund-5-3", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "e4263b7597fbee212263ed32946494cc65ca01235b3500f7976496dcafa90740" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [], + "id": "refund-5-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "a02cd1667f690a76af1b1719d8d18fcd4868e7750bd73264c1848cdf246d31e1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [], + "id": "refund-5-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5f6a0e34ff576a0ebc29af01225a2a7098920857c85199231bfcce4b3640ffb5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "completion_status" + ], + "id": "refund-6-0", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "31350a4758ede02f95dec893447b133494a2aacbf3d4c716f0cc16768d34c934" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "completion_status" + ], + "id": "refund-6-1", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "00ae197ceab0f1a201f556841cb42de0ee5f0e19c76f01477668b26b0d40154d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "completion_status" + ], + "id": "refund-6-2", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "e24387dae93302876c669696b12600f751aec9d39a605c6b285de4460ce1048a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "receipt_request" + ], + "id": "refund-6-3", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "ad00a18a0839d4dc6a7387b6e0cc112257922597feeeea56242bbe3d6bc22be9" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [], + "id": "refund-6-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "11c7484e3401b23a072b73587705d50fcb50229566e11e2f3da54f87c4c57ccd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [], + "id": "refund-6-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "131d4656c82f6ce43a8e50a3e6ec46bcd41e5742afb6b1fb087da19e20d75c72" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "completion_status" + ], + "id": "refund-7-0", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "8c48ec41dd6d04dabc02ff2c69afff046389d4b74d66036cfeb392a3311408da" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "completion_status" + ], + "id": "refund-7-1", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "59776e3cb3b7d053fdc756923213d545c00a861ba319018c169932ec06059715" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "completion_status" + ], + "id": "refund-7-2", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "ecf3771059b7d4db065532b54e2be9b7d0ce69c2c9c7b5d1758b7853c6b98af4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "receipt_request" + ], + "id": "refund-7-3", + "obligation": "refund", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "be5a55ceefe275822757e621b44986822d174567af2eb9af68a57802ed45c15a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [], + "id": "refund-7-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0167ec43674a8508cc13432dd4c7a3129f5fc2656d54fe36c8ebd24bc4eade46" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [], + "id": "refund-7-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b4e93f8e27c26696b6260c481d2dcffa7d5ed711ac8562f20566c6576b92d51f" + } + ], + "corpus_digest": "57e293e2ced394347abd60be88a272cf04f7857bc07a906241534db4cd76156b", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Authored refund policy.", + "id": "refund", + "scope": "contract" + }, + { + "description": "An inventoried untested obligation.", + "id": "refund-untested", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 16, + "undetermined": 32 + }, + "detected_faults": 0, + "detection_score": 0.0, + "eligible_controls": 16, + "eligible_faults": 32, + "obligations": [ + { + "assessment": "incomplete_evidence", + "case_ids": [ + "refund-0-0", + "refund-0-1", + "refund-0-2", + "refund-0-3", + "refund-0-4", + "refund-0-5", + "refund-1-0", + "refund-1-1", + "refund-1-2", + "refund-1-3", + "refund-1-4", + "refund-1-5", + "refund-2-0", + "refund-2-1", + "refund-2-2", + "refund-2-3", + "refund-2-4", + "refund-2-5", + "refund-3-0", + "refund-3-1", + "refund-3-2", + "refund-3-3", + "refund-3-4", + "refund-3-5", + "refund-4-0", + "refund-4-1", + "refund-4-2", + "refund-4-3", + "refund-4-4", + "refund-4-5", + "refund-5-0", + "refund-5-1", + "refund-5-2", + "refund-5-3", + "refund-5-4", + "refund-5-5", + "refund-6-0", + "refund-6-1", + "refund-6-2", + "refund-6-3", + "refund-6-4", + "refund-6-5", + "refund-7-0", + "refund-7-1", + "refund-7-2", + "refund-7-3", + "refund-7-4", + "refund-7-5" + ], + "counts": { + "preserved": 16, + "undetermined": 32 + }, + "description": "Authored refund policy.", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": true, + "has_incomplete_evidence": true, + "has_known_gaps": false, + "id": "refund", + "scope": "contract" + }, + { + "assessment": "untested", + "case_ids": [], + "counts": {}, + "description": "An inventoried untested obligation.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "refund-untested", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 16, + "total": 48 + }, + "validator_id": "controlled/refund/unknown-on-fault/1" + }, + "refund/wrong-diagnostic-ID": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "completion_status" + ], + "id": "refund-0-0", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "ccd37bbbe861aac51fb3417c8875bc16a79ad8b1b15650bfd3bd93db012c3e1a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "completion_status" + ], + "id": "refund-0-1", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "7d6365821a41313b32aeb27bcd64165be674007e585cc2017f8e9062cc50132a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "completion_status" + ], + "id": "refund-0-2", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "eb15248acc9779f09f4aee4e03341db85a5ee766d91be39efe6502380d99062a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [ + "receipt_request" + ], + "id": "refund-0-3", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "3949f9298e5f5649e74f83a1d8e62d96c5b64edff8f9378df92b8085684382d4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [], + "id": "refund-0-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "bcddd8511b297270f4047551ac9e6640c6771c2ef31edce71189059ebdf5ec57" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b6366fbe3416c6ba81d6421ab0b3b225214f4e0b82ccdfd458a737b18754c987", + "expected": [], + "id": "refund-0-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "4e408c96e8c13018a9b1bdfc04b38ff2c68a8a3122f83af9a253565b0538d3fc" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "completion_status" + ], + "id": "refund-1-0", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "6a9d900f69ef9075dbce2d7d04a8709ae7ff9029bc447b683e283f89f303731a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "completion_status" + ], + "id": "refund-1-1", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "74c2c49200a3a03c4c59aea9beae887cb3ccee6c2a0a2ba9ac3f2c6f1d5768ec" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "completion_status" + ], + "id": "refund-1-2", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "c3c21448ab0285666f840c72ce3191a9eb3b6d02f2445327a44f80df8361aadd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [ + "receipt_request" + ], + "id": "refund-1-3", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "bb92aa002539565fd017c122a240a57d93749b7eb3ed8328c812984da8858137" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [], + "id": "refund-1-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f2e8ccfca6cfc8d81233ad79e38f131662442f7725b727aa3c80ca285ddc9e7c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "5d99126e23fcaec8632fbd30c026b86458d68932599e5de7fac164d71d01f35c", + "expected": [], + "id": "refund-1-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "31344474c0e8afd5345be5cd26a211de7a50a32d0e37bf34f94d7d6ccec738eb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "completion_status" + ], + "id": "refund-2-0", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "54a66c0bb0365e4877b306eae4d66407a32ef33d3890461a23be376147fac726" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "completion_status" + ], + "id": "refund-2-1", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "32194e2df8816051ebb680eedd4e6c99f439f712454960574bfcacb0c22e508f" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "completion_status" + ], + "id": "refund-2-2", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "134e295e6b5d7cda0f4c2a06f4334f5204a84edd051d54a1a78abc3531dba904" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [ + "receipt_request" + ], + "id": "refund-2-3", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "2fc4b15d440d00482e36a499432d6d72c43a55ff64441550d9f02e3ac0ad8d26" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [], + "id": "refund-2-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "7e019a9656e1cae92ebb143b60725822f3712f715297f9394a0f79046f1af1d2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4faf5d1e2251c5d57dd652d928bb1e4f1325b13afc223a26a62a2b05c4f81a64", + "expected": [], + "id": "refund-2-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "a38bc25f3cb16d0b3c704a0b022794e7521c4658f52b6565316bb4658c14dd1c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "completion_status" + ], + "id": "refund-3-0", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "e7647198864504f2e9e12aacd644c263255cef198d13f952982f4c3e15aea2bd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "completion_status" + ], + "id": "refund-3-1", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "76cb0f8403a05996f178ad64163962622a145090a6f4bb07b1b55a3fd03094fa" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "completion_status" + ], + "id": "refund-3-2", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "ed6aed85348bf39def43220afc8ffd43cb41fc50dfb3cbc4a8e5ce45b81db5ac" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [ + "receipt_request" + ], + "id": "refund-3-3", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "e52d125362843d3d44f14e3668ebe62089654de977049b3dcad5352518000dfd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [], + "id": "refund-3-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "d51b5977bf6366c7e7e9aefaa56db51aef306321b28a25ed75ee55537536240d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "75c2a34a54330c2ab01293f36c8b3d2cdd2a3ac95ce46be48cf97e57cd168e50", + "expected": [], + "id": "refund-3-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0c2efddae9167df5d220b997e46801498da20707d4af962b6cbdd7e3e967e4b0" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "completion_status" + ], + "id": "refund-4-0", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "1d75a8187e9866f155a596451a99b0ef662fc8bd47468cec8132232c6927284d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "completion_status" + ], + "id": "refund-4-1", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "42c352cde2acf7abea3b16efb2b4eeaf8e64a1fb0550ef0593169add982fe25d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "completion_status" + ], + "id": "refund-4-2", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "39ae7f239158badba8909e1d2647a5e538f1ecbaa275777742f8a5dcefead5e6" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [ + "receipt_request" + ], + "id": "refund-4-3", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "280757cbd993d059788757ec2b3089139d8c77fe3c5c532cc838bcbd647df35c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [], + "id": "refund-4-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5000f458a8323a9c05c1dc32358e0e1475606a8dbfc3dc09030ac4791c1a1b18" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a6044469d9ef0d0885b2e996523ec9a98205bf32c412a478534c5319db3f2494", + "expected": [], + "id": "refund-4-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "75d40912f0d6f3b9d3ddf46ff1995665fc4abba879060050405d594bf53c4201" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "completion_status" + ], + "id": "refund-5-0", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "96b6a87a001bd42d0df33547bac4961c1d13612525accb66637b28d4a515c9ab" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "completion_status" + ], + "id": "refund-5-1", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "6c97be4ee3d68ec0b3bc39d85274b56f5d5cec7a2c6c105853a0eeb287da6149" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "completion_status" + ], + "id": "refund-5-2", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "ac589b53d44f747cb664fcd573e5cb584521c654d209fc192b848ac4d1ae7378" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [ + "receipt_request" + ], + "id": "refund-5-3", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "e4263b7597fbee212263ed32946494cc65ca01235b3500f7976496dcafa90740" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [], + "id": "refund-5-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "a02cd1667f690a76af1b1719d8d18fcd4868e7750bd73264c1848cdf246d31e1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "a176807091d644519f152d91d756dfc7a3649a53d38735f511217a854fc168fa", + "expected": [], + "id": "refund-5-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5f6a0e34ff576a0ebc29af01225a2a7098920857c85199231bfcce4b3640ffb5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "completion_status" + ], + "id": "refund-6-0", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "31350a4758ede02f95dec893447b133494a2aacbf3d4c716f0cc16768d34c934" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "completion_status" + ], + "id": "refund-6-1", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "00ae197ceab0f1a201f556841cb42de0ee5f0e19c76f01477668b26b0d40154d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "completion_status" + ], + "id": "refund-6-2", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "e24387dae93302876c669696b12600f751aec9d39a605c6b285de4460ce1048a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [ + "receipt_request" + ], + "id": "refund-6-3", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "ad00a18a0839d4dc6a7387b6e0cc112257922597feeeea56242bbe3d6bc22be9" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [], + "id": "refund-6-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "11c7484e3401b23a072b73587705d50fcb50229566e11e2f3da54f87c4c57ccd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "f63e97e575045f06020b80fb02aa1cb44759af77024ad6adf0f261d1e51cc8f0", + "expected": [], + "id": "refund-6-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "131d4656c82f6ce43a8e50a3e6ec46bcd41e5742afb6b1fb087da19e20d75c72" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "completion_status" + ], + "id": "refund-7-0", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "8c48ec41dd6d04dabc02ff2c69afff046389d4b74d66036cfeb392a3311408da" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "completion_status" + ], + "id": "refund-7-1", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "59776e3cb3b7d053fdc756923213d545c00a861ba319018c169932ec06059715" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "completion_status" + ], + "id": "refund-7-2", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "ecf3771059b7d4db065532b54e2be9b7d0ce69c2c9c7b5d1758b7853c6b98af4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [ + "receipt_request" + ], + "id": "refund-7-3", + "obligation": "refund", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "be5a55ceefe275822757e621b44986822d174567af2eb9af68a57802ed45c15a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [], + "id": "refund-7-4", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0167ec43674a8508cc13432dd4c7a3129f5fc2656d54fe36c8ebd24bc4eade46" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "2c88f4b73684a3510f8e2ab0ab47a454b04c791aa6895b7925a371732abfa883", + "expected": [], + "id": "refund-7-5", + "obligation": "refund", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b4e93f8e27c26696b6260c481d2dcffa7d5ed711ac8562f20566c6576b92d51f" + } + ], + "corpus_digest": "57e293e2ced394347abd60be88a272cf04f7857bc07a906241534db4cd76156b", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Authored refund policy.", + "id": "refund", + "scope": "contract" + }, + { + "description": "An inventoried untested obligation.", + "id": "refund-untested", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 16, + "unattributed_rejection": 32 + }, + "detected_faults": 0, + "detection_score": 0.0, + "eligible_controls": 16, + "eligible_faults": 32, + "obligations": [ + { + "assessment": "known_gaps", + "case_ids": [ + "refund-0-0", + "refund-0-1", + "refund-0-2", + "refund-0-3", + "refund-0-4", + "refund-0-5", + "refund-1-0", + "refund-1-1", + "refund-1-2", + "refund-1-3", + "refund-1-4", + "refund-1-5", + "refund-2-0", + "refund-2-1", + "refund-2-2", + "refund-2-3", + "refund-2-4", + "refund-2-5", + "refund-3-0", + "refund-3-1", + "refund-3-2", + "refund-3-3", + "refund-3-4", + "refund-3-5", + "refund-4-0", + "refund-4-1", + "refund-4-2", + "refund-4-3", + "refund-4-4", + "refund-4-5", + "refund-5-0", + "refund-5-1", + "refund-5-2", + "refund-5-3", + "refund-5-4", + "refund-5-5", + "refund-6-0", + "refund-6-1", + "refund-6-2", + "refund-6-3", + "refund-6-4", + "refund-6-5", + "refund-7-0", + "refund-7-1", + "refund-7-2", + "refund-7-3", + "refund-7-4", + "refund-7-5" + ], + "counts": { + "preserved": 16, + "unattributed_rejection": 32 + }, + "description": "Authored refund policy.", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": true, + "has_incomplete_evidence": false, + "has_known_gaps": true, + "id": "refund", + "scope": "contract" + }, + { + "assessment": "untested", + "case_ids": [], + "counts": {}, + "description": "An inventoried untested obligation.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "refund-untested", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 16, + "total": 48 + }, + "validator_id": "controlled/refund/wrong-diagnostic-ID/1" + }, + "reservation/accept-all": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "quantity" + ], + "id": "reservation-0-0", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "1fc600ada97a6695444df534d186265e57f62361511d24d15320277a8ec8d7d6" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "quantity" + ], + "id": "reservation-0-1", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "7374d2c95560fcaff9c4b9363e8fb8850f14b17619774bf41e5a8a5cf5e6ed67" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "quantity" + ], + "id": "reservation-0-2", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "071bbd02833d19d167739460785886f9219373f17f67a365bd8d7f211f87a2a4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "resource" + ], + "id": "reservation-0-3", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "65c361c5af1052ab083d5ec8e3ab8e01ba9711f37a206e3dbf73fd455adb214b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [], + "id": "reservation-0-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "e5f35c86979707b0033959bfb0dce2b3a1278e00ac7b3ddf1715a665b9e0c0f7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [], + "id": "reservation-0-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0e025e5f737a1a6da2b67bb77ee838e89643353618c82a149b8d67382add73a3" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "quantity" + ], + "id": "reservation-1-0", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "a31b0b40c4a2731c4d2301379229ac52601b53b9227406f93d5552e4b009a771" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "quantity" + ], + "id": "reservation-1-1", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "6649c4dccdf8da43ccea052a258c2774a22d61af8759b92a99919f33672c4b88" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "quantity" + ], + "id": "reservation-1-2", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "cc9d75acdf55d8ea8434d5d68b300393fb49fb1788a276071b85d1303ee8543b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "resource" + ], + "id": "reservation-1-3", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0a201dcd0fef1c32aefd5952616ebd57b63db5221c5e87897ff23537662831f4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [], + "id": "reservation-1-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "c5708bc72b664a3860bef73291bf3ceebd779a5c152453912301ddbc8705e53e" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [], + "id": "reservation-1-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f9c77126ba4d34c653c59728acbd83f396109df4e664d20d72d36a3f081a5c84" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "quantity" + ], + "id": "reservation-2-0", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "90896fef73bf1e7b5723e5d9ab7671da1af612939906be955acc8b0c4728644d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "quantity" + ], + "id": "reservation-2-1", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b01e41e049d1e0992c1688ff69d9cc7efccae519cea8c92353c56bfcf9ea73c2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "quantity" + ], + "id": "reservation-2-2", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "88404cc0ee9bf47f78a6000515c1744dad0a2e6977c607e6066b74c936de54c7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "resource" + ], + "id": "reservation-2-3", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0626666411f02ccfb32fa2955f027446488fb0f2b0ca04f16f82b913f5e9ae78" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [], + "id": "reservation-2-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b0eb9a4f3a0a1f1cc87d53d99c5219107c8f28f7f333235143a517cc62ce5afb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [], + "id": "reservation-2-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "c345f2f38bf72e218be04f3d6cb4307bc07c8fbef6520164458ee92b907973e1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "quantity" + ], + "id": "reservation-3-0", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "6a5668208d87825b0bc2f866471ef60278d10c040d8cdab3c95c304c27ab4949" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "quantity" + ], + "id": "reservation-3-1", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "2c71639065c74db9d2f6ca8db87db4a83d9fb1d6e1384ae79cb8daccb5dc3c79" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "quantity" + ], + "id": "reservation-3-2", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f1e4b5db85138fb919f24797e266d2032f71b6768780a82754aef1c753460594" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "resource" + ], + "id": "reservation-3-3", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "4b27551c1f51c816b65831f493826adc1796b98aec66977a047cff97ea200ce7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [], + "id": "reservation-3-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "cd5e432fece013c8fb0abe46f8f93cf8e292ca8104551a3f2e6c5a0a7ea61c04" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [], + "id": "reservation-3-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "185a339b16b268514b32eb70af1a0a13edee14a2e4f541c8fd3732d539622c24" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "quantity" + ], + "id": "reservation-4-0", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5878b509a9e6fd0681485727293077caabaf818e727241485d1a627710e33e06" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "quantity" + ], + "id": "reservation-4-1", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "ec67ee408d81f95b636d0f6e69f60cac5da67167e0a6cc5c710778b8d3106ecb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "quantity" + ], + "id": "reservation-4-2", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "351ea66efae7531aaef80ca3dab37066634614c9d09779728cd4b1804c068c4c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "resource" + ], + "id": "reservation-4-3", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "85ad70c43500fd9680301e4b2a0a0437e62e776e4c6e632fffa7b29fe2be54d1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [], + "id": "reservation-4-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b4cb6ec94bfbd656632fbec156f295258787fe64a99af7a2d1884ef2a2e2bb68" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [], + "id": "reservation-4-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "c26f6e66af343485dd6a1a95ff9d5808145c9f2ab0314f51ece7cb983374c992" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "quantity" + ], + "id": "reservation-5-0", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "e22941ec7c7f81e3821f97ac28f881166fcdd8a0453a18e14c6a6ac0ab113e77" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "quantity" + ], + "id": "reservation-5-1", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "73357e37bd70ff319ac3a84892dec5ed64c266b7b3338ac822e3cddf12d129fa" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "quantity" + ], + "id": "reservation-5-2", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "03d413ec27a19883b08aaa7097066a1bd5b13298bebb37ea0cd0ebd8bd467726" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "resource" + ], + "id": "reservation-5-3", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0eef9ad326105740dd04412212cc3bfb33485ffa2a5469c6e6ac30a815462c99" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [], + "id": "reservation-5-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "745b7d1c042e975a28642b758c639abad37b1c14e19c82da038d889e19b2d7a2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [], + "id": "reservation-5-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "07b3f742b38d89d4c720710cf6915cc9b0205627fffa31c6b5f710ceba6b04ad" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "quantity" + ], + "id": "reservation-6-0", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "1282a62f8e53345b39e7f2f143db41fc1de89c2b2984bd8e534202c52b3d917a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "quantity" + ], + "id": "reservation-6-1", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "06fa2d8be6a7e5c221da96bbd086e0849f6d8e5f1463c180907ca5a980a05e05" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "quantity" + ], + "id": "reservation-6-2", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "7fd2fbe4986d289d0c23fb07a18669e85549b4c6672bd141c9ef676e2a38149a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "resource" + ], + "id": "reservation-6-3", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "251c94a4b91b7662837c1e57ae9ddf87fb37db0b1ce37b3ed7e5829b80e41909" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [], + "id": "reservation-6-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "39941c278508db9b6de0b6e3907e7c35167d9abf98c54d1e454dd6c4b2b23699" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [], + "id": "reservation-6-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "541181d0d1b6ae2a12543230218e6c28ae3340bf5ec88c7473765c2978a19917" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "quantity" + ], + "id": "reservation-7-0", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f83b180009a1a5b7826b21e13a9d743dee4f6069ef3f4372c432f91da1650846" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "quantity" + ], + "id": "reservation-7-1", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5746690e63e142b29a4f5d5b1af20c5a678939ea825275e0fa96f18f1607e145" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "quantity" + ], + "id": "reservation-7-2", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "a3e719713ce65296687fe8cb3c8e296b1bd5d917ea305d09baedc47ba2ffefd1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "resource" + ], + "id": "reservation-7-3", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5b86ae17f6e473cadde2309412a1fc298e0c140c415816447a4bdf47d36df0ba" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [], + "id": "reservation-7-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "6e41a0ba04b3b4b95382a08b9f4426823762f18b270ee28226ec4df9c379fdf3" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [], + "id": "reservation-7-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f485010cb153d587e3f85174a65866fb804d90fbddfb5ca56505bb42ef26c656" + } + ], + "corpus_digest": "2997882c76109515e6f0e50c928f5f780dcad160d57ff300aebe292275f69e5e", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Authored reservation policy.", + "id": "reservation", + "scope": "contract" + }, + { + "description": "An inventoried untested obligation.", + "id": "reservation-untested", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 16, + "survived": 32 + }, + "detected_faults": 0, + "detection_score": 0.0, + "eligible_controls": 16, + "eligible_faults": 32, + "obligations": [ + { + "assessment": "known_gaps", + "case_ids": [ + "reservation-0-0", + "reservation-0-1", + "reservation-0-2", + "reservation-0-3", + "reservation-0-4", + "reservation-0-5", + "reservation-1-0", + "reservation-1-1", + "reservation-1-2", + "reservation-1-3", + "reservation-1-4", + "reservation-1-5", + "reservation-2-0", + "reservation-2-1", + "reservation-2-2", + "reservation-2-3", + "reservation-2-4", + "reservation-2-5", + "reservation-3-0", + "reservation-3-1", + "reservation-3-2", + "reservation-3-3", + "reservation-3-4", + "reservation-3-5", + "reservation-4-0", + "reservation-4-1", + "reservation-4-2", + "reservation-4-3", + "reservation-4-4", + "reservation-4-5", + "reservation-5-0", + "reservation-5-1", + "reservation-5-2", + "reservation-5-3", + "reservation-5-4", + "reservation-5-5", + "reservation-6-0", + "reservation-6-1", + "reservation-6-2", + "reservation-6-3", + "reservation-6-4", + "reservation-6-5", + "reservation-7-0", + "reservation-7-1", + "reservation-7-2", + "reservation-7-3", + "reservation-7-4", + "reservation-7-5" + ], + "counts": { + "preserved": 16, + "survived": 32 + }, + "description": "Authored reservation policy.", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": true, + "has_incomplete_evidence": false, + "has_known_gaps": true, + "id": "reservation", + "scope": "contract" + }, + { + "assessment": "untested", + "case_ids": [], + "counts": {}, + "description": "An inventoried untested obligation.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "reservation-untested", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 16, + "total": 48 + }, + "validator_id": "controlled/reservation/accept-all/1" + }, + "reservation/correct": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "quantity" + ], + "id": "reservation-0-0", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "1fc600ada97a6695444df534d186265e57f62361511d24d15320277a8ec8d7d6" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "quantity" + ], + "id": "reservation-0-1", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "7374d2c95560fcaff9c4b9363e8fb8850f14b17619774bf41e5a8a5cf5e6ed67" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "quantity" + ], + "id": "reservation-0-2", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "071bbd02833d19d167739460785886f9219373f17f67a365bd8d7f211f87a2a4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "resource" + ], + "id": "reservation-0-3", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "resource" + ] + }, + "variant_digest": "65c361c5af1052ab083d5ec8e3ab8e01ba9711f37a206e3dbf73fd455adb214b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [], + "id": "reservation-0-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "e5f35c86979707b0033959bfb0dce2b3a1278e00ac7b3ddf1715a665b9e0c0f7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [], + "id": "reservation-0-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0e025e5f737a1a6da2b67bb77ee838e89643353618c82a149b8d67382add73a3" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "quantity" + ], + "id": "reservation-1-0", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "a31b0b40c4a2731c4d2301379229ac52601b53b9227406f93d5552e4b009a771" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "quantity" + ], + "id": "reservation-1-1", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "6649c4dccdf8da43ccea052a258c2774a22d61af8759b92a99919f33672c4b88" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "quantity" + ], + "id": "reservation-1-2", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "cc9d75acdf55d8ea8434d5d68b300393fb49fb1788a276071b85d1303ee8543b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "resource" + ], + "id": "reservation-1-3", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "resource" + ] + }, + "variant_digest": "0a201dcd0fef1c32aefd5952616ebd57b63db5221c5e87897ff23537662831f4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [], + "id": "reservation-1-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "c5708bc72b664a3860bef73291bf3ceebd779a5c152453912301ddbc8705e53e" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [], + "id": "reservation-1-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f9c77126ba4d34c653c59728acbd83f396109df4e664d20d72d36a3f081a5c84" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "quantity" + ], + "id": "reservation-2-0", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "90896fef73bf1e7b5723e5d9ab7671da1af612939906be955acc8b0c4728644d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "quantity" + ], + "id": "reservation-2-1", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "b01e41e049d1e0992c1688ff69d9cc7efccae519cea8c92353c56bfcf9ea73c2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "quantity" + ], + "id": "reservation-2-2", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "88404cc0ee9bf47f78a6000515c1744dad0a2e6977c607e6066b74c936de54c7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "resource" + ], + "id": "reservation-2-3", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "resource" + ] + }, + "variant_digest": "0626666411f02ccfb32fa2955f027446488fb0f2b0ca04f16f82b913f5e9ae78" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [], + "id": "reservation-2-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b0eb9a4f3a0a1f1cc87d53d99c5219107c8f28f7f333235143a517cc62ce5afb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [], + "id": "reservation-2-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "c345f2f38bf72e218be04f3d6cb4307bc07c8fbef6520164458ee92b907973e1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "quantity" + ], + "id": "reservation-3-0", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "6a5668208d87825b0bc2f866471ef60278d10c040d8cdab3c95c304c27ab4949" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "quantity" + ], + "id": "reservation-3-1", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "2c71639065c74db9d2f6ca8db87db4a83d9fb1d6e1384ae79cb8daccb5dc3c79" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "quantity" + ], + "id": "reservation-3-2", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "f1e4b5db85138fb919f24797e266d2032f71b6768780a82754aef1c753460594" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "resource" + ], + "id": "reservation-3-3", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "resource" + ] + }, + "variant_digest": "4b27551c1f51c816b65831f493826adc1796b98aec66977a047cff97ea200ce7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [], + "id": "reservation-3-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "cd5e432fece013c8fb0abe46f8f93cf8e292ca8104551a3f2e6c5a0a7ea61c04" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [], + "id": "reservation-3-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "185a339b16b268514b32eb70af1a0a13edee14a2e4f541c8fd3732d539622c24" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "quantity" + ], + "id": "reservation-4-0", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "5878b509a9e6fd0681485727293077caabaf818e727241485d1a627710e33e06" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "quantity" + ], + "id": "reservation-4-1", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "ec67ee408d81f95b636d0f6e69f60cac5da67167e0a6cc5c710778b8d3106ecb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "quantity" + ], + "id": "reservation-4-2", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "351ea66efae7531aaef80ca3dab37066634614c9d09779728cd4b1804c068c4c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "resource" + ], + "id": "reservation-4-3", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "resource" + ] + }, + "variant_digest": "85ad70c43500fd9680301e4b2a0a0437e62e776e4c6e632fffa7b29fe2be54d1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [], + "id": "reservation-4-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b4cb6ec94bfbd656632fbec156f295258787fe64a99af7a2d1884ef2a2e2bb68" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [], + "id": "reservation-4-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "c26f6e66af343485dd6a1a95ff9d5808145c9f2ab0314f51ece7cb983374c992" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "quantity" + ], + "id": "reservation-5-0", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "e22941ec7c7f81e3821f97ac28f881166fcdd8a0453a18e14c6a6ac0ab113e77" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "quantity" + ], + "id": "reservation-5-1", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "73357e37bd70ff319ac3a84892dec5ed64c266b7b3338ac822e3cddf12d129fa" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "quantity" + ], + "id": "reservation-5-2", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "03d413ec27a19883b08aaa7097066a1bd5b13298bebb37ea0cd0ebd8bd467726" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "resource" + ], + "id": "reservation-5-3", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "resource" + ] + }, + "variant_digest": "0eef9ad326105740dd04412212cc3bfb33485ffa2a5469c6e6ac30a815462c99" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [], + "id": "reservation-5-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "745b7d1c042e975a28642b758c639abad37b1c14e19c82da038d889e19b2d7a2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [], + "id": "reservation-5-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "07b3f742b38d89d4c720710cf6915cc9b0205627fffa31c6b5f710ceba6b04ad" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "quantity" + ], + "id": "reservation-6-0", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "1282a62f8e53345b39e7f2f143db41fc1de89c2b2984bd8e534202c52b3d917a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "quantity" + ], + "id": "reservation-6-1", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "06fa2d8be6a7e5c221da96bbd086e0849f6d8e5f1463c180907ca5a980a05e05" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "quantity" + ], + "id": "reservation-6-2", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "7fd2fbe4986d289d0c23fb07a18669e85549b4c6672bd141c9ef676e2a38149a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "resource" + ], + "id": "reservation-6-3", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "resource" + ] + }, + "variant_digest": "251c94a4b91b7662837c1e57ae9ddf87fb37db0b1ce37b3ed7e5829b80e41909" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [], + "id": "reservation-6-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "39941c278508db9b6de0b6e3907e7c35167d9abf98c54d1e454dd6c4b2b23699" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [], + "id": "reservation-6-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "541181d0d1b6ae2a12543230218e6c28ae3340bf5ec88c7473765c2978a19917" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "quantity" + ], + "id": "reservation-7-0", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "f83b180009a1a5b7826b21e13a9d743dee4f6069ef3f4372c432f91da1650846" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "quantity" + ], + "id": "reservation-7-1", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "5746690e63e142b29a4f5d5b1af20c5a678939ea825275e0fa96f18f1607e145" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "quantity" + ], + "id": "reservation-7-2", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "a3e719713ce65296687fe8cb3c8e296b1bd5d917ea305d09baedc47ba2ffefd1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "resource" + ], + "id": "reservation-7-3", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "resource" + ] + }, + "variant_digest": "5b86ae17f6e473cadde2309412a1fc298e0c140c415816447a4bdf47d36df0ba" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [], + "id": "reservation-7-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "6e41a0ba04b3b4b95382a08b9f4426823762f18b270ee28226ec4df9c379fdf3" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [], + "id": "reservation-7-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f485010cb153d587e3f85174a65866fb804d90fbddfb5ca56505bb42ef26c656" + } + ], + "corpus_digest": "2997882c76109515e6f0e50c928f5f780dcad160d57ff300aebe292275f69e5e", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Authored reservation policy.", + "id": "reservation", + "scope": "contract" + }, + { + "description": "An inventoried untested obligation.", + "id": "reservation-untested", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "detected": 32, + "preserved": 16 + }, + "detected_faults": 32, + "detection_score": 1.0, + "eligible_controls": 16, + "eligible_faults": 32, + "obligations": [ + { + "assessment": "no_failures_observed", + "case_ids": [ + "reservation-0-0", + "reservation-0-1", + "reservation-0-2", + "reservation-0-3", + "reservation-0-4", + "reservation-0-5", + "reservation-1-0", + "reservation-1-1", + "reservation-1-2", + "reservation-1-3", + "reservation-1-4", + "reservation-1-5", + "reservation-2-0", + "reservation-2-1", + "reservation-2-2", + "reservation-2-3", + "reservation-2-4", + "reservation-2-5", + "reservation-3-0", + "reservation-3-1", + "reservation-3-2", + "reservation-3-3", + "reservation-3-4", + "reservation-3-5", + "reservation-4-0", + "reservation-4-1", + "reservation-4-2", + "reservation-4-3", + "reservation-4-4", + "reservation-4-5", + "reservation-5-0", + "reservation-5-1", + "reservation-5-2", + "reservation-5-3", + "reservation-5-4", + "reservation-5-5", + "reservation-6-0", + "reservation-6-1", + "reservation-6-2", + "reservation-6-3", + "reservation-6-4", + "reservation-6-5", + "reservation-7-0", + "reservation-7-1", + "reservation-7-2", + "reservation-7-3", + "reservation-7-4", + "reservation-7-5" + ], + "counts": { + "detected": 32, + "preserved": 16 + }, + "description": "Authored reservation policy.", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": true, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "reservation", + "scope": "contract" + }, + { + "assessment": "untested", + "case_ids": [], + "counts": {}, + "description": "An inventoried untested obligation.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "reservation-untested", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 16, + "total": 48 + }, + "validator_id": "controlled/reservation/correct/1" + }, + "reservation/crash-on-fault": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "quantity" + ], + "id": "reservation-0-0", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "1fc600ada97a6695444df534d186265e57f62361511d24d15320277a8ec8d7d6" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "quantity" + ], + "id": "reservation-0-1", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "7374d2c95560fcaff9c4b9363e8fb8850f14b17619774bf41e5a8a5cf5e6ed67" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "quantity" + ], + "id": "reservation-0-2", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "071bbd02833d19d167739460785886f9219373f17f67a365bd8d7f211f87a2a4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "resource" + ], + "id": "reservation-0-3", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "65c361c5af1052ab083d5ec8e3ab8e01ba9711f37a206e3dbf73fd455adb214b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [], + "id": "reservation-0-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "e5f35c86979707b0033959bfb0dce2b3a1278e00ac7b3ddf1715a665b9e0c0f7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [], + "id": "reservation-0-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0e025e5f737a1a6da2b67bb77ee838e89643353618c82a149b8d67382add73a3" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "quantity" + ], + "id": "reservation-1-0", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "a31b0b40c4a2731c4d2301379229ac52601b53b9227406f93d5552e4b009a771" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "quantity" + ], + "id": "reservation-1-1", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "6649c4dccdf8da43ccea052a258c2774a22d61af8759b92a99919f33672c4b88" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "quantity" + ], + "id": "reservation-1-2", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "cc9d75acdf55d8ea8434d5d68b300393fb49fb1788a276071b85d1303ee8543b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "resource" + ], + "id": "reservation-1-3", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "0a201dcd0fef1c32aefd5952616ebd57b63db5221c5e87897ff23537662831f4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [], + "id": "reservation-1-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "c5708bc72b664a3860bef73291bf3ceebd779a5c152453912301ddbc8705e53e" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [], + "id": "reservation-1-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f9c77126ba4d34c653c59728acbd83f396109df4e664d20d72d36a3f081a5c84" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "quantity" + ], + "id": "reservation-2-0", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "90896fef73bf1e7b5723e5d9ab7671da1af612939906be955acc8b0c4728644d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "quantity" + ], + "id": "reservation-2-1", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "b01e41e049d1e0992c1688ff69d9cc7efccae519cea8c92353c56bfcf9ea73c2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "quantity" + ], + "id": "reservation-2-2", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "88404cc0ee9bf47f78a6000515c1744dad0a2e6977c607e6066b74c936de54c7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "resource" + ], + "id": "reservation-2-3", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "0626666411f02ccfb32fa2955f027446488fb0f2b0ca04f16f82b913f5e9ae78" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [], + "id": "reservation-2-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b0eb9a4f3a0a1f1cc87d53d99c5219107c8f28f7f333235143a517cc62ce5afb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [], + "id": "reservation-2-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "c345f2f38bf72e218be04f3d6cb4307bc07c8fbef6520164458ee92b907973e1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "quantity" + ], + "id": "reservation-3-0", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "6a5668208d87825b0bc2f866471ef60278d10c040d8cdab3c95c304c27ab4949" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "quantity" + ], + "id": "reservation-3-1", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "2c71639065c74db9d2f6ca8db87db4a83d9fb1d6e1384ae79cb8daccb5dc3c79" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "quantity" + ], + "id": "reservation-3-2", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "f1e4b5db85138fb919f24797e266d2032f71b6768780a82754aef1c753460594" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "resource" + ], + "id": "reservation-3-3", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "4b27551c1f51c816b65831f493826adc1796b98aec66977a047cff97ea200ce7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [], + "id": "reservation-3-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "cd5e432fece013c8fb0abe46f8f93cf8e292ca8104551a3f2e6c5a0a7ea61c04" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [], + "id": "reservation-3-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "185a339b16b268514b32eb70af1a0a13edee14a2e4f541c8fd3732d539622c24" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "quantity" + ], + "id": "reservation-4-0", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "5878b509a9e6fd0681485727293077caabaf818e727241485d1a627710e33e06" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "quantity" + ], + "id": "reservation-4-1", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "ec67ee408d81f95b636d0f6e69f60cac5da67167e0a6cc5c710778b8d3106ecb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "quantity" + ], + "id": "reservation-4-2", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "351ea66efae7531aaef80ca3dab37066634614c9d09779728cd4b1804c068c4c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "resource" + ], + "id": "reservation-4-3", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "85ad70c43500fd9680301e4b2a0a0437e62e776e4c6e632fffa7b29fe2be54d1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [], + "id": "reservation-4-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b4cb6ec94bfbd656632fbec156f295258787fe64a99af7a2d1884ef2a2e2bb68" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [], + "id": "reservation-4-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "c26f6e66af343485dd6a1a95ff9d5808145c9f2ab0314f51ece7cb983374c992" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "quantity" + ], + "id": "reservation-5-0", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "e22941ec7c7f81e3821f97ac28f881166fcdd8a0453a18e14c6a6ac0ab113e77" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "quantity" + ], + "id": "reservation-5-1", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "73357e37bd70ff319ac3a84892dec5ed64c266b7b3338ac822e3cddf12d129fa" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "quantity" + ], + "id": "reservation-5-2", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "03d413ec27a19883b08aaa7097066a1bd5b13298bebb37ea0cd0ebd8bd467726" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "resource" + ], + "id": "reservation-5-3", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "0eef9ad326105740dd04412212cc3bfb33485ffa2a5469c6e6ac30a815462c99" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [], + "id": "reservation-5-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "745b7d1c042e975a28642b758c639abad37b1c14e19c82da038d889e19b2d7a2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [], + "id": "reservation-5-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "07b3f742b38d89d4c720710cf6915cc9b0205627fffa31c6b5f710ceba6b04ad" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "quantity" + ], + "id": "reservation-6-0", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "1282a62f8e53345b39e7f2f143db41fc1de89c2b2984bd8e534202c52b3d917a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "quantity" + ], + "id": "reservation-6-1", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "06fa2d8be6a7e5c221da96bbd086e0849f6d8e5f1463c180907ca5a980a05e05" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "quantity" + ], + "id": "reservation-6-2", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "7fd2fbe4986d289d0c23fb07a18669e85549b4c6672bd141c9ef676e2a38149a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "resource" + ], + "id": "reservation-6-3", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "251c94a4b91b7662837c1e57ae9ddf87fb37db0b1ce37b3ed7e5829b80e41909" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [], + "id": "reservation-6-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "39941c278508db9b6de0b6e3907e7c35167d9abf98c54d1e454dd6c4b2b23699" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [], + "id": "reservation-6-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "541181d0d1b6ae2a12543230218e6c28ae3340bf5ec88c7473765c2978a19917" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "quantity" + ], + "id": "reservation-7-0", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "f83b180009a1a5b7826b21e13a9d743dee4f6069ef3f4372c432f91da1650846" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "quantity" + ], + "id": "reservation-7-1", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "5746690e63e142b29a4f5d5b1af20c5a678939ea825275e0fa96f18f1607e145" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "quantity" + ], + "id": "reservation-7-2", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "a3e719713ce65296687fe8cb3c8e296b1bd5d917ea305d09baedc47ba2ffefd1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "resource" + ], + "id": "reservation-7-3", + "obligation": "reservation", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "5b86ae17f6e473cadde2309412a1fc298e0c140c415816447a4bdf47d36df0ba" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [], + "id": "reservation-7-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "6e41a0ba04b3b4b95382a08b9f4426823762f18b270ee28226ec4df9c379fdf3" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [], + "id": "reservation-7-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f485010cb153d587e3f85174a65866fb804d90fbddfb5ca56505bb42ef26c656" + } + ], + "corpus_digest": "2997882c76109515e6f0e50c928f5f780dcad160d57ff300aebe292275f69e5e", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Authored reservation policy.", + "id": "reservation", + "scope": "contract" + }, + { + "description": "An inventoried untested obligation.", + "id": "reservation-untested", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "error": 32, + "preserved": 16 + }, + "detected_faults": 0, + "detection_score": 0.0, + "eligible_controls": 16, + "eligible_faults": 32, + "obligations": [ + { + "assessment": "incomplete_evidence", + "case_ids": [ + "reservation-0-0", + "reservation-0-1", + "reservation-0-2", + "reservation-0-3", + "reservation-0-4", + "reservation-0-5", + "reservation-1-0", + "reservation-1-1", + "reservation-1-2", + "reservation-1-3", + "reservation-1-4", + "reservation-1-5", + "reservation-2-0", + "reservation-2-1", + "reservation-2-2", + "reservation-2-3", + "reservation-2-4", + "reservation-2-5", + "reservation-3-0", + "reservation-3-1", + "reservation-3-2", + "reservation-3-3", + "reservation-3-4", + "reservation-3-5", + "reservation-4-0", + "reservation-4-1", + "reservation-4-2", + "reservation-4-3", + "reservation-4-4", + "reservation-4-5", + "reservation-5-0", + "reservation-5-1", + "reservation-5-2", + "reservation-5-3", + "reservation-5-4", + "reservation-5-5", + "reservation-6-0", + "reservation-6-1", + "reservation-6-2", + "reservation-6-3", + "reservation-6-4", + "reservation-6-5", + "reservation-7-0", + "reservation-7-1", + "reservation-7-2", + "reservation-7-3", + "reservation-7-4", + "reservation-7-5" + ], + "counts": { + "error": 32, + "preserved": 16 + }, + "description": "Authored reservation policy.", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": true, + "has_incomplete_evidence": true, + "has_known_gaps": false, + "id": "reservation", + "scope": "contract" + }, + { + "assessment": "untested", + "case_ids": [], + "counts": {}, + "description": "An inventoried untested obligation.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "reservation-untested", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 16, + "total": 48 + }, + "validator_id": "controlled/reservation/crash-on-fault/1" + }, + "reservation/omit-first-obligation": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "quantity" + ], + "id": "reservation-0-0", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "1fc600ada97a6695444df534d186265e57f62361511d24d15320277a8ec8d7d6" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "quantity" + ], + "id": "reservation-0-1", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "7374d2c95560fcaff9c4b9363e8fb8850f14b17619774bf41e5a8a5cf5e6ed67" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "quantity" + ], + "id": "reservation-0-2", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "071bbd02833d19d167739460785886f9219373f17f67a365bd8d7f211f87a2a4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "resource" + ], + "id": "reservation-0-3", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "resource" + ] + }, + "variant_digest": "65c361c5af1052ab083d5ec8e3ab8e01ba9711f37a206e3dbf73fd455adb214b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [], + "id": "reservation-0-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "e5f35c86979707b0033959bfb0dce2b3a1278e00ac7b3ddf1715a665b9e0c0f7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [], + "id": "reservation-0-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0e025e5f737a1a6da2b67bb77ee838e89643353618c82a149b8d67382add73a3" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "quantity" + ], + "id": "reservation-1-0", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "a31b0b40c4a2731c4d2301379229ac52601b53b9227406f93d5552e4b009a771" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "quantity" + ], + "id": "reservation-1-1", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "6649c4dccdf8da43ccea052a258c2774a22d61af8759b92a99919f33672c4b88" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "quantity" + ], + "id": "reservation-1-2", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "cc9d75acdf55d8ea8434d5d68b300393fb49fb1788a276071b85d1303ee8543b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "resource" + ], + "id": "reservation-1-3", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "resource" + ] + }, + "variant_digest": "0a201dcd0fef1c32aefd5952616ebd57b63db5221c5e87897ff23537662831f4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [], + "id": "reservation-1-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "c5708bc72b664a3860bef73291bf3ceebd779a5c152453912301ddbc8705e53e" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [], + "id": "reservation-1-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f9c77126ba4d34c653c59728acbd83f396109df4e664d20d72d36a3f081a5c84" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "quantity" + ], + "id": "reservation-2-0", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "90896fef73bf1e7b5723e5d9ab7671da1af612939906be955acc8b0c4728644d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "quantity" + ], + "id": "reservation-2-1", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b01e41e049d1e0992c1688ff69d9cc7efccae519cea8c92353c56bfcf9ea73c2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "quantity" + ], + "id": "reservation-2-2", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "88404cc0ee9bf47f78a6000515c1744dad0a2e6977c607e6066b74c936de54c7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "resource" + ], + "id": "reservation-2-3", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "resource" + ] + }, + "variant_digest": "0626666411f02ccfb32fa2955f027446488fb0f2b0ca04f16f82b913f5e9ae78" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [], + "id": "reservation-2-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b0eb9a4f3a0a1f1cc87d53d99c5219107c8f28f7f333235143a517cc62ce5afb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [], + "id": "reservation-2-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "c345f2f38bf72e218be04f3d6cb4307bc07c8fbef6520164458ee92b907973e1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "quantity" + ], + "id": "reservation-3-0", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "6a5668208d87825b0bc2f866471ef60278d10c040d8cdab3c95c304c27ab4949" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "quantity" + ], + "id": "reservation-3-1", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "2c71639065c74db9d2f6ca8db87db4a83d9fb1d6e1384ae79cb8daccb5dc3c79" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "quantity" + ], + "id": "reservation-3-2", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f1e4b5db85138fb919f24797e266d2032f71b6768780a82754aef1c753460594" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "resource" + ], + "id": "reservation-3-3", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "resource" + ] + }, + "variant_digest": "4b27551c1f51c816b65831f493826adc1796b98aec66977a047cff97ea200ce7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [], + "id": "reservation-3-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "cd5e432fece013c8fb0abe46f8f93cf8e292ca8104551a3f2e6c5a0a7ea61c04" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [], + "id": "reservation-3-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "185a339b16b268514b32eb70af1a0a13edee14a2e4f541c8fd3732d539622c24" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "quantity" + ], + "id": "reservation-4-0", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5878b509a9e6fd0681485727293077caabaf818e727241485d1a627710e33e06" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "quantity" + ], + "id": "reservation-4-1", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "ec67ee408d81f95b636d0f6e69f60cac5da67167e0a6cc5c710778b8d3106ecb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "quantity" + ], + "id": "reservation-4-2", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "351ea66efae7531aaef80ca3dab37066634614c9d09779728cd4b1804c068c4c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "resource" + ], + "id": "reservation-4-3", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "resource" + ] + }, + "variant_digest": "85ad70c43500fd9680301e4b2a0a0437e62e776e4c6e632fffa7b29fe2be54d1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [], + "id": "reservation-4-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b4cb6ec94bfbd656632fbec156f295258787fe64a99af7a2d1884ef2a2e2bb68" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [], + "id": "reservation-4-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "c26f6e66af343485dd6a1a95ff9d5808145c9f2ab0314f51ece7cb983374c992" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "quantity" + ], + "id": "reservation-5-0", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "e22941ec7c7f81e3821f97ac28f881166fcdd8a0453a18e14c6a6ac0ab113e77" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "quantity" + ], + "id": "reservation-5-1", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "73357e37bd70ff319ac3a84892dec5ed64c266b7b3338ac822e3cddf12d129fa" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "quantity" + ], + "id": "reservation-5-2", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "03d413ec27a19883b08aaa7097066a1bd5b13298bebb37ea0cd0ebd8bd467726" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "resource" + ], + "id": "reservation-5-3", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "resource" + ] + }, + "variant_digest": "0eef9ad326105740dd04412212cc3bfb33485ffa2a5469c6e6ac30a815462c99" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [], + "id": "reservation-5-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "745b7d1c042e975a28642b758c639abad37b1c14e19c82da038d889e19b2d7a2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [], + "id": "reservation-5-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "07b3f742b38d89d4c720710cf6915cc9b0205627fffa31c6b5f710ceba6b04ad" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "quantity" + ], + "id": "reservation-6-0", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "1282a62f8e53345b39e7f2f143db41fc1de89c2b2984bd8e534202c52b3d917a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "quantity" + ], + "id": "reservation-6-1", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "06fa2d8be6a7e5c221da96bbd086e0849f6d8e5f1463c180907ca5a980a05e05" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "quantity" + ], + "id": "reservation-6-2", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "7fd2fbe4986d289d0c23fb07a18669e85549b4c6672bd141c9ef676e2a38149a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "resource" + ], + "id": "reservation-6-3", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "resource" + ] + }, + "variant_digest": "251c94a4b91b7662837c1e57ae9ddf87fb37db0b1ce37b3ed7e5829b80e41909" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [], + "id": "reservation-6-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "39941c278508db9b6de0b6e3907e7c35167d9abf98c54d1e454dd6c4b2b23699" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [], + "id": "reservation-6-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "541181d0d1b6ae2a12543230218e6c28ae3340bf5ec88c7473765c2978a19917" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "quantity" + ], + "id": "reservation-7-0", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f83b180009a1a5b7826b21e13a9d743dee4f6069ef3f4372c432f91da1650846" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "quantity" + ], + "id": "reservation-7-1", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5746690e63e142b29a4f5d5b1af20c5a678939ea825275e0fa96f18f1607e145" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "quantity" + ], + "id": "reservation-7-2", + "obligation": "reservation", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "a3e719713ce65296687fe8cb3c8e296b1bd5d917ea305d09baedc47ba2ffefd1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "resource" + ], + "id": "reservation-7-3", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "resource" + ] + }, + "variant_digest": "5b86ae17f6e473cadde2309412a1fc298e0c140c415816447a4bdf47d36df0ba" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [], + "id": "reservation-7-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "6e41a0ba04b3b4b95382a08b9f4426823762f18b270ee28226ec4df9c379fdf3" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [], + "id": "reservation-7-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f485010cb153d587e3f85174a65866fb804d90fbddfb5ca56505bb42ef26c656" + } + ], + "corpus_digest": "2997882c76109515e6f0e50c928f5f780dcad160d57ff300aebe292275f69e5e", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Authored reservation policy.", + "id": "reservation", + "scope": "contract" + }, + { + "description": "An inventoried untested obligation.", + "id": "reservation-untested", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "detected": 8, + "preserved": 16, + "survived": 24 + }, + "detected_faults": 8, + "detection_score": 0.25, + "eligible_controls": 16, + "eligible_faults": 32, + "obligations": [ + { + "assessment": "known_gaps", + "case_ids": [ + "reservation-0-0", + "reservation-0-1", + "reservation-0-2", + "reservation-0-3", + "reservation-0-4", + "reservation-0-5", + "reservation-1-0", + "reservation-1-1", + "reservation-1-2", + "reservation-1-3", + "reservation-1-4", + "reservation-1-5", + "reservation-2-0", + "reservation-2-1", + "reservation-2-2", + "reservation-2-3", + "reservation-2-4", + "reservation-2-5", + "reservation-3-0", + "reservation-3-1", + "reservation-3-2", + "reservation-3-3", + "reservation-3-4", + "reservation-3-5", + "reservation-4-0", + "reservation-4-1", + "reservation-4-2", + "reservation-4-3", + "reservation-4-4", + "reservation-4-5", + "reservation-5-0", + "reservation-5-1", + "reservation-5-2", + "reservation-5-3", + "reservation-5-4", + "reservation-5-5", + "reservation-6-0", + "reservation-6-1", + "reservation-6-2", + "reservation-6-3", + "reservation-6-4", + "reservation-6-5", + "reservation-7-0", + "reservation-7-1", + "reservation-7-2", + "reservation-7-3", + "reservation-7-4", + "reservation-7-5" + ], + "counts": { + "detected": 8, + "preserved": 16, + "survived": 24 + }, + "description": "Authored reservation policy.", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": true, + "has_incomplete_evidence": false, + "has_known_gaps": true, + "id": "reservation", + "scope": "contract" + }, + { + "assessment": "untested", + "case_ids": [], + "counts": {}, + "description": "An inventoried untested obligation.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "reservation-untested", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 16, + "total": 48 + }, + "validator_id": "controlled/reservation/omit-first-obligation/1" + }, + "reservation/reject-all": { + "cases": [ + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "quantity" + ], + "id": "reservation-0-0", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "1fc600ada97a6695444df534d186265e57f62361511d24d15320277a8ec8d7d6" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "quantity" + ], + "id": "reservation-0-1", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "7374d2c95560fcaff9c4b9363e8fb8850f14b17619774bf41e5a8a5cf5e6ed67" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "quantity" + ], + "id": "reservation-0-2", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "071bbd02833d19d167739460785886f9219373f17f67a365bd8d7f211f87a2a4" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "resource" + ], + "id": "reservation-0-3", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "65c361c5af1052ab083d5ec8e3ab8e01ba9711f37a206e3dbf73fd455adb214b" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [], + "id": "reservation-0-4", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "e5f35c86979707b0033959bfb0dce2b3a1278e00ac7b3ddf1715a665b9e0c0f7" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [], + "id": "reservation-0-5", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "0e025e5f737a1a6da2b67bb77ee838e89643353618c82a149b8d67382add73a3" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "quantity" + ], + "id": "reservation-1-0", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "a31b0b40c4a2731c4d2301379229ac52601b53b9227406f93d5552e4b009a771" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "quantity" + ], + "id": "reservation-1-1", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "6649c4dccdf8da43ccea052a258c2774a22d61af8759b92a99919f33672c4b88" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "quantity" + ], + "id": "reservation-1-2", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "cc9d75acdf55d8ea8434d5d68b300393fb49fb1788a276071b85d1303ee8543b" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "resource" + ], + "id": "reservation-1-3", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "0a201dcd0fef1c32aefd5952616ebd57b63db5221c5e87897ff23537662831f4" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [], + "id": "reservation-1-4", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "c5708bc72b664a3860bef73291bf3ceebd779a5c152453912301ddbc8705e53e" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [], + "id": "reservation-1-5", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "f9c77126ba4d34c653c59728acbd83f396109df4e664d20d72d36a3f081a5c84" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "quantity" + ], + "id": "reservation-2-0", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "90896fef73bf1e7b5723e5d9ab7671da1af612939906be955acc8b0c4728644d" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "quantity" + ], + "id": "reservation-2-1", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "b01e41e049d1e0992c1688ff69d9cc7efccae519cea8c92353c56bfcf9ea73c2" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "quantity" + ], + "id": "reservation-2-2", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "88404cc0ee9bf47f78a6000515c1744dad0a2e6977c607e6066b74c936de54c7" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "resource" + ], + "id": "reservation-2-3", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "0626666411f02ccfb32fa2955f027446488fb0f2b0ca04f16f82b913f5e9ae78" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [], + "id": "reservation-2-4", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "b0eb9a4f3a0a1f1cc87d53d99c5219107c8f28f7f333235143a517cc62ce5afb" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [], + "id": "reservation-2-5", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "c345f2f38bf72e218be04f3d6cb4307bc07c8fbef6520164458ee92b907973e1" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "quantity" + ], + "id": "reservation-3-0", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "6a5668208d87825b0bc2f866471ef60278d10c040d8cdab3c95c304c27ab4949" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "quantity" + ], + "id": "reservation-3-1", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "2c71639065c74db9d2f6ca8db87db4a83d9fb1d6e1384ae79cb8daccb5dc3c79" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "quantity" + ], + "id": "reservation-3-2", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "f1e4b5db85138fb919f24797e266d2032f71b6768780a82754aef1c753460594" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "resource" + ], + "id": "reservation-3-3", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "4b27551c1f51c816b65831f493826adc1796b98aec66977a047cff97ea200ce7" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [], + "id": "reservation-3-4", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "cd5e432fece013c8fb0abe46f8f93cf8e292ca8104551a3f2e6c5a0a7ea61c04" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [], + "id": "reservation-3-5", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "185a339b16b268514b32eb70af1a0a13edee14a2e4f541c8fd3732d539622c24" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "quantity" + ], + "id": "reservation-4-0", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "5878b509a9e6fd0681485727293077caabaf818e727241485d1a627710e33e06" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "quantity" + ], + "id": "reservation-4-1", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "ec67ee408d81f95b636d0f6e69f60cac5da67167e0a6cc5c710778b8d3106ecb" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "quantity" + ], + "id": "reservation-4-2", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "351ea66efae7531aaef80ca3dab37066634614c9d09779728cd4b1804c068c4c" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "resource" + ], + "id": "reservation-4-3", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "85ad70c43500fd9680301e4b2a0a0437e62e776e4c6e632fffa7b29fe2be54d1" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [], + "id": "reservation-4-4", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "b4cb6ec94bfbd656632fbec156f295258787fe64a99af7a2d1884ef2a2e2bb68" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [], + "id": "reservation-4-5", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "c26f6e66af343485dd6a1a95ff9d5808145c9f2ab0314f51ece7cb983374c992" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "quantity" + ], + "id": "reservation-5-0", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "e22941ec7c7f81e3821f97ac28f881166fcdd8a0453a18e14c6a6ac0ab113e77" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "quantity" + ], + "id": "reservation-5-1", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "73357e37bd70ff319ac3a84892dec5ed64c266b7b3338ac822e3cddf12d129fa" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "quantity" + ], + "id": "reservation-5-2", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "03d413ec27a19883b08aaa7097066a1bd5b13298bebb37ea0cd0ebd8bd467726" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "resource" + ], + "id": "reservation-5-3", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "0eef9ad326105740dd04412212cc3bfb33485ffa2a5469c6e6ac30a815462c99" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [], + "id": "reservation-5-4", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "745b7d1c042e975a28642b758c639abad37b1c14e19c82da038d889e19b2d7a2" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [], + "id": "reservation-5-5", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "07b3f742b38d89d4c720710cf6915cc9b0205627fffa31c6b5f710ceba6b04ad" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "quantity" + ], + "id": "reservation-6-0", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "1282a62f8e53345b39e7f2f143db41fc1de89c2b2984bd8e534202c52b3d917a" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "quantity" + ], + "id": "reservation-6-1", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "06fa2d8be6a7e5c221da96bbd086e0849f6d8e5f1463c180907ca5a980a05e05" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "quantity" + ], + "id": "reservation-6-2", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "7fd2fbe4986d289d0c23fb07a18669e85549b4c6672bd141c9ef676e2a38149a" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "resource" + ], + "id": "reservation-6-3", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "251c94a4b91b7662837c1e57ae9ddf87fb37db0b1ce37b3ed7e5829b80e41909" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [], + "id": "reservation-6-4", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "39941c278508db9b6de0b6e3907e7c35167d9abf98c54d1e454dd6c4b2b23699" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [], + "id": "reservation-6-5", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "541181d0d1b6ae2a12543230218e6c28ae3340bf5ec88c7473765c2978a19917" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "quantity" + ], + "id": "reservation-7-0", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "f83b180009a1a5b7826b21e13a9d743dee4f6069ef3f4372c432f91da1650846" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "quantity" + ], + "id": "reservation-7-1", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "5746690e63e142b29a4f5d5b1af20c5a678939ea825275e0fa96f18f1607e145" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "quantity" + ], + "id": "reservation-7-2", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "a3e719713ce65296687fe8cb3c8e296b1bd5d917ea305d09baedc47ba2ffefd1" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "resource" + ], + "id": "reservation-7-3", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "5b86ae17f6e473cadde2309412a1fc298e0c140c415816447a4bdf47d36df0ba" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [], + "id": "reservation-7-4", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "6e41a0ba04b3b4b95382a08b9f4426823762f18b270ee28226ec4df9c379fdf3" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [], + "id": "reservation-7-5", + "obligation": "reservation", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "f485010cb153d587e3f85174a65866fb804d90fbddfb5ca56505bb42ef26c656" + } + ], + "corpus_digest": "2997882c76109515e6f0e50c928f5f780dcad160d57ff300aebe292275f69e5e", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Authored reservation policy.", + "id": "reservation", + "scope": "contract" + }, + { + "description": "An inventoried untested obligation.", + "id": "reservation-untested", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "baseline_failed": 48 + }, + "detected_faults": 0, + "detection_score": null, + "eligible_controls": 0, + "eligible_faults": 0, + "obligations": [ + { + "assessment": "incomplete_evidence", + "case_ids": [ + "reservation-0-0", + "reservation-0-1", + "reservation-0-2", + "reservation-0-3", + "reservation-0-4", + "reservation-0-5", + "reservation-1-0", + "reservation-1-1", + "reservation-1-2", + "reservation-1-3", + "reservation-1-4", + "reservation-1-5", + "reservation-2-0", + "reservation-2-1", + "reservation-2-2", + "reservation-2-3", + "reservation-2-4", + "reservation-2-5", + "reservation-3-0", + "reservation-3-1", + "reservation-3-2", + "reservation-3-3", + "reservation-3-4", + "reservation-3-5", + "reservation-4-0", + "reservation-4-1", + "reservation-4-2", + "reservation-4-3", + "reservation-4-4", + "reservation-4-5", + "reservation-5-0", + "reservation-5-1", + "reservation-5-2", + "reservation-5-3", + "reservation-5-4", + "reservation-5-5", + "reservation-6-0", + "reservation-6-1", + "reservation-6-2", + "reservation-6-3", + "reservation-6-4", + "reservation-6-5", + "reservation-7-0", + "reservation-7-1", + "reservation-7-2", + "reservation-7-3", + "reservation-7-4", + "reservation-7-5" + ], + "counts": { + "baseline_failed": 48 + }, + "description": "Authored reservation policy.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": true, + "has_known_gaps": false, + "id": "reservation", + "scope": "contract" + }, + { + "assessment": "untested", + "case_ids": [], + "counts": {}, + "description": "An inventoried untested obligation.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "reservation-untested", + "scope": "contract" + } + ], + "preservation_rate": null, + "preserved_controls": 0, + "total": 48 + }, + "validator_id": "controlled/reservation/reject-all/1" + }, + "reservation/reject-valid-variation": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "quantity" + ], + "id": "reservation-0-0", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "1fc600ada97a6695444df534d186265e57f62361511d24d15320277a8ec8d7d6" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "quantity" + ], + "id": "reservation-0-1", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "7374d2c95560fcaff9c4b9363e8fb8850f14b17619774bf41e5a8a5cf5e6ed67" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "quantity" + ], + "id": "reservation-0-2", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "071bbd02833d19d167739460785886f9219373f17f67a365bd8d7f211f87a2a4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "resource" + ], + "id": "reservation-0-3", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "resource" + ] + }, + "variant_digest": "65c361c5af1052ab083d5ec8e3ab8e01ba9711f37a206e3dbf73fd455adb214b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [], + "id": "reservation-0-4", + "obligation": "reservation", + "outcome": "regressed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "wording" + ] + }, + "variant_digest": "e5f35c86979707b0033959bfb0dce2b3a1278e00ac7b3ddf1715a665b9e0c0f7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [], + "id": "reservation-0-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0e025e5f737a1a6da2b67bb77ee838e89643353618c82a149b8d67382add73a3" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "quantity" + ], + "id": "reservation-1-0", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "a31b0b40c4a2731c4d2301379229ac52601b53b9227406f93d5552e4b009a771" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "quantity" + ], + "id": "reservation-1-1", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "6649c4dccdf8da43ccea052a258c2774a22d61af8759b92a99919f33672c4b88" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "quantity" + ], + "id": "reservation-1-2", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "cc9d75acdf55d8ea8434d5d68b300393fb49fb1788a276071b85d1303ee8543b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "resource" + ], + "id": "reservation-1-3", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "resource" + ] + }, + "variant_digest": "0a201dcd0fef1c32aefd5952616ebd57b63db5221c5e87897ff23537662831f4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [], + "id": "reservation-1-4", + "obligation": "reservation", + "outcome": "regressed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "wording" + ] + }, + "variant_digest": "c5708bc72b664a3860bef73291bf3ceebd779a5c152453912301ddbc8705e53e" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [], + "id": "reservation-1-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f9c77126ba4d34c653c59728acbd83f396109df4e664d20d72d36a3f081a5c84" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "quantity" + ], + "id": "reservation-2-0", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "90896fef73bf1e7b5723e5d9ab7671da1af612939906be955acc8b0c4728644d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "quantity" + ], + "id": "reservation-2-1", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "b01e41e049d1e0992c1688ff69d9cc7efccae519cea8c92353c56bfcf9ea73c2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "quantity" + ], + "id": "reservation-2-2", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "88404cc0ee9bf47f78a6000515c1744dad0a2e6977c607e6066b74c936de54c7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "resource" + ], + "id": "reservation-2-3", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "resource" + ] + }, + "variant_digest": "0626666411f02ccfb32fa2955f027446488fb0f2b0ca04f16f82b913f5e9ae78" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [], + "id": "reservation-2-4", + "obligation": "reservation", + "outcome": "regressed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "wording" + ] + }, + "variant_digest": "b0eb9a4f3a0a1f1cc87d53d99c5219107c8f28f7f333235143a517cc62ce5afb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [], + "id": "reservation-2-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "c345f2f38bf72e218be04f3d6cb4307bc07c8fbef6520164458ee92b907973e1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "quantity" + ], + "id": "reservation-3-0", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "6a5668208d87825b0bc2f866471ef60278d10c040d8cdab3c95c304c27ab4949" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "quantity" + ], + "id": "reservation-3-1", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "2c71639065c74db9d2f6ca8db87db4a83d9fb1d6e1384ae79cb8daccb5dc3c79" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "quantity" + ], + "id": "reservation-3-2", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "f1e4b5db85138fb919f24797e266d2032f71b6768780a82754aef1c753460594" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "resource" + ], + "id": "reservation-3-3", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "resource" + ] + }, + "variant_digest": "4b27551c1f51c816b65831f493826adc1796b98aec66977a047cff97ea200ce7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [], + "id": "reservation-3-4", + "obligation": "reservation", + "outcome": "regressed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "wording" + ] + }, + "variant_digest": "cd5e432fece013c8fb0abe46f8f93cf8e292ca8104551a3f2e6c5a0a7ea61c04" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [], + "id": "reservation-3-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "185a339b16b268514b32eb70af1a0a13edee14a2e4f541c8fd3732d539622c24" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "quantity" + ], + "id": "reservation-4-0", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "5878b509a9e6fd0681485727293077caabaf818e727241485d1a627710e33e06" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "quantity" + ], + "id": "reservation-4-1", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "ec67ee408d81f95b636d0f6e69f60cac5da67167e0a6cc5c710778b8d3106ecb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "quantity" + ], + "id": "reservation-4-2", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "351ea66efae7531aaef80ca3dab37066634614c9d09779728cd4b1804c068c4c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "resource" + ], + "id": "reservation-4-3", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "resource" + ] + }, + "variant_digest": "85ad70c43500fd9680301e4b2a0a0437e62e776e4c6e632fffa7b29fe2be54d1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [], + "id": "reservation-4-4", + "obligation": "reservation", + "outcome": "regressed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "wording" + ] + }, + "variant_digest": "b4cb6ec94bfbd656632fbec156f295258787fe64a99af7a2d1884ef2a2e2bb68" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [], + "id": "reservation-4-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "c26f6e66af343485dd6a1a95ff9d5808145c9f2ab0314f51ece7cb983374c992" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "quantity" + ], + "id": "reservation-5-0", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "e22941ec7c7f81e3821f97ac28f881166fcdd8a0453a18e14c6a6ac0ab113e77" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "quantity" + ], + "id": "reservation-5-1", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "73357e37bd70ff319ac3a84892dec5ed64c266b7b3338ac822e3cddf12d129fa" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "quantity" + ], + "id": "reservation-5-2", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "03d413ec27a19883b08aaa7097066a1bd5b13298bebb37ea0cd0ebd8bd467726" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "resource" + ], + "id": "reservation-5-3", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "resource" + ] + }, + "variant_digest": "0eef9ad326105740dd04412212cc3bfb33485ffa2a5469c6e6ac30a815462c99" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [], + "id": "reservation-5-4", + "obligation": "reservation", + "outcome": "regressed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "wording" + ] + }, + "variant_digest": "745b7d1c042e975a28642b758c639abad37b1c14e19c82da038d889e19b2d7a2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [], + "id": "reservation-5-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "07b3f742b38d89d4c720710cf6915cc9b0205627fffa31c6b5f710ceba6b04ad" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "quantity" + ], + "id": "reservation-6-0", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "1282a62f8e53345b39e7f2f143db41fc1de89c2b2984bd8e534202c52b3d917a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "quantity" + ], + "id": "reservation-6-1", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "06fa2d8be6a7e5c221da96bbd086e0849f6d8e5f1463c180907ca5a980a05e05" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "quantity" + ], + "id": "reservation-6-2", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "7fd2fbe4986d289d0c23fb07a18669e85549b4c6672bd141c9ef676e2a38149a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "resource" + ], + "id": "reservation-6-3", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "resource" + ] + }, + "variant_digest": "251c94a4b91b7662837c1e57ae9ddf87fb37db0b1ce37b3ed7e5829b80e41909" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [], + "id": "reservation-6-4", + "obligation": "reservation", + "outcome": "regressed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "wording" + ] + }, + "variant_digest": "39941c278508db9b6de0b6e3907e7c35167d9abf98c54d1e454dd6c4b2b23699" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [], + "id": "reservation-6-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "541181d0d1b6ae2a12543230218e6c28ae3340bf5ec88c7473765c2978a19917" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "quantity" + ], + "id": "reservation-7-0", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "f83b180009a1a5b7826b21e13a9d743dee4f6069ef3f4372c432f91da1650846" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "quantity" + ], + "id": "reservation-7-1", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "5746690e63e142b29a4f5d5b1af20c5a678939ea825275e0fa96f18f1607e145" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "quantity" + ], + "id": "reservation-7-2", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "quantity" + ] + }, + "variant_digest": "a3e719713ce65296687fe8cb3c8e296b1bd5d917ea305d09baedc47ba2ffefd1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "resource" + ], + "id": "reservation-7-3", + "obligation": "reservation", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "resource" + ] + }, + "variant_digest": "5b86ae17f6e473cadde2309412a1fc298e0c140c415816447a4bdf47d36df0ba" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [], + "id": "reservation-7-4", + "obligation": "reservation", + "outcome": "regressed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "wording" + ] + }, + "variant_digest": "6e41a0ba04b3b4b95382a08b9f4426823762f18b270ee28226ec4df9c379fdf3" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [], + "id": "reservation-7-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f485010cb153d587e3f85174a65866fb804d90fbddfb5ca56505bb42ef26c656" + } + ], + "corpus_digest": "2997882c76109515e6f0e50c928f5f780dcad160d57ff300aebe292275f69e5e", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Authored reservation policy.", + "id": "reservation", + "scope": "contract" + }, + { + "description": "An inventoried untested obligation.", + "id": "reservation-untested", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "detected": 32, + "preserved": 8, + "regressed": 8 + }, + "detected_faults": 32, + "detection_score": 1.0, + "eligible_controls": 16, + "eligible_faults": 32, + "obligations": [ + { + "assessment": "known_gaps", + "case_ids": [ + "reservation-0-0", + "reservation-0-1", + "reservation-0-2", + "reservation-0-3", + "reservation-0-4", + "reservation-0-5", + "reservation-1-0", + "reservation-1-1", + "reservation-1-2", + "reservation-1-3", + "reservation-1-4", + "reservation-1-5", + "reservation-2-0", + "reservation-2-1", + "reservation-2-2", + "reservation-2-3", + "reservation-2-4", + "reservation-2-5", + "reservation-3-0", + "reservation-3-1", + "reservation-3-2", + "reservation-3-3", + "reservation-3-4", + "reservation-3-5", + "reservation-4-0", + "reservation-4-1", + "reservation-4-2", + "reservation-4-3", + "reservation-4-4", + "reservation-4-5", + "reservation-5-0", + "reservation-5-1", + "reservation-5-2", + "reservation-5-3", + "reservation-5-4", + "reservation-5-5", + "reservation-6-0", + "reservation-6-1", + "reservation-6-2", + "reservation-6-3", + "reservation-6-4", + "reservation-6-5", + "reservation-7-0", + "reservation-7-1", + "reservation-7-2", + "reservation-7-3", + "reservation-7-4", + "reservation-7-5" + ], + "counts": { + "detected": 32, + "preserved": 8, + "regressed": 8 + }, + "description": "Authored reservation policy.", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": true, + "has_incomplete_evidence": false, + "has_known_gaps": true, + "id": "reservation", + "scope": "contract" + }, + { + "assessment": "untested", + "case_ids": [], + "counts": {}, + "description": "An inventoried untested obligation.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "reservation-untested", + "scope": "contract" + } + ], + "preservation_rate": 0.5, + "preserved_controls": 8, + "total": 48 + }, + "validator_id": "controlled/reservation/reject-valid-variation/1" + }, + "reservation/unknown-on-fault": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "quantity" + ], + "id": "reservation-0-0", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "1fc600ada97a6695444df534d186265e57f62361511d24d15320277a8ec8d7d6" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "quantity" + ], + "id": "reservation-0-1", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "7374d2c95560fcaff9c4b9363e8fb8850f14b17619774bf41e5a8a5cf5e6ed67" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "quantity" + ], + "id": "reservation-0-2", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "071bbd02833d19d167739460785886f9219373f17f67a365bd8d7f211f87a2a4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "resource" + ], + "id": "reservation-0-3", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "65c361c5af1052ab083d5ec8e3ab8e01ba9711f37a206e3dbf73fd455adb214b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [], + "id": "reservation-0-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "e5f35c86979707b0033959bfb0dce2b3a1278e00ac7b3ddf1715a665b9e0c0f7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [], + "id": "reservation-0-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0e025e5f737a1a6da2b67bb77ee838e89643353618c82a149b8d67382add73a3" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "quantity" + ], + "id": "reservation-1-0", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "a31b0b40c4a2731c4d2301379229ac52601b53b9227406f93d5552e4b009a771" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "quantity" + ], + "id": "reservation-1-1", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "6649c4dccdf8da43ccea052a258c2774a22d61af8759b92a99919f33672c4b88" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "quantity" + ], + "id": "reservation-1-2", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "cc9d75acdf55d8ea8434d5d68b300393fb49fb1788a276071b85d1303ee8543b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "resource" + ], + "id": "reservation-1-3", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "0a201dcd0fef1c32aefd5952616ebd57b63db5221c5e87897ff23537662831f4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [], + "id": "reservation-1-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "c5708bc72b664a3860bef73291bf3ceebd779a5c152453912301ddbc8705e53e" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [], + "id": "reservation-1-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f9c77126ba4d34c653c59728acbd83f396109df4e664d20d72d36a3f081a5c84" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "quantity" + ], + "id": "reservation-2-0", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "90896fef73bf1e7b5723e5d9ab7671da1af612939906be955acc8b0c4728644d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "quantity" + ], + "id": "reservation-2-1", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "b01e41e049d1e0992c1688ff69d9cc7efccae519cea8c92353c56bfcf9ea73c2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "quantity" + ], + "id": "reservation-2-2", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "88404cc0ee9bf47f78a6000515c1744dad0a2e6977c607e6066b74c936de54c7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "resource" + ], + "id": "reservation-2-3", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "0626666411f02ccfb32fa2955f027446488fb0f2b0ca04f16f82b913f5e9ae78" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [], + "id": "reservation-2-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b0eb9a4f3a0a1f1cc87d53d99c5219107c8f28f7f333235143a517cc62ce5afb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [], + "id": "reservation-2-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "c345f2f38bf72e218be04f3d6cb4307bc07c8fbef6520164458ee92b907973e1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "quantity" + ], + "id": "reservation-3-0", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "6a5668208d87825b0bc2f866471ef60278d10c040d8cdab3c95c304c27ab4949" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "quantity" + ], + "id": "reservation-3-1", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "2c71639065c74db9d2f6ca8db87db4a83d9fb1d6e1384ae79cb8daccb5dc3c79" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "quantity" + ], + "id": "reservation-3-2", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "f1e4b5db85138fb919f24797e266d2032f71b6768780a82754aef1c753460594" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "resource" + ], + "id": "reservation-3-3", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "4b27551c1f51c816b65831f493826adc1796b98aec66977a047cff97ea200ce7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [], + "id": "reservation-3-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "cd5e432fece013c8fb0abe46f8f93cf8e292ca8104551a3f2e6c5a0a7ea61c04" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [], + "id": "reservation-3-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "185a339b16b268514b32eb70af1a0a13edee14a2e4f541c8fd3732d539622c24" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "quantity" + ], + "id": "reservation-4-0", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "5878b509a9e6fd0681485727293077caabaf818e727241485d1a627710e33e06" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "quantity" + ], + "id": "reservation-4-1", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "ec67ee408d81f95b636d0f6e69f60cac5da67167e0a6cc5c710778b8d3106ecb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "quantity" + ], + "id": "reservation-4-2", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "351ea66efae7531aaef80ca3dab37066634614c9d09779728cd4b1804c068c4c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "resource" + ], + "id": "reservation-4-3", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "85ad70c43500fd9680301e4b2a0a0437e62e776e4c6e632fffa7b29fe2be54d1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [], + "id": "reservation-4-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b4cb6ec94bfbd656632fbec156f295258787fe64a99af7a2d1884ef2a2e2bb68" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [], + "id": "reservation-4-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "c26f6e66af343485dd6a1a95ff9d5808145c9f2ab0314f51ece7cb983374c992" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "quantity" + ], + "id": "reservation-5-0", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "e22941ec7c7f81e3821f97ac28f881166fcdd8a0453a18e14c6a6ac0ab113e77" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "quantity" + ], + "id": "reservation-5-1", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "73357e37bd70ff319ac3a84892dec5ed64c266b7b3338ac822e3cddf12d129fa" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "quantity" + ], + "id": "reservation-5-2", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "03d413ec27a19883b08aaa7097066a1bd5b13298bebb37ea0cd0ebd8bd467726" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "resource" + ], + "id": "reservation-5-3", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "0eef9ad326105740dd04412212cc3bfb33485ffa2a5469c6e6ac30a815462c99" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [], + "id": "reservation-5-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "745b7d1c042e975a28642b758c639abad37b1c14e19c82da038d889e19b2d7a2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [], + "id": "reservation-5-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "07b3f742b38d89d4c720710cf6915cc9b0205627fffa31c6b5f710ceba6b04ad" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "quantity" + ], + "id": "reservation-6-0", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "1282a62f8e53345b39e7f2f143db41fc1de89c2b2984bd8e534202c52b3d917a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "quantity" + ], + "id": "reservation-6-1", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "06fa2d8be6a7e5c221da96bbd086e0849f6d8e5f1463c180907ca5a980a05e05" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "quantity" + ], + "id": "reservation-6-2", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "7fd2fbe4986d289d0c23fb07a18669e85549b4c6672bd141c9ef676e2a38149a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "resource" + ], + "id": "reservation-6-3", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "251c94a4b91b7662837c1e57ae9ddf87fb37db0b1ce37b3ed7e5829b80e41909" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [], + "id": "reservation-6-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "39941c278508db9b6de0b6e3907e7c35167d9abf98c54d1e454dd6c4b2b23699" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [], + "id": "reservation-6-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "541181d0d1b6ae2a12543230218e6c28ae3340bf5ec88c7473765c2978a19917" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "quantity" + ], + "id": "reservation-7-0", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "f83b180009a1a5b7826b21e13a9d743dee4f6069ef3f4372c432f91da1650846" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "quantity" + ], + "id": "reservation-7-1", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "5746690e63e142b29a4f5d5b1af20c5a678939ea825275e0fa96f18f1607e145" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "quantity" + ], + "id": "reservation-7-2", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "a3e719713ce65296687fe8cb3c8e296b1bd5d917ea305d09baedc47ba2ffefd1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "resource" + ], + "id": "reservation-7-3", + "obligation": "reservation", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "5b86ae17f6e473cadde2309412a1fc298e0c140c415816447a4bdf47d36df0ba" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [], + "id": "reservation-7-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "6e41a0ba04b3b4b95382a08b9f4426823762f18b270ee28226ec4df9c379fdf3" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [], + "id": "reservation-7-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f485010cb153d587e3f85174a65866fb804d90fbddfb5ca56505bb42ef26c656" + } + ], + "corpus_digest": "2997882c76109515e6f0e50c928f5f780dcad160d57ff300aebe292275f69e5e", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Authored reservation policy.", + "id": "reservation", + "scope": "contract" + }, + { + "description": "An inventoried untested obligation.", + "id": "reservation-untested", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 16, + "undetermined": 32 + }, + "detected_faults": 0, + "detection_score": 0.0, + "eligible_controls": 16, + "eligible_faults": 32, + "obligations": [ + { + "assessment": "incomplete_evidence", + "case_ids": [ + "reservation-0-0", + "reservation-0-1", + "reservation-0-2", + "reservation-0-3", + "reservation-0-4", + "reservation-0-5", + "reservation-1-0", + "reservation-1-1", + "reservation-1-2", + "reservation-1-3", + "reservation-1-4", + "reservation-1-5", + "reservation-2-0", + "reservation-2-1", + "reservation-2-2", + "reservation-2-3", + "reservation-2-4", + "reservation-2-5", + "reservation-3-0", + "reservation-3-1", + "reservation-3-2", + "reservation-3-3", + "reservation-3-4", + "reservation-3-5", + "reservation-4-0", + "reservation-4-1", + "reservation-4-2", + "reservation-4-3", + "reservation-4-4", + "reservation-4-5", + "reservation-5-0", + "reservation-5-1", + "reservation-5-2", + "reservation-5-3", + "reservation-5-4", + "reservation-5-5", + "reservation-6-0", + "reservation-6-1", + "reservation-6-2", + "reservation-6-3", + "reservation-6-4", + "reservation-6-5", + "reservation-7-0", + "reservation-7-1", + "reservation-7-2", + "reservation-7-3", + "reservation-7-4", + "reservation-7-5" + ], + "counts": { + "preserved": 16, + "undetermined": 32 + }, + "description": "Authored reservation policy.", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": true, + "has_incomplete_evidence": true, + "has_known_gaps": false, + "id": "reservation", + "scope": "contract" + }, + { + "assessment": "untested", + "case_ids": [], + "counts": {}, + "description": "An inventoried untested obligation.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "reservation-untested", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 16, + "total": 48 + }, + "validator_id": "controlled/reservation/unknown-on-fault/1" + }, + "reservation/wrong-diagnostic-ID": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "quantity" + ], + "id": "reservation-0-0", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "1fc600ada97a6695444df534d186265e57f62361511d24d15320277a8ec8d7d6" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "quantity" + ], + "id": "reservation-0-1", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "7374d2c95560fcaff9c4b9363e8fb8850f14b17619774bf41e5a8a5cf5e6ed67" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "quantity" + ], + "id": "reservation-0-2", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "071bbd02833d19d167739460785886f9219373f17f67a365bd8d7f211f87a2a4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [ + "resource" + ], + "id": "reservation-0-3", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "65c361c5af1052ab083d5ec8e3ab8e01ba9711f37a206e3dbf73fd455adb214b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [], + "id": "reservation-0-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "e5f35c86979707b0033959bfb0dce2b3a1278e00ac7b3ddf1715a665b9e0c0f7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "44f7a379729dcfd97401c40dc8173a5c1bd74a01b7105ff97fdbd10e30b583d8", + "expected": [], + "id": "reservation-0-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0e025e5f737a1a6da2b67bb77ee838e89643353618c82a149b8d67382add73a3" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "quantity" + ], + "id": "reservation-1-0", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "a31b0b40c4a2731c4d2301379229ac52601b53b9227406f93d5552e4b009a771" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "quantity" + ], + "id": "reservation-1-1", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "6649c4dccdf8da43ccea052a258c2774a22d61af8759b92a99919f33672c4b88" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "quantity" + ], + "id": "reservation-1-2", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "cc9d75acdf55d8ea8434d5d68b300393fb49fb1788a276071b85d1303ee8543b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [ + "resource" + ], + "id": "reservation-1-3", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "0a201dcd0fef1c32aefd5952616ebd57b63db5221c5e87897ff23537662831f4" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [], + "id": "reservation-1-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "c5708bc72b664a3860bef73291bf3ceebd779a5c152453912301ddbc8705e53e" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "882500c4e53d243d15e7d708d21b40a90a1e9fcccc9af5e955f65b4efc750782", + "expected": [], + "id": "reservation-1-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f9c77126ba4d34c653c59728acbd83f396109df4e664d20d72d36a3f081a5c84" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "quantity" + ], + "id": "reservation-2-0", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "90896fef73bf1e7b5723e5d9ab7671da1af612939906be955acc8b0c4728644d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "quantity" + ], + "id": "reservation-2-1", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "b01e41e049d1e0992c1688ff69d9cc7efccae519cea8c92353c56bfcf9ea73c2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "quantity" + ], + "id": "reservation-2-2", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "88404cc0ee9bf47f78a6000515c1744dad0a2e6977c607e6066b74c936de54c7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [ + "resource" + ], + "id": "reservation-2-3", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "0626666411f02ccfb32fa2955f027446488fb0f2b0ca04f16f82b913f5e9ae78" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [], + "id": "reservation-2-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b0eb9a4f3a0a1f1cc87d53d99c5219107c8f28f7f333235143a517cc62ce5afb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "cefe122d92f51afb1173a7ee4cadecd921b487980466dff1e68b361156f5b3ec", + "expected": [], + "id": "reservation-2-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "c345f2f38bf72e218be04f3d6cb4307bc07c8fbef6520164458ee92b907973e1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "quantity" + ], + "id": "reservation-3-0", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "6a5668208d87825b0bc2f866471ef60278d10c040d8cdab3c95c304c27ab4949" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "quantity" + ], + "id": "reservation-3-1", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "2c71639065c74db9d2f6ca8db87db4a83d9fb1d6e1384ae79cb8daccb5dc3c79" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "quantity" + ], + "id": "reservation-3-2", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "f1e4b5db85138fb919f24797e266d2032f71b6768780a82754aef1c753460594" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [ + "resource" + ], + "id": "reservation-3-3", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "4b27551c1f51c816b65831f493826adc1796b98aec66977a047cff97ea200ce7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [], + "id": "reservation-3-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "cd5e432fece013c8fb0abe46f8f93cf8e292ca8104551a3f2e6c5a0a7ea61c04" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "73b1836f9d1b1c568f702fd1d9ea004f79d5a2b8fe872ff68dbe218a7e7f47b4", + "expected": [], + "id": "reservation-3-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "185a339b16b268514b32eb70af1a0a13edee14a2e4f541c8fd3732d539622c24" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "quantity" + ], + "id": "reservation-4-0", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "5878b509a9e6fd0681485727293077caabaf818e727241485d1a627710e33e06" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "quantity" + ], + "id": "reservation-4-1", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "ec67ee408d81f95b636d0f6e69f60cac5da67167e0a6cc5c710778b8d3106ecb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "quantity" + ], + "id": "reservation-4-2", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "351ea66efae7531aaef80ca3dab37066634614c9d09779728cd4b1804c068c4c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [ + "resource" + ], + "id": "reservation-4-3", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "85ad70c43500fd9680301e4b2a0a0437e62e776e4c6e632fffa7b29fe2be54d1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [], + "id": "reservation-4-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b4cb6ec94bfbd656632fbec156f295258787fe64a99af7a2d1884ef2a2e2bb68" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "ab463ce815dbbcea3abbd9bc61ae8ab057186264852d85f290776388def5ab99", + "expected": [], + "id": "reservation-4-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "c26f6e66af343485dd6a1a95ff9d5808145c9f2ab0314f51ece7cb983374c992" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "quantity" + ], + "id": "reservation-5-0", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "e22941ec7c7f81e3821f97ac28f881166fcdd8a0453a18e14c6a6ac0ab113e77" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "quantity" + ], + "id": "reservation-5-1", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "73357e37bd70ff319ac3a84892dec5ed64c266b7b3338ac822e3cddf12d129fa" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "quantity" + ], + "id": "reservation-5-2", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "03d413ec27a19883b08aaa7097066a1bd5b13298bebb37ea0cd0ebd8bd467726" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [ + "resource" + ], + "id": "reservation-5-3", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "0eef9ad326105740dd04412212cc3bfb33485ffa2a5469c6e6ac30a815462c99" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [], + "id": "reservation-5-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "745b7d1c042e975a28642b758c639abad37b1c14e19c82da038d889e19b2d7a2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "22e9a10c6807d99358139c667e62c26bc8560d711ee6c7b0b3acb5476e51cf49", + "expected": [], + "id": "reservation-5-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "07b3f742b38d89d4c720710cf6915cc9b0205627fffa31c6b5f710ceba6b04ad" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "quantity" + ], + "id": "reservation-6-0", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "1282a62f8e53345b39e7f2f143db41fc1de89c2b2984bd8e534202c52b3d917a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "quantity" + ], + "id": "reservation-6-1", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "06fa2d8be6a7e5c221da96bbd086e0849f6d8e5f1463c180907ca5a980a05e05" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "quantity" + ], + "id": "reservation-6-2", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "7fd2fbe4986d289d0c23fb07a18669e85549b4c6672bd141c9ef676e2a38149a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [ + "resource" + ], + "id": "reservation-6-3", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "251c94a4b91b7662837c1e57ae9ddf87fb37db0b1ce37b3ed7e5829b80e41909" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [], + "id": "reservation-6-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "39941c278508db9b6de0b6e3907e7c35167d9abf98c54d1e454dd6c4b2b23699" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7e59abe7fc99c50dc8897631e2c222c1f79686797c9cae1d8b6a8578d16018f6", + "expected": [], + "id": "reservation-6-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "541181d0d1b6ae2a12543230218e6c28ae3340bf5ec88c7473765c2978a19917" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "quantity" + ], + "id": "reservation-7-0", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "f83b180009a1a5b7826b21e13a9d743dee4f6069ef3f4372c432f91da1650846" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "quantity" + ], + "id": "reservation-7-1", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "5746690e63e142b29a4f5d5b1af20c5a678939ea825275e0fa96f18f1607e145" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "quantity" + ], + "id": "reservation-7-2", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "a3e719713ce65296687fe8cb3c8e296b1bd5d917ea305d09baedc47ba2ffefd1" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [ + "resource" + ], + "id": "reservation-7-3", + "obligation": "reservation", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "5b86ae17f6e473cadde2309412a1fc298e0c140c415816447a4bdf47d36df0ba" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [], + "id": "reservation-7-4", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "6e41a0ba04b3b4b95382a08b9f4426823762f18b270ee28226ec4df9c379fdf3" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "443ec4c78476bdcd9afccbf9eb432d3ea64844f4ce4eaba67d9b6bab8e3d990e", + "expected": [], + "id": "reservation-7-5", + "obligation": "reservation", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f485010cb153d587e3f85174a65866fb804d90fbddfb5ca56505bb42ef26c656" + } + ], + "corpus_digest": "2997882c76109515e6f0e50c928f5f780dcad160d57ff300aebe292275f69e5e", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Authored reservation policy.", + "id": "reservation", + "scope": "contract" + }, + { + "description": "An inventoried untested obligation.", + "id": "reservation-untested", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 16, + "unattributed_rejection": 32 + }, + "detected_faults": 0, + "detection_score": 0.0, + "eligible_controls": 16, + "eligible_faults": 32, + "obligations": [ + { + "assessment": "known_gaps", + "case_ids": [ + "reservation-0-0", + "reservation-0-1", + "reservation-0-2", + "reservation-0-3", + "reservation-0-4", + "reservation-0-5", + "reservation-1-0", + "reservation-1-1", + "reservation-1-2", + "reservation-1-3", + "reservation-1-4", + "reservation-1-5", + "reservation-2-0", + "reservation-2-1", + "reservation-2-2", + "reservation-2-3", + "reservation-2-4", + "reservation-2-5", + "reservation-3-0", + "reservation-3-1", + "reservation-3-2", + "reservation-3-3", + "reservation-3-4", + "reservation-3-5", + "reservation-4-0", + "reservation-4-1", + "reservation-4-2", + "reservation-4-3", + "reservation-4-4", + "reservation-4-5", + "reservation-5-0", + "reservation-5-1", + "reservation-5-2", + "reservation-5-3", + "reservation-5-4", + "reservation-5-5", + "reservation-6-0", + "reservation-6-1", + "reservation-6-2", + "reservation-6-3", + "reservation-6-4", + "reservation-6-5", + "reservation-7-0", + "reservation-7-1", + "reservation-7-2", + "reservation-7-3", + "reservation-7-4", + "reservation-7-5" + ], + "counts": { + "preserved": 16, + "unattributed_rejection": 32 + }, + "description": "Authored reservation policy.", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": true, + "has_incomplete_evidence": false, + "has_known_gaps": true, + "id": "reservation", + "scope": "contract" + }, + { + "assessment": "untested", + "case_ids": [], + "counts": {}, + "description": "An inventoried untested obligation.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "reservation-untested", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 16, + "total": 48 + }, + "validator_id": "controlled/reservation/wrong-diagnostic-ID/1" + }, + "shipping/accept-all": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "transition" + ], + "id": "shipping-0-0", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "6e6df86f499f432b835ce2a22e09f5da43ab6d07f64743eadcdc8e0d9bb4b38c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "transition" + ], + "id": "shipping-0-1", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "45cca396d27b8e5d76027a3b81c89f08f17ae55c549f7238ceee4407d7694708" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "transition" + ], + "id": "shipping-0-2", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "343e45827ff55aad95a876c3489f397e62d976df9d2c860d8da11a1284117ff5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "customer" + ], + "id": "shipping-0-3", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "916e9872ad95408281f8098491e7aefb59367d0fb61a86fc34258ccd2013a173" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [], + "id": "shipping-0-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "2f29b5bbc788eb077edb08f8156d387014f9f7ecae4dd7ee7f650c18298a2b96" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [], + "id": "shipping-0-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b64bb7c4c05e70cb2a89ed659ae714326ebeb285a56c3f509aa62c11b4093b7b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "transition" + ], + "id": "shipping-1-0", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "ecfbb40f54baa2f0c346028075b3ac644053c8cec4f9986eaa28f08b989be879" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "transition" + ], + "id": "shipping-1-1", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5eb74e50dc6223d94b98ecd4b7508abeb51b0603d5f5bcea77f829add60a8ad0" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "transition" + ], + "id": "shipping-1-2", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "ca13966541516b59ee62f9e6850acc3b89d65b4b96985c712308ece700d5aeef" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "customer" + ], + "id": "shipping-1-3", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "906744a4b69806069b187aa2d110a45480982705a905c4b980db51aec719a1a5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [], + "id": "shipping-1-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b60f3e4ee4774128012322db2418348c367aed8ac57f2a10727fca1d5d92856c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [], + "id": "shipping-1-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f2dcb169caa367a8998cd10f329451495430f2e25aad0180a134eceec27acb2b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "transition" + ], + "id": "shipping-2-0", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "76cff55e3c528f03f429682b032626e2ccd43a545a449d8e16ee30721163f91a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "transition" + ], + "id": "shipping-2-1", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "53751a76a524cd848711b923868de7b97d7b8bd08e2e1422ef9ad6381e22106d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "transition" + ], + "id": "shipping-2-2", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "df20de1155b45c5787d2c981474c4097d1218439901a30de8002642ac31748b2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "customer" + ], + "id": "shipping-2-3", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "2855c55d4c664a754eb4180f5a108d2cbdc97fcebe00977eb562d70ee7ce3e0b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [], + "id": "shipping-2-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "3ffe4da57c5277e2eded8def8717e68666af778e6c6803eb5103b22df78e7cb9" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [], + "id": "shipping-2-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5908f595c520d4b1ac2ce88cdc26e874dd7141d5e3f75212e34bc494b48e43fc" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "transition" + ], + "id": "shipping-3-0", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "972d4c6d080e11ec44d2c099b43193ac9fc1f89a97e48269318d9aadf43d1d56" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "transition" + ], + "id": "shipping-3-1", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "94ad5dfbe87f069211ebef4a1a83d3f9e4d28be053b70c87fba822f4b54a4e86" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "transition" + ], + "id": "shipping-3-2", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "127996d11a3348937638b8b1102b729687fd2da1194fb27073ea299441d8ca5a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "customer" + ], + "id": "shipping-3-3", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "c3642774a8eced35b06347d7a2937217f1417b2b06f54262a3b5dfeeaf7d0de8" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [], + "id": "shipping-3-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0c38cd97e27be9f85d067b3caec349bbcc07ad96b6ca0b3f412925e8137cff9f" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [], + "id": "shipping-3-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0d92bcd8ee1670d27e31f14dcfd6d8802dd6c2d7ae532c33c908728814c8f20a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "transition" + ], + "id": "shipping-4-0", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f6ab61a4db984fe1c40b21ffd802f252e12134cc1afb6933ec3ee92262a038b5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "transition" + ], + "id": "shipping-4-1", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "6219028ae1bf0898edd8c5d18d2d1846804bccb98bdc698a212edabbf4a833e2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "transition" + ], + "id": "shipping-4-2", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "7e6da2823e0745995856dd8070e761fcc6672afa6ebdf67b73a982d99c75e8ea" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "customer" + ], + "id": "shipping-4-3", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "e023f284af0639e13c3ed08848bd9331e8820209915ce0f6842c5a8481d29558" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [], + "id": "shipping-4-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "480c79699756274e1c21cf037640f69232304cada155baf5a220e909b26a1d33" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [], + "id": "shipping-4-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "e5fc4b53a9af939545e40873984d3237d488479c8cd2ffce5342b5d2bff890e0" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "transition" + ], + "id": "shipping-5-0", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0779f9da8ec7f1879162f383cd24adc4a220265e4fda2e8a8e89e9f5c6fbf72b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "transition" + ], + "id": "shipping-5-1", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "1e1e9f8e4d1755c1ba9bc4f383158c17aa9806542a2313695077c61a1efd4e1c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "transition" + ], + "id": "shipping-5-2", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "cda0f972e5e399bce17c8f8e11c3043838aa93d9dcc1ce08c7f541ede22ba58e" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "customer" + ], + "id": "shipping-5-3", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "3dca8c0659f6893f73d801e38ec5adb08f76622b1c57afde2cee61f5691df088" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [], + "id": "shipping-5-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "908d101b13d170c4649dc35d2f635701124eae6715290a63f994f607072385be" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [], + "id": "shipping-5-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f35e7e326becb5464d058b2665b6228c9227651e4796333f91b70a3d58745428" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "transition" + ], + "id": "shipping-6-0", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "030a1b1cc2e60d8d170c4188aef1f36d1260ddfbfc2967ab7e401a7af2c5e8b7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "transition" + ], + "id": "shipping-6-1", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "26ce3b6c308be577fef6d94290c97f2960576f9298b4a4c97c4f04c617bfe703" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "transition" + ], + "id": "shipping-6-2", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "fab4ca9d87f8760e52e613253812e8bd6f28c794ed64f356f193ec6f86e719cb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "customer" + ], + "id": "shipping-6-3", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "4632657e6d9306a9b026d21c442ac7902068588971663d9ce4e8a574c23d3759" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [], + "id": "shipping-6-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "23d6b32115270745f96f809ec564ee95c38db35b32e31f4466b972f6b74143e7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [], + "id": "shipping-6-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "07d15e4d5f292fa1b1c6c48739a072f37f8463ceabad7cc45b0affc6c2b9e89d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "transition" + ], + "id": "shipping-7-0", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "50b4b4f37292ab8a1b11febbd5436fceaad4c5b2f85f40ffc49f7c95e5403093" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "transition" + ], + "id": "shipping-7-1", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "83f992c673474985618eb5917801b666254d8d9c8c3bf8d159c7cc30bc360516" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "transition" + ], + "id": "shipping-7-2", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "fad67ed224d45f485dc30dc781ffd6f6ae313a47c34182a68b41c50587868066" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "customer" + ], + "id": "shipping-7-3", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "1d717e4f924bcc178efa928b9b9f22a0f91eae26899f65339263e9e399809bcd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [], + "id": "shipping-7-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "34e85d38401fe044cc05a08e69a8f1ed38e59dfcec960383d256df3808b7137d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [], + "id": "shipping-7-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "185bc1e327a9770ce9f5548e1cc24f04e623fc50d963142daa1efd26e92cff7f" + } + ], + "corpus_digest": "2a36b7fadd6316e0ae0215675c9596235b0bc951457053908a7ff87135a83a0b", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Authored shipping policy.", + "id": "shipping", + "scope": "contract" + }, + { + "description": "An inventoried untested obligation.", + "id": "shipping-untested", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 16, + "survived": 32 + }, + "detected_faults": 0, + "detection_score": 0.0, + "eligible_controls": 16, + "eligible_faults": 32, + "obligations": [ + { + "assessment": "known_gaps", + "case_ids": [ + "shipping-0-0", + "shipping-0-1", + "shipping-0-2", + "shipping-0-3", + "shipping-0-4", + "shipping-0-5", + "shipping-1-0", + "shipping-1-1", + "shipping-1-2", + "shipping-1-3", + "shipping-1-4", + "shipping-1-5", + "shipping-2-0", + "shipping-2-1", + "shipping-2-2", + "shipping-2-3", + "shipping-2-4", + "shipping-2-5", + "shipping-3-0", + "shipping-3-1", + "shipping-3-2", + "shipping-3-3", + "shipping-3-4", + "shipping-3-5", + "shipping-4-0", + "shipping-4-1", + "shipping-4-2", + "shipping-4-3", + "shipping-4-4", + "shipping-4-5", + "shipping-5-0", + "shipping-5-1", + "shipping-5-2", + "shipping-5-3", + "shipping-5-4", + "shipping-5-5", + "shipping-6-0", + "shipping-6-1", + "shipping-6-2", + "shipping-6-3", + "shipping-6-4", + "shipping-6-5", + "shipping-7-0", + "shipping-7-1", + "shipping-7-2", + "shipping-7-3", + "shipping-7-4", + "shipping-7-5" + ], + "counts": { + "preserved": 16, + "survived": 32 + }, + "description": "Authored shipping policy.", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": true, + "has_incomplete_evidence": false, + "has_known_gaps": true, + "id": "shipping", + "scope": "contract" + }, + { + "assessment": "untested", + "case_ids": [], + "counts": {}, + "description": "An inventoried untested obligation.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "shipping-untested", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 16, + "total": 48 + }, + "validator_id": "controlled/shipping/accept-all/1" + }, + "shipping/correct": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "transition" + ], + "id": "shipping-0-0", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "6e6df86f499f432b835ce2a22e09f5da43ab6d07f64743eadcdc8e0d9bb4b38c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "transition" + ], + "id": "shipping-0-1", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "45cca396d27b8e5d76027a3b81c89f08f17ae55c549f7238ceee4407d7694708" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "transition" + ], + "id": "shipping-0-2", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "343e45827ff55aad95a876c3489f397e62d976df9d2c860d8da11a1284117ff5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "customer" + ], + "id": "shipping-0-3", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "customer" + ] + }, + "variant_digest": "916e9872ad95408281f8098491e7aefb59367d0fb61a86fc34258ccd2013a173" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [], + "id": "shipping-0-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "2f29b5bbc788eb077edb08f8156d387014f9f7ecae4dd7ee7f650c18298a2b96" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [], + "id": "shipping-0-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b64bb7c4c05e70cb2a89ed659ae714326ebeb285a56c3f509aa62c11b4093b7b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "transition" + ], + "id": "shipping-1-0", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "ecfbb40f54baa2f0c346028075b3ac644053c8cec4f9986eaa28f08b989be879" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "transition" + ], + "id": "shipping-1-1", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "5eb74e50dc6223d94b98ecd4b7508abeb51b0603d5f5bcea77f829add60a8ad0" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "transition" + ], + "id": "shipping-1-2", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "ca13966541516b59ee62f9e6850acc3b89d65b4b96985c712308ece700d5aeef" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "customer" + ], + "id": "shipping-1-3", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "customer" + ] + }, + "variant_digest": "906744a4b69806069b187aa2d110a45480982705a905c4b980db51aec719a1a5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [], + "id": "shipping-1-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b60f3e4ee4774128012322db2418348c367aed8ac57f2a10727fca1d5d92856c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [], + "id": "shipping-1-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f2dcb169caa367a8998cd10f329451495430f2e25aad0180a134eceec27acb2b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "transition" + ], + "id": "shipping-2-0", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "76cff55e3c528f03f429682b032626e2ccd43a545a449d8e16ee30721163f91a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "transition" + ], + "id": "shipping-2-1", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "53751a76a524cd848711b923868de7b97d7b8bd08e2e1422ef9ad6381e22106d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "transition" + ], + "id": "shipping-2-2", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "df20de1155b45c5787d2c981474c4097d1218439901a30de8002642ac31748b2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "customer" + ], + "id": "shipping-2-3", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "customer" + ] + }, + "variant_digest": "2855c55d4c664a754eb4180f5a108d2cbdc97fcebe00977eb562d70ee7ce3e0b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [], + "id": "shipping-2-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "3ffe4da57c5277e2eded8def8717e68666af778e6c6803eb5103b22df78e7cb9" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [], + "id": "shipping-2-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5908f595c520d4b1ac2ce88cdc26e874dd7141d5e3f75212e34bc494b48e43fc" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "transition" + ], + "id": "shipping-3-0", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "972d4c6d080e11ec44d2c099b43193ac9fc1f89a97e48269318d9aadf43d1d56" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "transition" + ], + "id": "shipping-3-1", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "94ad5dfbe87f069211ebef4a1a83d3f9e4d28be053b70c87fba822f4b54a4e86" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "transition" + ], + "id": "shipping-3-2", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "127996d11a3348937638b8b1102b729687fd2da1194fb27073ea299441d8ca5a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "customer" + ], + "id": "shipping-3-3", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "customer" + ] + }, + "variant_digest": "c3642774a8eced35b06347d7a2937217f1417b2b06f54262a3b5dfeeaf7d0de8" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [], + "id": "shipping-3-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0c38cd97e27be9f85d067b3caec349bbcc07ad96b6ca0b3f412925e8137cff9f" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [], + "id": "shipping-3-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0d92bcd8ee1670d27e31f14dcfd6d8802dd6c2d7ae532c33c908728814c8f20a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "transition" + ], + "id": "shipping-4-0", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "f6ab61a4db984fe1c40b21ffd802f252e12134cc1afb6933ec3ee92262a038b5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "transition" + ], + "id": "shipping-4-1", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "6219028ae1bf0898edd8c5d18d2d1846804bccb98bdc698a212edabbf4a833e2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "transition" + ], + "id": "shipping-4-2", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "7e6da2823e0745995856dd8070e761fcc6672afa6ebdf67b73a982d99c75e8ea" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "customer" + ], + "id": "shipping-4-3", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "customer" + ] + }, + "variant_digest": "e023f284af0639e13c3ed08848bd9331e8820209915ce0f6842c5a8481d29558" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [], + "id": "shipping-4-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "480c79699756274e1c21cf037640f69232304cada155baf5a220e909b26a1d33" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [], + "id": "shipping-4-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "e5fc4b53a9af939545e40873984d3237d488479c8cd2ffce5342b5d2bff890e0" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "transition" + ], + "id": "shipping-5-0", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "0779f9da8ec7f1879162f383cd24adc4a220265e4fda2e8a8e89e9f5c6fbf72b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "transition" + ], + "id": "shipping-5-1", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "1e1e9f8e4d1755c1ba9bc4f383158c17aa9806542a2313695077c61a1efd4e1c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "transition" + ], + "id": "shipping-5-2", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "cda0f972e5e399bce17c8f8e11c3043838aa93d9dcc1ce08c7f541ede22ba58e" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "customer" + ], + "id": "shipping-5-3", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "customer" + ] + }, + "variant_digest": "3dca8c0659f6893f73d801e38ec5adb08f76622b1c57afde2cee61f5691df088" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [], + "id": "shipping-5-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "908d101b13d170c4649dc35d2f635701124eae6715290a63f994f607072385be" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [], + "id": "shipping-5-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f35e7e326becb5464d058b2665b6228c9227651e4796333f91b70a3d58745428" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "transition" + ], + "id": "shipping-6-0", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "030a1b1cc2e60d8d170c4188aef1f36d1260ddfbfc2967ab7e401a7af2c5e8b7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "transition" + ], + "id": "shipping-6-1", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "26ce3b6c308be577fef6d94290c97f2960576f9298b4a4c97c4f04c617bfe703" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "transition" + ], + "id": "shipping-6-2", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "fab4ca9d87f8760e52e613253812e8bd6f28c794ed64f356f193ec6f86e719cb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "customer" + ], + "id": "shipping-6-3", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "customer" + ] + }, + "variant_digest": "4632657e6d9306a9b026d21c442ac7902068588971663d9ce4e8a574c23d3759" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [], + "id": "shipping-6-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "23d6b32115270745f96f809ec564ee95c38db35b32e31f4466b972f6b74143e7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [], + "id": "shipping-6-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "07d15e4d5f292fa1b1c6c48739a072f37f8463ceabad7cc45b0affc6c2b9e89d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "transition" + ], + "id": "shipping-7-0", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "50b4b4f37292ab8a1b11febbd5436fceaad4c5b2f85f40ffc49f7c95e5403093" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "transition" + ], + "id": "shipping-7-1", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "83f992c673474985618eb5917801b666254d8d9c8c3bf8d159c7cc30bc360516" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "transition" + ], + "id": "shipping-7-2", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "fad67ed224d45f485dc30dc781ffd6f6ae313a47c34182a68b41c50587868066" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "customer" + ], + "id": "shipping-7-3", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "customer" + ] + }, + "variant_digest": "1d717e4f924bcc178efa928b9b9f22a0f91eae26899f65339263e9e399809bcd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [], + "id": "shipping-7-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "34e85d38401fe044cc05a08e69a8f1ed38e59dfcec960383d256df3808b7137d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [], + "id": "shipping-7-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "185bc1e327a9770ce9f5548e1cc24f04e623fc50d963142daa1efd26e92cff7f" + } + ], + "corpus_digest": "2a36b7fadd6316e0ae0215675c9596235b0bc951457053908a7ff87135a83a0b", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Authored shipping policy.", + "id": "shipping", + "scope": "contract" + }, + { + "description": "An inventoried untested obligation.", + "id": "shipping-untested", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "detected": 32, + "preserved": 16 + }, + "detected_faults": 32, + "detection_score": 1.0, + "eligible_controls": 16, + "eligible_faults": 32, + "obligations": [ + { + "assessment": "no_failures_observed", + "case_ids": [ + "shipping-0-0", + "shipping-0-1", + "shipping-0-2", + "shipping-0-3", + "shipping-0-4", + "shipping-0-5", + "shipping-1-0", + "shipping-1-1", + "shipping-1-2", + "shipping-1-3", + "shipping-1-4", + "shipping-1-5", + "shipping-2-0", + "shipping-2-1", + "shipping-2-2", + "shipping-2-3", + "shipping-2-4", + "shipping-2-5", + "shipping-3-0", + "shipping-3-1", + "shipping-3-2", + "shipping-3-3", + "shipping-3-4", + "shipping-3-5", + "shipping-4-0", + "shipping-4-1", + "shipping-4-2", + "shipping-4-3", + "shipping-4-4", + "shipping-4-5", + "shipping-5-0", + "shipping-5-1", + "shipping-5-2", + "shipping-5-3", + "shipping-5-4", + "shipping-5-5", + "shipping-6-0", + "shipping-6-1", + "shipping-6-2", + "shipping-6-3", + "shipping-6-4", + "shipping-6-5", + "shipping-7-0", + "shipping-7-1", + "shipping-7-2", + "shipping-7-3", + "shipping-7-4", + "shipping-7-5" + ], + "counts": { + "detected": 32, + "preserved": 16 + }, + "description": "Authored shipping policy.", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": true, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "shipping", + "scope": "contract" + }, + { + "assessment": "untested", + "case_ids": [], + "counts": {}, + "description": "An inventoried untested obligation.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "shipping-untested", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 16, + "total": 48 + }, + "validator_id": "controlled/shipping/correct/1" + }, + "shipping/crash-on-fault": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "transition" + ], + "id": "shipping-0-0", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "6e6df86f499f432b835ce2a22e09f5da43ab6d07f64743eadcdc8e0d9bb4b38c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "transition" + ], + "id": "shipping-0-1", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "45cca396d27b8e5d76027a3b81c89f08f17ae55c549f7238ceee4407d7694708" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "transition" + ], + "id": "shipping-0-2", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "343e45827ff55aad95a876c3489f397e62d976df9d2c860d8da11a1284117ff5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "customer" + ], + "id": "shipping-0-3", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "916e9872ad95408281f8098491e7aefb59367d0fb61a86fc34258ccd2013a173" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [], + "id": "shipping-0-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "2f29b5bbc788eb077edb08f8156d387014f9f7ecae4dd7ee7f650c18298a2b96" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [], + "id": "shipping-0-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b64bb7c4c05e70cb2a89ed659ae714326ebeb285a56c3f509aa62c11b4093b7b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "transition" + ], + "id": "shipping-1-0", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "ecfbb40f54baa2f0c346028075b3ac644053c8cec4f9986eaa28f08b989be879" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "transition" + ], + "id": "shipping-1-1", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "5eb74e50dc6223d94b98ecd4b7508abeb51b0603d5f5bcea77f829add60a8ad0" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "transition" + ], + "id": "shipping-1-2", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "ca13966541516b59ee62f9e6850acc3b89d65b4b96985c712308ece700d5aeef" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "customer" + ], + "id": "shipping-1-3", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "906744a4b69806069b187aa2d110a45480982705a905c4b980db51aec719a1a5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [], + "id": "shipping-1-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b60f3e4ee4774128012322db2418348c367aed8ac57f2a10727fca1d5d92856c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [], + "id": "shipping-1-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f2dcb169caa367a8998cd10f329451495430f2e25aad0180a134eceec27acb2b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "transition" + ], + "id": "shipping-2-0", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "76cff55e3c528f03f429682b032626e2ccd43a545a449d8e16ee30721163f91a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "transition" + ], + "id": "shipping-2-1", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "53751a76a524cd848711b923868de7b97d7b8bd08e2e1422ef9ad6381e22106d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "transition" + ], + "id": "shipping-2-2", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "df20de1155b45c5787d2c981474c4097d1218439901a30de8002642ac31748b2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "customer" + ], + "id": "shipping-2-3", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "2855c55d4c664a754eb4180f5a108d2cbdc97fcebe00977eb562d70ee7ce3e0b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [], + "id": "shipping-2-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "3ffe4da57c5277e2eded8def8717e68666af778e6c6803eb5103b22df78e7cb9" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [], + "id": "shipping-2-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5908f595c520d4b1ac2ce88cdc26e874dd7141d5e3f75212e34bc494b48e43fc" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "transition" + ], + "id": "shipping-3-0", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "972d4c6d080e11ec44d2c099b43193ac9fc1f89a97e48269318d9aadf43d1d56" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "transition" + ], + "id": "shipping-3-1", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "94ad5dfbe87f069211ebef4a1a83d3f9e4d28be053b70c87fba822f4b54a4e86" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "transition" + ], + "id": "shipping-3-2", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "127996d11a3348937638b8b1102b729687fd2da1194fb27073ea299441d8ca5a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "customer" + ], + "id": "shipping-3-3", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "c3642774a8eced35b06347d7a2937217f1417b2b06f54262a3b5dfeeaf7d0de8" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [], + "id": "shipping-3-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0c38cd97e27be9f85d067b3caec349bbcc07ad96b6ca0b3f412925e8137cff9f" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [], + "id": "shipping-3-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0d92bcd8ee1670d27e31f14dcfd6d8802dd6c2d7ae532c33c908728814c8f20a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "transition" + ], + "id": "shipping-4-0", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "f6ab61a4db984fe1c40b21ffd802f252e12134cc1afb6933ec3ee92262a038b5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "transition" + ], + "id": "shipping-4-1", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "6219028ae1bf0898edd8c5d18d2d1846804bccb98bdc698a212edabbf4a833e2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "transition" + ], + "id": "shipping-4-2", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "7e6da2823e0745995856dd8070e761fcc6672afa6ebdf67b73a982d99c75e8ea" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "customer" + ], + "id": "shipping-4-3", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "e023f284af0639e13c3ed08848bd9331e8820209915ce0f6842c5a8481d29558" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [], + "id": "shipping-4-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "480c79699756274e1c21cf037640f69232304cada155baf5a220e909b26a1d33" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [], + "id": "shipping-4-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "e5fc4b53a9af939545e40873984d3237d488479c8cd2ffce5342b5d2bff890e0" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "transition" + ], + "id": "shipping-5-0", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "0779f9da8ec7f1879162f383cd24adc4a220265e4fda2e8a8e89e9f5c6fbf72b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "transition" + ], + "id": "shipping-5-1", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "1e1e9f8e4d1755c1ba9bc4f383158c17aa9806542a2313695077c61a1efd4e1c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "transition" + ], + "id": "shipping-5-2", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "cda0f972e5e399bce17c8f8e11c3043838aa93d9dcc1ce08c7f541ede22ba58e" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "customer" + ], + "id": "shipping-5-3", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "3dca8c0659f6893f73d801e38ec5adb08f76622b1c57afde2cee61f5691df088" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [], + "id": "shipping-5-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "908d101b13d170c4649dc35d2f635701124eae6715290a63f994f607072385be" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [], + "id": "shipping-5-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f35e7e326becb5464d058b2665b6228c9227651e4796333f91b70a3d58745428" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "transition" + ], + "id": "shipping-6-0", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "030a1b1cc2e60d8d170c4188aef1f36d1260ddfbfc2967ab7e401a7af2c5e8b7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "transition" + ], + "id": "shipping-6-1", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "26ce3b6c308be577fef6d94290c97f2960576f9298b4a4c97c4f04c617bfe703" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "transition" + ], + "id": "shipping-6-2", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "fab4ca9d87f8760e52e613253812e8bd6f28c794ed64f356f193ec6f86e719cb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "customer" + ], + "id": "shipping-6-3", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "4632657e6d9306a9b026d21c442ac7902068588971663d9ce4e8a574c23d3759" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [], + "id": "shipping-6-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "23d6b32115270745f96f809ec564ee95c38db35b32e31f4466b972f6b74143e7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [], + "id": "shipping-6-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "07d15e4d5f292fa1b1c6c48739a072f37f8463ceabad7cc45b0affc6c2b9e89d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "transition" + ], + "id": "shipping-7-0", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "50b4b4f37292ab8a1b11febbd5436fceaad4c5b2f85f40ffc49f7c95e5403093" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "transition" + ], + "id": "shipping-7-1", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "83f992c673474985618eb5917801b666254d8d9c8c3bf8d159c7cc30bc360516" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "transition" + ], + "id": "shipping-7-2", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "fad67ed224d45f485dc30dc781ffd6f6ae313a47c34182a68b41c50587868066" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "customer" + ], + "id": "shipping-7-3", + "obligation": "shipping", + "outcome": "error", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_error: RuntimeError: RuntimeError: authored validator crash", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "1d717e4f924bcc178efa928b9b9f22a0f91eae26899f65339263e9e399809bcd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [], + "id": "shipping-7-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "34e85d38401fe044cc05a08e69a8f1ed38e59dfcec960383d256df3808b7137d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [], + "id": "shipping-7-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "185bc1e327a9770ce9f5548e1cc24f04e623fc50d963142daa1efd26e92cff7f" + } + ], + "corpus_digest": "2a36b7fadd6316e0ae0215675c9596235b0bc951457053908a7ff87135a83a0b", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Authored shipping policy.", + "id": "shipping", + "scope": "contract" + }, + { + "description": "An inventoried untested obligation.", + "id": "shipping-untested", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "error": 32, + "preserved": 16 + }, + "detected_faults": 0, + "detection_score": 0.0, + "eligible_controls": 16, + "eligible_faults": 32, + "obligations": [ + { + "assessment": "incomplete_evidence", + "case_ids": [ + "shipping-0-0", + "shipping-0-1", + "shipping-0-2", + "shipping-0-3", + "shipping-0-4", + "shipping-0-5", + "shipping-1-0", + "shipping-1-1", + "shipping-1-2", + "shipping-1-3", + "shipping-1-4", + "shipping-1-5", + "shipping-2-0", + "shipping-2-1", + "shipping-2-2", + "shipping-2-3", + "shipping-2-4", + "shipping-2-5", + "shipping-3-0", + "shipping-3-1", + "shipping-3-2", + "shipping-3-3", + "shipping-3-4", + "shipping-3-5", + "shipping-4-0", + "shipping-4-1", + "shipping-4-2", + "shipping-4-3", + "shipping-4-4", + "shipping-4-5", + "shipping-5-0", + "shipping-5-1", + "shipping-5-2", + "shipping-5-3", + "shipping-5-4", + "shipping-5-5", + "shipping-6-0", + "shipping-6-1", + "shipping-6-2", + "shipping-6-3", + "shipping-6-4", + "shipping-6-5", + "shipping-7-0", + "shipping-7-1", + "shipping-7-2", + "shipping-7-3", + "shipping-7-4", + "shipping-7-5" + ], + "counts": { + "error": 32, + "preserved": 16 + }, + "description": "Authored shipping policy.", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": true, + "has_incomplete_evidence": true, + "has_known_gaps": false, + "id": "shipping", + "scope": "contract" + }, + { + "assessment": "untested", + "case_ids": [], + "counts": {}, + "description": "An inventoried untested obligation.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "shipping-untested", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 16, + "total": 48 + }, + "validator_id": "controlled/shipping/crash-on-fault/1" + }, + "shipping/omit-first-obligation": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "transition" + ], + "id": "shipping-0-0", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "6e6df86f499f432b835ce2a22e09f5da43ab6d07f64743eadcdc8e0d9bb4b38c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "transition" + ], + "id": "shipping-0-1", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "45cca396d27b8e5d76027a3b81c89f08f17ae55c549f7238ceee4407d7694708" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "transition" + ], + "id": "shipping-0-2", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "343e45827ff55aad95a876c3489f397e62d976df9d2c860d8da11a1284117ff5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "customer" + ], + "id": "shipping-0-3", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "customer" + ] + }, + "variant_digest": "916e9872ad95408281f8098491e7aefb59367d0fb61a86fc34258ccd2013a173" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [], + "id": "shipping-0-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "2f29b5bbc788eb077edb08f8156d387014f9f7ecae4dd7ee7f650c18298a2b96" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [], + "id": "shipping-0-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b64bb7c4c05e70cb2a89ed659ae714326ebeb285a56c3f509aa62c11b4093b7b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "transition" + ], + "id": "shipping-1-0", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "ecfbb40f54baa2f0c346028075b3ac644053c8cec4f9986eaa28f08b989be879" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "transition" + ], + "id": "shipping-1-1", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5eb74e50dc6223d94b98ecd4b7508abeb51b0603d5f5bcea77f829add60a8ad0" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "transition" + ], + "id": "shipping-1-2", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "ca13966541516b59ee62f9e6850acc3b89d65b4b96985c712308ece700d5aeef" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "customer" + ], + "id": "shipping-1-3", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "customer" + ] + }, + "variant_digest": "906744a4b69806069b187aa2d110a45480982705a905c4b980db51aec719a1a5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [], + "id": "shipping-1-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b60f3e4ee4774128012322db2418348c367aed8ac57f2a10727fca1d5d92856c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [], + "id": "shipping-1-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f2dcb169caa367a8998cd10f329451495430f2e25aad0180a134eceec27acb2b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "transition" + ], + "id": "shipping-2-0", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "76cff55e3c528f03f429682b032626e2ccd43a545a449d8e16ee30721163f91a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "transition" + ], + "id": "shipping-2-1", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "53751a76a524cd848711b923868de7b97d7b8bd08e2e1422ef9ad6381e22106d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "transition" + ], + "id": "shipping-2-2", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "df20de1155b45c5787d2c981474c4097d1218439901a30de8002642ac31748b2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "customer" + ], + "id": "shipping-2-3", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "customer" + ] + }, + "variant_digest": "2855c55d4c664a754eb4180f5a108d2cbdc97fcebe00977eb562d70ee7ce3e0b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [], + "id": "shipping-2-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "3ffe4da57c5277e2eded8def8717e68666af778e6c6803eb5103b22df78e7cb9" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [], + "id": "shipping-2-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5908f595c520d4b1ac2ce88cdc26e874dd7141d5e3f75212e34bc494b48e43fc" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "transition" + ], + "id": "shipping-3-0", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "972d4c6d080e11ec44d2c099b43193ac9fc1f89a97e48269318d9aadf43d1d56" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "transition" + ], + "id": "shipping-3-1", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "94ad5dfbe87f069211ebef4a1a83d3f9e4d28be053b70c87fba822f4b54a4e86" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "transition" + ], + "id": "shipping-3-2", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "127996d11a3348937638b8b1102b729687fd2da1194fb27073ea299441d8ca5a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "customer" + ], + "id": "shipping-3-3", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "customer" + ] + }, + "variant_digest": "c3642774a8eced35b06347d7a2937217f1417b2b06f54262a3b5dfeeaf7d0de8" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [], + "id": "shipping-3-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0c38cd97e27be9f85d067b3caec349bbcc07ad96b6ca0b3f412925e8137cff9f" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [], + "id": "shipping-3-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0d92bcd8ee1670d27e31f14dcfd6d8802dd6c2d7ae532c33c908728814c8f20a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "transition" + ], + "id": "shipping-4-0", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f6ab61a4db984fe1c40b21ffd802f252e12134cc1afb6933ec3ee92262a038b5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "transition" + ], + "id": "shipping-4-1", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "6219028ae1bf0898edd8c5d18d2d1846804bccb98bdc698a212edabbf4a833e2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "transition" + ], + "id": "shipping-4-2", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "7e6da2823e0745995856dd8070e761fcc6672afa6ebdf67b73a982d99c75e8ea" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "customer" + ], + "id": "shipping-4-3", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "customer" + ] + }, + "variant_digest": "e023f284af0639e13c3ed08848bd9331e8820209915ce0f6842c5a8481d29558" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [], + "id": "shipping-4-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "480c79699756274e1c21cf037640f69232304cada155baf5a220e909b26a1d33" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [], + "id": "shipping-4-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "e5fc4b53a9af939545e40873984d3237d488479c8cd2ffce5342b5d2bff890e0" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "transition" + ], + "id": "shipping-5-0", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0779f9da8ec7f1879162f383cd24adc4a220265e4fda2e8a8e89e9f5c6fbf72b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "transition" + ], + "id": "shipping-5-1", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "1e1e9f8e4d1755c1ba9bc4f383158c17aa9806542a2313695077c61a1efd4e1c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "transition" + ], + "id": "shipping-5-2", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "cda0f972e5e399bce17c8f8e11c3043838aa93d9dcc1ce08c7f541ede22ba58e" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "customer" + ], + "id": "shipping-5-3", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "customer" + ] + }, + "variant_digest": "3dca8c0659f6893f73d801e38ec5adb08f76622b1c57afde2cee61f5691df088" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [], + "id": "shipping-5-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "908d101b13d170c4649dc35d2f635701124eae6715290a63f994f607072385be" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [], + "id": "shipping-5-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f35e7e326becb5464d058b2665b6228c9227651e4796333f91b70a3d58745428" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "transition" + ], + "id": "shipping-6-0", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "030a1b1cc2e60d8d170c4188aef1f36d1260ddfbfc2967ab7e401a7af2c5e8b7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "transition" + ], + "id": "shipping-6-1", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "26ce3b6c308be577fef6d94290c97f2960576f9298b4a4c97c4f04c617bfe703" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "transition" + ], + "id": "shipping-6-2", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "fab4ca9d87f8760e52e613253812e8bd6f28c794ed64f356f193ec6f86e719cb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "customer" + ], + "id": "shipping-6-3", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "customer" + ] + }, + "variant_digest": "4632657e6d9306a9b026d21c442ac7902068588971663d9ce4e8a574c23d3759" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [], + "id": "shipping-6-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "23d6b32115270745f96f809ec564ee95c38db35b32e31f4466b972f6b74143e7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [], + "id": "shipping-6-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "07d15e4d5f292fa1b1c6c48739a072f37f8463ceabad7cc45b0affc6c2b9e89d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "transition" + ], + "id": "shipping-7-0", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "50b4b4f37292ab8a1b11febbd5436fceaad4c5b2f85f40ffc49f7c95e5403093" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "transition" + ], + "id": "shipping-7-1", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "83f992c673474985618eb5917801b666254d8d9c8c3bf8d159c7cc30bc360516" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "transition" + ], + "id": "shipping-7-2", + "obligation": "shipping", + "outcome": "survived", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "invalid_variant_accepted", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "fad67ed224d45f485dc30dc781ffd6f6ae313a47c34182a68b41c50587868066" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "customer" + ], + "id": "shipping-7-3", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "customer" + ] + }, + "variant_digest": "1d717e4f924bcc178efa928b9b9f22a0f91eae26899f65339263e9e399809bcd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [], + "id": "shipping-7-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "34e85d38401fe044cc05a08e69a8f1ed38e59dfcec960383d256df3808b7137d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [], + "id": "shipping-7-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "185bc1e327a9770ce9f5548e1cc24f04e623fc50d963142daa1efd26e92cff7f" + } + ], + "corpus_digest": "2a36b7fadd6316e0ae0215675c9596235b0bc951457053908a7ff87135a83a0b", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Authored shipping policy.", + "id": "shipping", + "scope": "contract" + }, + { + "description": "An inventoried untested obligation.", + "id": "shipping-untested", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "detected": 8, + "preserved": 16, + "survived": 24 + }, + "detected_faults": 8, + "detection_score": 0.25, + "eligible_controls": 16, + "eligible_faults": 32, + "obligations": [ + { + "assessment": "known_gaps", + "case_ids": [ + "shipping-0-0", + "shipping-0-1", + "shipping-0-2", + "shipping-0-3", + "shipping-0-4", + "shipping-0-5", + "shipping-1-0", + "shipping-1-1", + "shipping-1-2", + "shipping-1-3", + "shipping-1-4", + "shipping-1-5", + "shipping-2-0", + "shipping-2-1", + "shipping-2-2", + "shipping-2-3", + "shipping-2-4", + "shipping-2-5", + "shipping-3-0", + "shipping-3-1", + "shipping-3-2", + "shipping-3-3", + "shipping-3-4", + "shipping-3-5", + "shipping-4-0", + "shipping-4-1", + "shipping-4-2", + "shipping-4-3", + "shipping-4-4", + "shipping-4-5", + "shipping-5-0", + "shipping-5-1", + "shipping-5-2", + "shipping-5-3", + "shipping-5-4", + "shipping-5-5", + "shipping-6-0", + "shipping-6-1", + "shipping-6-2", + "shipping-6-3", + "shipping-6-4", + "shipping-6-5", + "shipping-7-0", + "shipping-7-1", + "shipping-7-2", + "shipping-7-3", + "shipping-7-4", + "shipping-7-5" + ], + "counts": { + "detected": 8, + "preserved": 16, + "survived": 24 + }, + "description": "Authored shipping policy.", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": true, + "has_incomplete_evidence": false, + "has_known_gaps": true, + "id": "shipping", + "scope": "contract" + }, + { + "assessment": "untested", + "case_ids": [], + "counts": {}, + "description": "An inventoried untested obligation.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "shipping-untested", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 16, + "total": 48 + }, + "validator_id": "controlled/shipping/omit-first-obligation/1" + }, + "shipping/reject-all": { + "cases": [ + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "transition" + ], + "id": "shipping-0-0", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "6e6df86f499f432b835ce2a22e09f5da43ab6d07f64743eadcdc8e0d9bb4b38c" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "transition" + ], + "id": "shipping-0-1", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "45cca396d27b8e5d76027a3b81c89f08f17ae55c549f7238ceee4407d7694708" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "transition" + ], + "id": "shipping-0-2", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "343e45827ff55aad95a876c3489f397e62d976df9d2c860d8da11a1284117ff5" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "customer" + ], + "id": "shipping-0-3", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "916e9872ad95408281f8098491e7aefb59367d0fb61a86fc34258ccd2013a173" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [], + "id": "shipping-0-4", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "2f29b5bbc788eb077edb08f8156d387014f9f7ecae4dd7ee7f650c18298a2b96" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [], + "id": "shipping-0-5", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "b64bb7c4c05e70cb2a89ed659ae714326ebeb285a56c3f509aa62c11b4093b7b" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "transition" + ], + "id": "shipping-1-0", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "ecfbb40f54baa2f0c346028075b3ac644053c8cec4f9986eaa28f08b989be879" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "transition" + ], + "id": "shipping-1-1", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "5eb74e50dc6223d94b98ecd4b7508abeb51b0603d5f5bcea77f829add60a8ad0" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "transition" + ], + "id": "shipping-1-2", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "ca13966541516b59ee62f9e6850acc3b89d65b4b96985c712308ece700d5aeef" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "customer" + ], + "id": "shipping-1-3", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "906744a4b69806069b187aa2d110a45480982705a905c4b980db51aec719a1a5" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [], + "id": "shipping-1-4", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "b60f3e4ee4774128012322db2418348c367aed8ac57f2a10727fca1d5d92856c" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [], + "id": "shipping-1-5", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "f2dcb169caa367a8998cd10f329451495430f2e25aad0180a134eceec27acb2b" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "transition" + ], + "id": "shipping-2-0", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "76cff55e3c528f03f429682b032626e2ccd43a545a449d8e16ee30721163f91a" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "transition" + ], + "id": "shipping-2-1", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "53751a76a524cd848711b923868de7b97d7b8bd08e2e1422ef9ad6381e22106d" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "transition" + ], + "id": "shipping-2-2", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "df20de1155b45c5787d2c981474c4097d1218439901a30de8002642ac31748b2" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "customer" + ], + "id": "shipping-2-3", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "2855c55d4c664a754eb4180f5a108d2cbdc97fcebe00977eb562d70ee7ce3e0b" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [], + "id": "shipping-2-4", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "3ffe4da57c5277e2eded8def8717e68666af778e6c6803eb5103b22df78e7cb9" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [], + "id": "shipping-2-5", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "5908f595c520d4b1ac2ce88cdc26e874dd7141d5e3f75212e34bc494b48e43fc" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "transition" + ], + "id": "shipping-3-0", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "972d4c6d080e11ec44d2c099b43193ac9fc1f89a97e48269318d9aadf43d1d56" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "transition" + ], + "id": "shipping-3-1", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "94ad5dfbe87f069211ebef4a1a83d3f9e4d28be053b70c87fba822f4b54a4e86" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "transition" + ], + "id": "shipping-3-2", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "127996d11a3348937638b8b1102b729687fd2da1194fb27073ea299441d8ca5a" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "customer" + ], + "id": "shipping-3-3", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "c3642774a8eced35b06347d7a2937217f1417b2b06f54262a3b5dfeeaf7d0de8" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [], + "id": "shipping-3-4", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "0c38cd97e27be9f85d067b3caec349bbcc07ad96b6ca0b3f412925e8137cff9f" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [], + "id": "shipping-3-5", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "0d92bcd8ee1670d27e31f14dcfd6d8802dd6c2d7ae532c33c908728814c8f20a" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "transition" + ], + "id": "shipping-4-0", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "f6ab61a4db984fe1c40b21ffd802f252e12134cc1afb6933ec3ee92262a038b5" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "transition" + ], + "id": "shipping-4-1", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "6219028ae1bf0898edd8c5d18d2d1846804bccb98bdc698a212edabbf4a833e2" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "transition" + ], + "id": "shipping-4-2", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "7e6da2823e0745995856dd8070e761fcc6672afa6ebdf67b73a982d99c75e8ea" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "customer" + ], + "id": "shipping-4-3", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "e023f284af0639e13c3ed08848bd9331e8820209915ce0f6842c5a8481d29558" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [], + "id": "shipping-4-4", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "480c79699756274e1c21cf037640f69232304cada155baf5a220e909b26a1d33" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [], + "id": "shipping-4-5", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "e5fc4b53a9af939545e40873984d3237d488479c8cd2ffce5342b5d2bff890e0" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "transition" + ], + "id": "shipping-5-0", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "0779f9da8ec7f1879162f383cd24adc4a220265e4fda2e8a8e89e9f5c6fbf72b" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "transition" + ], + "id": "shipping-5-1", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "1e1e9f8e4d1755c1ba9bc4f383158c17aa9806542a2313695077c61a1efd4e1c" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "transition" + ], + "id": "shipping-5-2", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "cda0f972e5e399bce17c8f8e11c3043838aa93d9dcc1ce08c7f541ede22ba58e" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "customer" + ], + "id": "shipping-5-3", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "3dca8c0659f6893f73d801e38ec5adb08f76622b1c57afde2cee61f5691df088" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [], + "id": "shipping-5-4", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "908d101b13d170c4649dc35d2f635701124eae6715290a63f994f607072385be" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [], + "id": "shipping-5-5", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "f35e7e326becb5464d058b2665b6228c9227651e4796333f91b70a3d58745428" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "transition" + ], + "id": "shipping-6-0", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "030a1b1cc2e60d8d170c4188aef1f36d1260ddfbfc2967ab7e401a7af2c5e8b7" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "transition" + ], + "id": "shipping-6-1", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "26ce3b6c308be577fef6d94290c97f2960576f9298b4a4c97c4f04c617bfe703" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "transition" + ], + "id": "shipping-6-2", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "fab4ca9d87f8760e52e613253812e8bd6f28c794ed64f356f193ec6f86e719cb" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "customer" + ], + "id": "shipping-6-3", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "4632657e6d9306a9b026d21c442ac7902068588971663d9ce4e8a574c23d3759" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [], + "id": "shipping-6-4", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "23d6b32115270745f96f809ec564ee95c38db35b32e31f4466b972f6b74143e7" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [], + "id": "shipping-6-5", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "07d15e4d5f292fa1b1c6c48739a072f37f8463ceabad7cc45b0affc6c2b9e89d" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "transition" + ], + "id": "shipping-7-0", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "50b4b4f37292ab8a1b11febbd5436fceaad4c5b2f85f40ffc49f7c95e5403093" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "transition" + ], + "id": "shipping-7-1", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "83f992c673474985618eb5917801b666254d8d9c8c3bf8d159c7cc30bc360516" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "transition" + ], + "id": "shipping-7-2", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "fad67ed224d45f485dc30dc781ffd6f6ae313a47c34182a68b41c50587868066" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "customer" + ], + "id": "shipping-7-3", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "violation", + "validity": "valid", + "variant": null, + "variant_digest": "1d717e4f924bcc178efa928b9b9f22a0f91eae26899f65339263e9e399809bcd" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [], + "id": "shipping-7-4", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "34e85d38401fe044cc05a08e69a8f1ed38e59dfcec960383d256df3808b7137d" + }, + { + "baseline": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "reject_all" + ] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [], + "id": "shipping-7-5", + "obligation": "shipping", + "outcome": "baseline_failed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "baseline_not_accepted_and_complete", + "relation": "preserve", + "validity": "valid", + "variant": null, + "variant_digest": "185bc1e327a9770ce9f5548e1cc24f04e623fc50d963142daa1efd26e92cff7f" + } + ], + "corpus_digest": "2a36b7fadd6316e0ae0215675c9596235b0bc951457053908a7ff87135a83a0b", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Authored shipping policy.", + "id": "shipping", + "scope": "contract" + }, + { + "description": "An inventoried untested obligation.", + "id": "shipping-untested", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "baseline_failed": 48 + }, + "detected_faults": 0, + "detection_score": null, + "eligible_controls": 0, + "eligible_faults": 0, + "obligations": [ + { + "assessment": "incomplete_evidence", + "case_ids": [ + "shipping-0-0", + "shipping-0-1", + "shipping-0-2", + "shipping-0-3", + "shipping-0-4", + "shipping-0-5", + "shipping-1-0", + "shipping-1-1", + "shipping-1-2", + "shipping-1-3", + "shipping-1-4", + "shipping-1-5", + "shipping-2-0", + "shipping-2-1", + "shipping-2-2", + "shipping-2-3", + "shipping-2-4", + "shipping-2-5", + "shipping-3-0", + "shipping-3-1", + "shipping-3-2", + "shipping-3-3", + "shipping-3-4", + "shipping-3-5", + "shipping-4-0", + "shipping-4-1", + "shipping-4-2", + "shipping-4-3", + "shipping-4-4", + "shipping-4-5", + "shipping-5-0", + "shipping-5-1", + "shipping-5-2", + "shipping-5-3", + "shipping-5-4", + "shipping-5-5", + "shipping-6-0", + "shipping-6-1", + "shipping-6-2", + "shipping-6-3", + "shipping-6-4", + "shipping-6-5", + "shipping-7-0", + "shipping-7-1", + "shipping-7-2", + "shipping-7-3", + "shipping-7-4", + "shipping-7-5" + ], + "counts": { + "baseline_failed": 48 + }, + "description": "Authored shipping policy.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": true, + "has_known_gaps": false, + "id": "shipping", + "scope": "contract" + }, + { + "assessment": "untested", + "case_ids": [], + "counts": {}, + "description": "An inventoried untested obligation.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "shipping-untested", + "scope": "contract" + } + ], + "preservation_rate": null, + "preserved_controls": 0, + "total": 48 + }, + "validator_id": "controlled/shipping/reject-all/1" + }, + "shipping/reject-valid-variation": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "transition" + ], + "id": "shipping-0-0", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "6e6df86f499f432b835ce2a22e09f5da43ab6d07f64743eadcdc8e0d9bb4b38c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "transition" + ], + "id": "shipping-0-1", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "45cca396d27b8e5d76027a3b81c89f08f17ae55c549f7238ceee4407d7694708" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "transition" + ], + "id": "shipping-0-2", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "343e45827ff55aad95a876c3489f397e62d976df9d2c860d8da11a1284117ff5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "customer" + ], + "id": "shipping-0-3", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "customer" + ] + }, + "variant_digest": "916e9872ad95408281f8098491e7aefb59367d0fb61a86fc34258ccd2013a173" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [], + "id": "shipping-0-4", + "obligation": "shipping", + "outcome": "regressed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "wording" + ] + }, + "variant_digest": "2f29b5bbc788eb077edb08f8156d387014f9f7ecae4dd7ee7f650c18298a2b96" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [], + "id": "shipping-0-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b64bb7c4c05e70cb2a89ed659ae714326ebeb285a56c3f509aa62c11b4093b7b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "transition" + ], + "id": "shipping-1-0", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "ecfbb40f54baa2f0c346028075b3ac644053c8cec4f9986eaa28f08b989be879" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "transition" + ], + "id": "shipping-1-1", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "5eb74e50dc6223d94b98ecd4b7508abeb51b0603d5f5bcea77f829add60a8ad0" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "transition" + ], + "id": "shipping-1-2", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "ca13966541516b59ee62f9e6850acc3b89d65b4b96985c712308ece700d5aeef" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "customer" + ], + "id": "shipping-1-3", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "customer" + ] + }, + "variant_digest": "906744a4b69806069b187aa2d110a45480982705a905c4b980db51aec719a1a5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [], + "id": "shipping-1-4", + "obligation": "shipping", + "outcome": "regressed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "wording" + ] + }, + "variant_digest": "b60f3e4ee4774128012322db2418348c367aed8ac57f2a10727fca1d5d92856c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [], + "id": "shipping-1-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f2dcb169caa367a8998cd10f329451495430f2e25aad0180a134eceec27acb2b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "transition" + ], + "id": "shipping-2-0", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "76cff55e3c528f03f429682b032626e2ccd43a545a449d8e16ee30721163f91a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "transition" + ], + "id": "shipping-2-1", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "53751a76a524cd848711b923868de7b97d7b8bd08e2e1422ef9ad6381e22106d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "transition" + ], + "id": "shipping-2-2", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "df20de1155b45c5787d2c981474c4097d1218439901a30de8002642ac31748b2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "customer" + ], + "id": "shipping-2-3", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "customer" + ] + }, + "variant_digest": "2855c55d4c664a754eb4180f5a108d2cbdc97fcebe00977eb562d70ee7ce3e0b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [], + "id": "shipping-2-4", + "obligation": "shipping", + "outcome": "regressed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "wording" + ] + }, + "variant_digest": "3ffe4da57c5277e2eded8def8717e68666af778e6c6803eb5103b22df78e7cb9" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [], + "id": "shipping-2-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5908f595c520d4b1ac2ce88cdc26e874dd7141d5e3f75212e34bc494b48e43fc" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "transition" + ], + "id": "shipping-3-0", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "972d4c6d080e11ec44d2c099b43193ac9fc1f89a97e48269318d9aadf43d1d56" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "transition" + ], + "id": "shipping-3-1", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "94ad5dfbe87f069211ebef4a1a83d3f9e4d28be053b70c87fba822f4b54a4e86" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "transition" + ], + "id": "shipping-3-2", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "127996d11a3348937638b8b1102b729687fd2da1194fb27073ea299441d8ca5a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "customer" + ], + "id": "shipping-3-3", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "customer" + ] + }, + "variant_digest": "c3642774a8eced35b06347d7a2937217f1417b2b06f54262a3b5dfeeaf7d0de8" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [], + "id": "shipping-3-4", + "obligation": "shipping", + "outcome": "regressed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "wording" + ] + }, + "variant_digest": "0c38cd97e27be9f85d067b3caec349bbcc07ad96b6ca0b3f412925e8137cff9f" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [], + "id": "shipping-3-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0d92bcd8ee1670d27e31f14dcfd6d8802dd6c2d7ae532c33c908728814c8f20a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "transition" + ], + "id": "shipping-4-0", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "f6ab61a4db984fe1c40b21ffd802f252e12134cc1afb6933ec3ee92262a038b5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "transition" + ], + "id": "shipping-4-1", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "6219028ae1bf0898edd8c5d18d2d1846804bccb98bdc698a212edabbf4a833e2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "transition" + ], + "id": "shipping-4-2", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "7e6da2823e0745995856dd8070e761fcc6672afa6ebdf67b73a982d99c75e8ea" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "customer" + ], + "id": "shipping-4-3", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "customer" + ] + }, + "variant_digest": "e023f284af0639e13c3ed08848bd9331e8820209915ce0f6842c5a8481d29558" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [], + "id": "shipping-4-4", + "obligation": "shipping", + "outcome": "regressed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "wording" + ] + }, + "variant_digest": "480c79699756274e1c21cf037640f69232304cada155baf5a220e909b26a1d33" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [], + "id": "shipping-4-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "e5fc4b53a9af939545e40873984d3237d488479c8cd2ffce5342b5d2bff890e0" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "transition" + ], + "id": "shipping-5-0", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "0779f9da8ec7f1879162f383cd24adc4a220265e4fda2e8a8e89e9f5c6fbf72b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "transition" + ], + "id": "shipping-5-1", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "1e1e9f8e4d1755c1ba9bc4f383158c17aa9806542a2313695077c61a1efd4e1c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "transition" + ], + "id": "shipping-5-2", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "cda0f972e5e399bce17c8f8e11c3043838aa93d9dcc1ce08c7f541ede22ba58e" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "customer" + ], + "id": "shipping-5-3", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "customer" + ] + }, + "variant_digest": "3dca8c0659f6893f73d801e38ec5adb08f76622b1c57afde2cee61f5691df088" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [], + "id": "shipping-5-4", + "obligation": "shipping", + "outcome": "regressed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "wording" + ] + }, + "variant_digest": "908d101b13d170c4649dc35d2f635701124eae6715290a63f994f607072385be" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [], + "id": "shipping-5-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f35e7e326becb5464d058b2665b6228c9227651e4796333f91b70a3d58745428" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "transition" + ], + "id": "shipping-6-0", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "030a1b1cc2e60d8d170c4188aef1f36d1260ddfbfc2967ab7e401a7af2c5e8b7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "transition" + ], + "id": "shipping-6-1", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "26ce3b6c308be577fef6d94290c97f2960576f9298b4a4c97c4f04c617bfe703" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "transition" + ], + "id": "shipping-6-2", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "fab4ca9d87f8760e52e613253812e8bd6f28c794ed64f356f193ec6f86e719cb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "customer" + ], + "id": "shipping-6-3", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "customer" + ] + }, + "variant_digest": "4632657e6d9306a9b026d21c442ac7902068588971663d9ce4e8a574c23d3759" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [], + "id": "shipping-6-4", + "obligation": "shipping", + "outcome": "regressed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "wording" + ] + }, + "variant_digest": "23d6b32115270745f96f809ec564ee95c38db35b32e31f4466b972f6b74143e7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [], + "id": "shipping-6-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "07d15e4d5f292fa1b1c6c48739a072f37f8463ceabad7cc45b0affc6c2b9e89d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "transition" + ], + "id": "shipping-7-0", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "50b4b4f37292ab8a1b11febbd5436fceaad4c5b2f85f40ffc49f7c95e5403093" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "transition" + ], + "id": "shipping-7-1", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "83f992c673474985618eb5917801b666254d8d9c8c3bf8d159c7cc30bc360516" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "transition" + ], + "id": "shipping-7-2", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "transition" + ] + }, + "variant_digest": "fad67ed224d45f485dc30dc781ffd6f6ae313a47c34182a68b41c50587868066" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "customer" + ], + "id": "shipping-7-3", + "obligation": "shipping", + "outcome": "detected", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "all_expected_violations", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "customer" + ] + }, + "variant_digest": "1d717e4f924bcc178efa928b9b9f22a0f91eae26899f65339263e9e399809bcd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [], + "id": "shipping-7-4", + "obligation": "shipping", + "outcome": "regressed", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "valid_control_rejected", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "wording" + ] + }, + "variant_digest": "34e85d38401fe044cc05a08e69a8f1ed38e59dfcec960383d256df3808b7137d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [], + "id": "shipping-7-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "185bc1e327a9770ce9f5548e1cc24f04e623fc50d963142daa1efd26e92cff7f" + } + ], + "corpus_digest": "2a36b7fadd6316e0ae0215675c9596235b0bc951457053908a7ff87135a83a0b", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Authored shipping policy.", + "id": "shipping", + "scope": "contract" + }, + { + "description": "An inventoried untested obligation.", + "id": "shipping-untested", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "detected": 32, + "preserved": 8, + "regressed": 8 + }, + "detected_faults": 32, + "detection_score": 1.0, + "eligible_controls": 16, + "eligible_faults": 32, + "obligations": [ + { + "assessment": "known_gaps", + "case_ids": [ + "shipping-0-0", + "shipping-0-1", + "shipping-0-2", + "shipping-0-3", + "shipping-0-4", + "shipping-0-5", + "shipping-1-0", + "shipping-1-1", + "shipping-1-2", + "shipping-1-3", + "shipping-1-4", + "shipping-1-5", + "shipping-2-0", + "shipping-2-1", + "shipping-2-2", + "shipping-2-3", + "shipping-2-4", + "shipping-2-5", + "shipping-3-0", + "shipping-3-1", + "shipping-3-2", + "shipping-3-3", + "shipping-3-4", + "shipping-3-5", + "shipping-4-0", + "shipping-4-1", + "shipping-4-2", + "shipping-4-3", + "shipping-4-4", + "shipping-4-5", + "shipping-5-0", + "shipping-5-1", + "shipping-5-2", + "shipping-5-3", + "shipping-5-4", + "shipping-5-5", + "shipping-6-0", + "shipping-6-1", + "shipping-6-2", + "shipping-6-3", + "shipping-6-4", + "shipping-6-5", + "shipping-7-0", + "shipping-7-1", + "shipping-7-2", + "shipping-7-3", + "shipping-7-4", + "shipping-7-5" + ], + "counts": { + "detected": 32, + "preserved": 8, + "regressed": 8 + }, + "description": "Authored shipping policy.", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": true, + "has_incomplete_evidence": false, + "has_known_gaps": true, + "id": "shipping", + "scope": "contract" + }, + { + "assessment": "untested", + "case_ids": [], + "counts": {}, + "description": "An inventoried untested obligation.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "shipping-untested", + "scope": "contract" + } + ], + "preservation_rate": 0.5, + "preserved_controls": 8, + "total": 48 + }, + "validator_id": "controlled/shipping/reject-valid-variation/1" + }, + "shipping/unknown-on-fault": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "transition" + ], + "id": "shipping-0-0", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "6e6df86f499f432b835ce2a22e09f5da43ab6d07f64743eadcdc8e0d9bb4b38c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "transition" + ], + "id": "shipping-0-1", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "45cca396d27b8e5d76027a3b81c89f08f17ae55c549f7238ceee4407d7694708" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "transition" + ], + "id": "shipping-0-2", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "343e45827ff55aad95a876c3489f397e62d976df9d2c860d8da11a1284117ff5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "customer" + ], + "id": "shipping-0-3", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "916e9872ad95408281f8098491e7aefb59367d0fb61a86fc34258ccd2013a173" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [], + "id": "shipping-0-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "2f29b5bbc788eb077edb08f8156d387014f9f7ecae4dd7ee7f650c18298a2b96" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [], + "id": "shipping-0-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b64bb7c4c05e70cb2a89ed659ae714326ebeb285a56c3f509aa62c11b4093b7b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "transition" + ], + "id": "shipping-1-0", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "ecfbb40f54baa2f0c346028075b3ac644053c8cec4f9986eaa28f08b989be879" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "transition" + ], + "id": "shipping-1-1", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "5eb74e50dc6223d94b98ecd4b7508abeb51b0603d5f5bcea77f829add60a8ad0" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "transition" + ], + "id": "shipping-1-2", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "ca13966541516b59ee62f9e6850acc3b89d65b4b96985c712308ece700d5aeef" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "customer" + ], + "id": "shipping-1-3", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "906744a4b69806069b187aa2d110a45480982705a905c4b980db51aec719a1a5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [], + "id": "shipping-1-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b60f3e4ee4774128012322db2418348c367aed8ac57f2a10727fca1d5d92856c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [], + "id": "shipping-1-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f2dcb169caa367a8998cd10f329451495430f2e25aad0180a134eceec27acb2b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "transition" + ], + "id": "shipping-2-0", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "76cff55e3c528f03f429682b032626e2ccd43a545a449d8e16ee30721163f91a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "transition" + ], + "id": "shipping-2-1", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "53751a76a524cd848711b923868de7b97d7b8bd08e2e1422ef9ad6381e22106d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "transition" + ], + "id": "shipping-2-2", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "df20de1155b45c5787d2c981474c4097d1218439901a30de8002642ac31748b2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "customer" + ], + "id": "shipping-2-3", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "2855c55d4c664a754eb4180f5a108d2cbdc97fcebe00977eb562d70ee7ce3e0b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [], + "id": "shipping-2-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "3ffe4da57c5277e2eded8def8717e68666af778e6c6803eb5103b22df78e7cb9" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [], + "id": "shipping-2-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5908f595c520d4b1ac2ce88cdc26e874dd7141d5e3f75212e34bc494b48e43fc" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "transition" + ], + "id": "shipping-3-0", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "972d4c6d080e11ec44d2c099b43193ac9fc1f89a97e48269318d9aadf43d1d56" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "transition" + ], + "id": "shipping-3-1", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "94ad5dfbe87f069211ebef4a1a83d3f9e4d28be053b70c87fba822f4b54a4e86" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "transition" + ], + "id": "shipping-3-2", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "127996d11a3348937638b8b1102b729687fd2da1194fb27073ea299441d8ca5a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "customer" + ], + "id": "shipping-3-3", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "c3642774a8eced35b06347d7a2937217f1417b2b06f54262a3b5dfeeaf7d0de8" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [], + "id": "shipping-3-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0c38cd97e27be9f85d067b3caec349bbcc07ad96b6ca0b3f412925e8137cff9f" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [], + "id": "shipping-3-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0d92bcd8ee1670d27e31f14dcfd6d8802dd6c2d7ae532c33c908728814c8f20a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "transition" + ], + "id": "shipping-4-0", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "f6ab61a4db984fe1c40b21ffd802f252e12134cc1afb6933ec3ee92262a038b5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "transition" + ], + "id": "shipping-4-1", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "6219028ae1bf0898edd8c5d18d2d1846804bccb98bdc698a212edabbf4a833e2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "transition" + ], + "id": "shipping-4-2", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "7e6da2823e0745995856dd8070e761fcc6672afa6ebdf67b73a982d99c75e8ea" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "customer" + ], + "id": "shipping-4-3", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "e023f284af0639e13c3ed08848bd9331e8820209915ce0f6842c5a8481d29558" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [], + "id": "shipping-4-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "480c79699756274e1c21cf037640f69232304cada155baf5a220e909b26a1d33" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [], + "id": "shipping-4-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "e5fc4b53a9af939545e40873984d3237d488479c8cd2ffce5342b5d2bff890e0" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "transition" + ], + "id": "shipping-5-0", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "0779f9da8ec7f1879162f383cd24adc4a220265e4fda2e8a8e89e9f5c6fbf72b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "transition" + ], + "id": "shipping-5-1", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "1e1e9f8e4d1755c1ba9bc4f383158c17aa9806542a2313695077c61a1efd4e1c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "transition" + ], + "id": "shipping-5-2", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "cda0f972e5e399bce17c8f8e11c3043838aa93d9dcc1ce08c7f541ede22ba58e" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "customer" + ], + "id": "shipping-5-3", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "3dca8c0659f6893f73d801e38ec5adb08f76622b1c57afde2cee61f5691df088" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [], + "id": "shipping-5-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "908d101b13d170c4649dc35d2f635701124eae6715290a63f994f607072385be" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [], + "id": "shipping-5-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f35e7e326becb5464d058b2665b6228c9227651e4796333f91b70a3d58745428" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "transition" + ], + "id": "shipping-6-0", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "030a1b1cc2e60d8d170c4188aef1f36d1260ddfbfc2967ab7e401a7af2c5e8b7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "transition" + ], + "id": "shipping-6-1", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "26ce3b6c308be577fef6d94290c97f2960576f9298b4a4c97c4f04c617bfe703" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "transition" + ], + "id": "shipping-6-2", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "fab4ca9d87f8760e52e613253812e8bd6f28c794ed64f356f193ec6f86e719cb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "customer" + ], + "id": "shipping-6-3", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "4632657e6d9306a9b026d21c442ac7902068588971663d9ce4e8a574c23d3759" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [], + "id": "shipping-6-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "23d6b32115270745f96f809ec564ee95c38db35b32e31f4466b972f6b74143e7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [], + "id": "shipping-6-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "07d15e4d5f292fa1b1c6c48739a072f37f8463ceabad7cc45b0affc6c2b9e89d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "transition" + ], + "id": "shipping-7-0", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "50b4b4f37292ab8a1b11febbd5436fceaad4c5b2f85f40ffc49f7c95e5403093" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "transition" + ], + "id": "shipping-7-1", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "83f992c673474985618eb5917801b666254d8d9c8c3bf8d159c7cc30bc360516" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "transition" + ], + "id": "shipping-7-2", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "fad67ed224d45f485dc30dc781ffd6f6ae313a47c34182a68b41c50587868066" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "customer" + ], + "id": "shipping-7-3", + "obligation": "shipping", + "outcome": "undetermined", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "variant_evidence_incomplete", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": false, + "evidence": [], + "violations": [] + }, + "variant_digest": "1d717e4f924bcc178efa928b9b9f22a0f91eae26899f65339263e9e399809bcd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [], + "id": "shipping-7-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "34e85d38401fe044cc05a08e69a8f1ed38e59dfcec960383d256df3808b7137d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [], + "id": "shipping-7-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "185bc1e327a9770ce9f5548e1cc24f04e623fc50d963142daa1efd26e92cff7f" + } + ], + "corpus_digest": "2a36b7fadd6316e0ae0215675c9596235b0bc951457053908a7ff87135a83a0b", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Authored shipping policy.", + "id": "shipping", + "scope": "contract" + }, + { + "description": "An inventoried untested obligation.", + "id": "shipping-untested", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 16, + "undetermined": 32 + }, + "detected_faults": 0, + "detection_score": 0.0, + "eligible_controls": 16, + "eligible_faults": 32, + "obligations": [ + { + "assessment": "incomplete_evidence", + "case_ids": [ + "shipping-0-0", + "shipping-0-1", + "shipping-0-2", + "shipping-0-3", + "shipping-0-4", + "shipping-0-5", + "shipping-1-0", + "shipping-1-1", + "shipping-1-2", + "shipping-1-3", + "shipping-1-4", + "shipping-1-5", + "shipping-2-0", + "shipping-2-1", + "shipping-2-2", + "shipping-2-3", + "shipping-2-4", + "shipping-2-5", + "shipping-3-0", + "shipping-3-1", + "shipping-3-2", + "shipping-3-3", + "shipping-3-4", + "shipping-3-5", + "shipping-4-0", + "shipping-4-1", + "shipping-4-2", + "shipping-4-3", + "shipping-4-4", + "shipping-4-5", + "shipping-5-0", + "shipping-5-1", + "shipping-5-2", + "shipping-5-3", + "shipping-5-4", + "shipping-5-5", + "shipping-6-0", + "shipping-6-1", + "shipping-6-2", + "shipping-6-3", + "shipping-6-4", + "shipping-6-5", + "shipping-7-0", + "shipping-7-1", + "shipping-7-2", + "shipping-7-3", + "shipping-7-4", + "shipping-7-5" + ], + "counts": { + "preserved": 16, + "undetermined": 32 + }, + "description": "Authored shipping policy.", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": true, + "has_incomplete_evidence": true, + "has_known_gaps": false, + "id": "shipping", + "scope": "contract" + }, + { + "assessment": "untested", + "case_ids": [], + "counts": {}, + "description": "An inventoried untested obligation.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "shipping-untested", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 16, + "total": 48 + }, + "validator_id": "controlled/shipping/unknown-on-fault/1" + }, + "shipping/wrong-diagnostic-ID": { + "cases": [ + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "transition" + ], + "id": "shipping-0-0", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "6e6df86f499f432b835ce2a22e09f5da43ab6d07f64743eadcdc8e0d9bb4b38c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "transition" + ], + "id": "shipping-0-1", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "45cca396d27b8e5d76027a3b81c89f08f17ae55c549f7238ceee4407d7694708" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "transition" + ], + "id": "shipping-0-2", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "343e45827ff55aad95a876c3489f397e62d976df9d2c860d8da11a1284117ff5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [ + "customer" + ], + "id": "shipping-0-3", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "916e9872ad95408281f8098491e7aefb59367d0fb61a86fc34258ccd2013a173" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [], + "id": "shipping-0-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "2f29b5bbc788eb077edb08f8156d387014f9f7ecae4dd7ee7f650c18298a2b96" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "d5b2b9ccb0cdfda127a1d260a7076c2c9ffda822eb214d45089012bd17da435a", + "expected": [], + "id": "shipping-0-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b64bb7c4c05e70cb2a89ed659ae714326ebeb285a56c3f509aa62c11b4093b7b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "transition" + ], + "id": "shipping-1-0", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "ecfbb40f54baa2f0c346028075b3ac644053c8cec4f9986eaa28f08b989be879" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "transition" + ], + "id": "shipping-1-1", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "5eb74e50dc6223d94b98ecd4b7508abeb51b0603d5f5bcea77f829add60a8ad0" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "transition" + ], + "id": "shipping-1-2", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "ca13966541516b59ee62f9e6850acc3b89d65b4b96985c712308ece700d5aeef" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [ + "customer" + ], + "id": "shipping-1-3", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "906744a4b69806069b187aa2d110a45480982705a905c4b980db51aec719a1a5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [], + "id": "shipping-1-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "b60f3e4ee4774128012322db2418348c367aed8ac57f2a10727fca1d5d92856c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "7f8903e83541f61ff73545c2c3a2bdd8945e33bd2533a646d11bffe4d668eeb5", + "expected": [], + "id": "shipping-1-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f2dcb169caa367a8998cd10f329451495430f2e25aad0180a134eceec27acb2b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "transition" + ], + "id": "shipping-2-0", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "76cff55e3c528f03f429682b032626e2ccd43a545a449d8e16ee30721163f91a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "transition" + ], + "id": "shipping-2-1", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "53751a76a524cd848711b923868de7b97d7b8bd08e2e1422ef9ad6381e22106d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "transition" + ], + "id": "shipping-2-2", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "df20de1155b45c5787d2c981474c4097d1218439901a30de8002642ac31748b2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [ + "customer" + ], + "id": "shipping-2-3", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "2855c55d4c664a754eb4180f5a108d2cbdc97fcebe00977eb562d70ee7ce3e0b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [], + "id": "shipping-2-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "3ffe4da57c5277e2eded8def8717e68666af778e6c6803eb5103b22df78e7cb9" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "461d1e5760062064658e30593ed99c52f2c7f6b9dcab5c023d9d90cab4af0afe", + "expected": [], + "id": "shipping-2-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "5908f595c520d4b1ac2ce88cdc26e874dd7141d5e3f75212e34bc494b48e43fc" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "transition" + ], + "id": "shipping-3-0", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "972d4c6d080e11ec44d2c099b43193ac9fc1f89a97e48269318d9aadf43d1d56" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "transition" + ], + "id": "shipping-3-1", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "94ad5dfbe87f069211ebef4a1a83d3f9e4d28be053b70c87fba822f4b54a4e86" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "transition" + ], + "id": "shipping-3-2", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "127996d11a3348937638b8b1102b729687fd2da1194fb27073ea299441d8ca5a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [ + "customer" + ], + "id": "shipping-3-3", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "c3642774a8eced35b06347d7a2937217f1417b2b06f54262a3b5dfeeaf7d0de8" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [], + "id": "shipping-3-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0c38cd97e27be9f85d067b3caec349bbcc07ad96b6ca0b3f412925e8137cff9f" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "4d66a8d354f85470c7fe2ae6501e0f7b2104279961ea2f1328ad9cad13626105", + "expected": [], + "id": "shipping-3-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "0d92bcd8ee1670d27e31f14dcfd6d8802dd6c2d7ae532c33c908728814c8f20a" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "transition" + ], + "id": "shipping-4-0", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "f6ab61a4db984fe1c40b21ffd802f252e12134cc1afb6933ec3ee92262a038b5" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "transition" + ], + "id": "shipping-4-1", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "6219028ae1bf0898edd8c5d18d2d1846804bccb98bdc698a212edabbf4a833e2" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "transition" + ], + "id": "shipping-4-2", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "7e6da2823e0745995856dd8070e761fcc6672afa6ebdf67b73a982d99c75e8ea" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [ + "customer" + ], + "id": "shipping-4-3", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "e023f284af0639e13c3ed08848bd9331e8820209915ce0f6842c5a8481d29558" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [], + "id": "shipping-4-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "480c79699756274e1c21cf037640f69232304cada155baf5a220e909b26a1d33" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "079a44390649beb03d04b3a56b0108c960aaf22b46c48d7c51157a9f50195488", + "expected": [], + "id": "shipping-4-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "e5fc4b53a9af939545e40873984d3237d488479c8cd2ffce5342b5d2bff890e0" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "transition" + ], + "id": "shipping-5-0", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "0779f9da8ec7f1879162f383cd24adc4a220265e4fda2e8a8e89e9f5c6fbf72b" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "transition" + ], + "id": "shipping-5-1", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "1e1e9f8e4d1755c1ba9bc4f383158c17aa9806542a2313695077c61a1efd4e1c" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "transition" + ], + "id": "shipping-5-2", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "cda0f972e5e399bce17c8f8e11c3043838aa93d9dcc1ce08c7f541ede22ba58e" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [ + "customer" + ], + "id": "shipping-5-3", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "3dca8c0659f6893f73d801e38ec5adb08f76622b1c57afde2cee61f5691df088" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [], + "id": "shipping-5-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "908d101b13d170c4649dc35d2f635701124eae6715290a63f994f607072385be" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "b66203c924aea51b9d629a8b86d02749aa98ac18066fcf98c64169fc98606a10", + "expected": [], + "id": "shipping-5-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "f35e7e326becb5464d058b2665b6228c9227651e4796333f91b70a3d58745428" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "transition" + ], + "id": "shipping-6-0", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "030a1b1cc2e60d8d170c4188aef1f36d1260ddfbfc2967ab7e401a7af2c5e8b7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "transition" + ], + "id": "shipping-6-1", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "26ce3b6c308be577fef6d94290c97f2960576f9298b4a4c97c4f04c617bfe703" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "transition" + ], + "id": "shipping-6-2", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "fab4ca9d87f8760e52e613253812e8bd6f28c794ed64f356f193ec6f86e719cb" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [ + "customer" + ], + "id": "shipping-6-3", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "4632657e6d9306a9b026d21c442ac7902068588971663d9ce4e8a574c23d3759" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [], + "id": "shipping-6-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "23d6b32115270745f96f809ec564ee95c38db35b32e31f4466b972f6b74143e7" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "20821d9bb27d2fd4e1770f477417c125ea4af2051e439be9ae77939d500fec91", + "expected": [], + "id": "shipping-6-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "07d15e4d5f292fa1b1c6c48739a072f37f8463ceabad7cc45b0affc6c2b9e89d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "transition" + ], + "id": "shipping-7-0", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "50b4b4f37292ab8a1b11febbd5436fceaad4c5b2f85f40ffc49f7c95e5403093" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "transition" + ], + "id": "shipping-7-1", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "83f992c673474985618eb5917801b666254d8d9c8c3bf8d159c7cc30bc360516" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "transition" + ], + "id": "shipping-7-2", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "fad67ed224d45f485dc30dc781ffd6f6ae313a47c34182a68b41c50587868066" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [ + "customer" + ], + "id": "shipping-7-3", + "obligation": "shipping", + "outcome": "unattributed_rejection", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "expected_violations_missing", + "relation": "violation", + "validity": "valid", + "variant": { + "accepted": false, + "complete": true, + "evidence": [], + "violations": [ + "unrelated_reason" + ] + }, + "variant_digest": "1d717e4f924bcc178efa928b9b9f22a0f91eae26899f65339263e9e399809bcd" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [], + "id": "shipping-7-4", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "34e85d38401fe044cc05a08e69a8f1ed38e59dfcec960383d256df3808b7137d" + }, + { + "baseline": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "baseline_digest": "567d6126af421084cc4363d3d28f0bc265e5f03ec7a99db0369330f74d43edee", + "expected": [], + "id": "shipping-7-5", + "obligation": "shipping", + "outcome": "preserved", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "reason": "accepted_valid_control", + "relation": "preserve", + "validity": "valid", + "variant": { + "accepted": true, + "complete": true, + "evidence": [], + "violations": [] + }, + "variant_digest": "185bc1e327a9770ce9f5548e1cc24f04e623fc50d963142daa1efd26e92cff7f" + } + ], + "corpus_digest": "2a36b7fadd6316e0ae0215675c9596235b0bc951457053908a7ff87135a83a0b", + "library_version": "0.1.0a3", + "obligations": [ + { + "description": "Authored shipping policy.", + "id": "shipping", + "scope": "contract" + }, + { + "description": "An inventoried untested obligation.", + "id": "shipping-untested", + "scope": "contract" + } + ], + "provenance": null, + "schema_version": 2, + "summary": { + "counts": { + "preserved": 16, + "unattributed_rejection": 32 + }, + "detected_faults": 0, + "detection_score": 0.0, + "eligible_controls": 16, + "eligible_faults": 32, + "obligations": [ + { + "assessment": "known_gaps", + "case_ids": [ + "shipping-0-0", + "shipping-0-1", + "shipping-0-2", + "shipping-0-3", + "shipping-0-4", + "shipping-0-5", + "shipping-1-0", + "shipping-1-1", + "shipping-1-2", + "shipping-1-3", + "shipping-1-4", + "shipping-1-5", + "shipping-2-0", + "shipping-2-1", + "shipping-2-2", + "shipping-2-3", + "shipping-2-4", + "shipping-2-5", + "shipping-3-0", + "shipping-3-1", + "shipping-3-2", + "shipping-3-3", + "shipping-3-4", + "shipping-3-5", + "shipping-4-0", + "shipping-4-1", + "shipping-4-2", + "shipping-4-3", + "shipping-4-4", + "shipping-4-5", + "shipping-5-0", + "shipping-5-1", + "shipping-5-2", + "shipping-5-3", + "shipping-5-4", + "shipping-5-5", + "shipping-6-0", + "shipping-6-1", + "shipping-6-2", + "shipping-6-3", + "shipping-6-4", + "shipping-6-5", + "shipping-7-0", + "shipping-7-1", + "shipping-7-2", + "shipping-7-3", + "shipping-7-4", + "shipping-7-5" + ], + "counts": { + "preserved": 16, + "unattributed_rejection": 32 + }, + "description": "Authored shipping policy.", + "has_controls": true, + "has_exclusions": false, + "has_fault_tests": true, + "has_incomplete_evidence": false, + "has_known_gaps": true, + "id": "shipping", + "scope": "contract" + }, + { + "assessment": "untested", + "case_ids": [], + "counts": {}, + "description": "An inventoried untested obligation.", + "has_controls": false, + "has_exclusions": false, + "has_fault_tests": false, + "has_incomplete_evidence": false, + "has_known_gaps": false, + "id": "shipping-untested", + "scope": "contract" + } + ], + "preservation_rate": 1.0, + "preserved_controls": 16, + "total": 48 + }, + "validator_id": "controlled/shipping/wrong-diagnostic-ID/1" + } +} diff --git a/benchmarks/validator-study/evaluation/comparisons.json b/benchmarks/validator-study/evaluation/comparisons.json new file mode 100644 index 0000000..2f3dd2b --- /dev/null +++ b/benchmarks/validator-study/evaluation/comparisons.json @@ -0,0 +1,11522 @@ +[ + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-0", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-1", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-2", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-3", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-4", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-5", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-0", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-1", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-2", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-3", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-4", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-5", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-0", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-1", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-2", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-3", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-4", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-5", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-0", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-1", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-2", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-3", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-4", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-5", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-0", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-1", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-2", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-3", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-4", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-5", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-0", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-1", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-2", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-3", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-4", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-5", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-0", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-1", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-2", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-3", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-4", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-5", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-0", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-1", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-2", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-3", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-4", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-5", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-0", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-1", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-2", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-3", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-4", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-5", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-0", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-1", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-2", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-3", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-4", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-5", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-0", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-1", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-2", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-3", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-4", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-5", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-0", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-1", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-2", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-3", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-4", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-5", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-0", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-1", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-2", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-3", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-4", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-5", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-0", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-1", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-2", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-3", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-4", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-5", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-0", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-1", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-2", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-3", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-4", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-5", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-0", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-1", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-2", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-3", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-4", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-5", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-0", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-1", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-2", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-3", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-4", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-5", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-0", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-1", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-2", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-3", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-4", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-5", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-0", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-1", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-2", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-3", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-4", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-5", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-0", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-1", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-2", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-3", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-4", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-5", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-0", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-1", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-2", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-3", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-4", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-5", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-0", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-1", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-2", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-3", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-4", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-5", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-0", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-1", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-2", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-3", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-4", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-5", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-0", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-1", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-2", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-3", + "profile": "correct", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-4", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-5", + "profile": "correct", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-0", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-1", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-2", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-3", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-4", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-5", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-0", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-1", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-2", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-3", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-4", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-5", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-0", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-1", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-2", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-3", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-4", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-5", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-0", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-1", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-2", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-3", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-4", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-5", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-0", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-1", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-2", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-3", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-4", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-5", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-0", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-1", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-2", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-3", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-4", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-5", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-0", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-1", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-2", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-3", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-4", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-5", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-0", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-1", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-2", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-3", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-4", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-5", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-0", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-1", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-2", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-3", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-4", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-5", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-0", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-1", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-2", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-3", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-4", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-5", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-0", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-1", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-2", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-3", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-4", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-5", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-0", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-1", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-2", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-3", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-4", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-5", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-0", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-1", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-2", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-3", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-4", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-5", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-0", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-1", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-2", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-3", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-4", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-5", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-0", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-1", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-2", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-3", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-4", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-5", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-0", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-1", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-2", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-3", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-4", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-5", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-0", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-1", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-2", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-3", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-4", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-5", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-0", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-1", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-2", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-3", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-4", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-5", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-0", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-1", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-2", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-3", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-4", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-5", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-0", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-1", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-2", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-3", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-4", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-5", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-0", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-1", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-2", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-3", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-4", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-5", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-0", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-1", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-2", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-3", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-4", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-5", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-0", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-1", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-2", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-3", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-4", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-5", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-0", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-1", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-2", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-3", + "profile": "omit-first-obligation", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-4", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-5", + "profile": "omit-first-obligation", + "relation": "preserve" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-0", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-1", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-2", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-3", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-4", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-5", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-0", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-1", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-2", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-3", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-4", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-5", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-0", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-1", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-2", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-3", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-4", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-5", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-0", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-1", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-2", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-3", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-4", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-5", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-0", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-1", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-2", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-3", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-4", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-5", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-0", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-1", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-2", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-3", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-4", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-5", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-0", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-1", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-2", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-3", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-4", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-5", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-0", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-1", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-2", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-3", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-4", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-5", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-0", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-1", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-2", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-3", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-4", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-5", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-0", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-1", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-2", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-3", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-4", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-5", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-0", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-1", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-2", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-3", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-4", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-5", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-0", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-1", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-2", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-3", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-4", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-5", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-0", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-1", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-2", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-3", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-4", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-5", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-0", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-1", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-2", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-3", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-4", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-5", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-0", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-1", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-2", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-3", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-4", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-5", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-0", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-1", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-2", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-3", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-4", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-5", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-0", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-1", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-2", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-3", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-4", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-5", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-0", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-1", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-2", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-3", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-4", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-5", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-0", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-1", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-2", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-3", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-4", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-5", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-0", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-1", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-2", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-3", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-4", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-5", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-0", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-1", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-2", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-3", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-4", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-5", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-0", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-1", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-2", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-3", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-4", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-5", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-0", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-1", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-2", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-3", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-4", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-5", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-0", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-1", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-2", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "unattributed_rejection", + "boolean": "detected", + "detailed_pytest": "unattributed_rejection", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-3", + "profile": "wrong-diagnostic-ID", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-4", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-5", + "profile": "wrong-diagnostic-ID", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-0", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-1", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-2", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-3", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "regressed", + "boolean": "regressed", + "detailed_pytest": "regressed", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-4", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-5", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-0", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-1", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-2", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-3", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "regressed", + "boolean": "regressed", + "detailed_pytest": "regressed", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-4", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-5", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-0", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-1", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-2", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-3", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "regressed", + "boolean": "regressed", + "detailed_pytest": "regressed", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-4", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-5", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-0", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-1", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-2", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-3", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "regressed", + "boolean": "regressed", + "detailed_pytest": "regressed", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-4", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-5", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-0", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-1", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-2", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-3", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "regressed", + "boolean": "regressed", + "detailed_pytest": "regressed", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-4", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-5", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-0", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-1", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-2", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-3", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "regressed", + "boolean": "regressed", + "detailed_pytest": "regressed", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-4", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-5", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-0", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-1", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-2", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-3", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "regressed", + "boolean": "regressed", + "detailed_pytest": "regressed", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-4", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-5", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-0", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-1", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-2", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-3", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "regressed", + "boolean": "regressed", + "detailed_pytest": "regressed", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-4", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-5", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-0", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-1", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-2", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-3", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "regressed", + "boolean": "regressed", + "detailed_pytest": "regressed", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-4", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-5", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-0", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-1", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-2", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-3", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "regressed", + "boolean": "regressed", + "detailed_pytest": "regressed", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-4", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-5", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-0", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-1", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-2", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-3", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "regressed", + "boolean": "regressed", + "detailed_pytest": "regressed", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-4", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-5", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-0", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-1", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-2", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-3", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "regressed", + "boolean": "regressed", + "detailed_pytest": "regressed", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-4", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-5", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-0", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-1", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-2", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-3", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "regressed", + "boolean": "regressed", + "detailed_pytest": "regressed", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-4", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-5", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-0", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-1", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-2", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-3", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "regressed", + "boolean": "regressed", + "detailed_pytest": "regressed", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-4", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-5", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-0", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-1", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-2", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-3", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "regressed", + "boolean": "regressed", + "detailed_pytest": "regressed", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-4", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-5", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-0", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-1", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-2", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-3", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "regressed", + "boolean": "regressed", + "detailed_pytest": "regressed", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-4", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-5", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-0", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-1", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-2", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-3", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "regressed", + "boolean": "regressed", + "detailed_pytest": "regressed", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-4", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-5", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-0", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-1", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-2", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-3", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "regressed", + "boolean": "regressed", + "detailed_pytest": "regressed", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-4", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-5", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-0", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-1", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-2", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-3", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "regressed", + "boolean": "regressed", + "detailed_pytest": "regressed", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-4", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-5", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-0", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-1", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-2", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-3", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "regressed", + "boolean": "regressed", + "detailed_pytest": "regressed", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-4", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-5", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-0", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-1", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-2", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-3", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "regressed", + "boolean": "regressed", + "detailed_pytest": "regressed", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-4", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-5", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-0", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-1", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-2", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-3", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "regressed", + "boolean": "regressed", + "detailed_pytest": "regressed", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-4", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-5", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-0", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-1", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-2", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-3", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "regressed", + "boolean": "regressed", + "detailed_pytest": "regressed", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-4", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-5", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-0", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-1", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-2", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "detected", + "boolean": "detected", + "detailed_pytest": "detected", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-3", + "profile": "reject-valid-variation", + "relation": "violation" + }, + { + "audit": "regressed", + "boolean": "regressed", + "detailed_pytest": "regressed", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-4", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-5", + "profile": "reject-valid-variation", + "relation": "preserve" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-0", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-1", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-2", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-3", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-4", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-5", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-0", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-1", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-2", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-3", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-4", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-5", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-0", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-1", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-2", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-3", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-4", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-5", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-0", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-1", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-2", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-3", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-4", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-5", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-0", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-1", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-2", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-3", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-4", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-5", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-0", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-1", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-2", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-3", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-4", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-5", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-0", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-1", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-2", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-3", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-4", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-5", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-0", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-1", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-2", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-3", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-4", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-5", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-0", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-1", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-2", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-3", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-4", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-5", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-0", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-1", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-2", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-3", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-4", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-5", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-0", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-1", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-2", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-3", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-4", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-5", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-0", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-1", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-2", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-3", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-4", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-5", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-0", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-1", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-2", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-3", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-4", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-5", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-0", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-1", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-2", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-3", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-4", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-5", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-0", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-1", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-2", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-3", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-4", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-5", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-0", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-1", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-2", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-3", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-4", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-5", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-0", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-1", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-2", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-3", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-4", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-5", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-0", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-1", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-2", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-3", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-4", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-5", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-0", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-1", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-2", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-3", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-4", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-5", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-0", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-1", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-2", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-3", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-4", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-5", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-0", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-1", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-2", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-3", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-4", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-5", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-0", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-1", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-2", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-3", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-4", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-5", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-0", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-1", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-2", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-3", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-4", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-5", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-0", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-1", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-2", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "error", + "boolean": "error", + "detailed_pytest": "error", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-3", + "profile": "crash-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-4", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-5", + "profile": "crash-on-fault", + "relation": "preserve" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-0", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-1", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-2", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-3", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-4", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-5", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-0", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-1", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-2", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-3", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-4", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-5", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-0", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-1", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-2", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-3", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-4", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-5", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-0", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-1", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-2", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-3", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-4", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-5", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-0", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-1", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-2", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-3", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-4", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-5", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-0", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-1", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-2", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-3", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-4", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-5", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-0", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-1", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-2", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-3", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-4", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-5", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-0", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-1", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-2", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-3", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-4", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-5", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-0", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-1", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-2", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-3", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-4", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-5", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-0", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-1", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-2", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-3", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-4", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-5", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-0", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-1", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-2", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-3", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-4", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-5", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-0", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-1", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-2", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-3", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-4", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-5", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-0", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-1", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-2", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-3", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-4", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-5", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-0", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-1", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-2", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-3", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-4", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-5", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-0", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-1", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-2", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-3", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-4", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-5", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-0", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-1", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-2", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-3", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-4", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-5", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-0", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-1", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-2", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-3", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-4", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-5", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-0", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-1", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-2", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-3", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-4", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-5", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-0", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-1", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-2", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-3", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-4", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-5", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-0", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-1", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-2", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-3", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-4", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-5", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-0", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-1", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-2", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-3", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-4", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-5", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-0", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-1", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-2", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-3", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-4", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-5", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-0", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-1", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-2", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-3", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-4", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-5", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-0", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-1", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-2", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "undetermined", + "boolean": "undetermined", + "detailed_pytest": "undetermined", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-3", + "profile": "unknown-on-fault", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-4", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-5", + "profile": "unknown-on-fault", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-0", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-1", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-2", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-3", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-4", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-5", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-0", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-1", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-2", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-3", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-4", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-5", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-0", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-1", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-2", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-3", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-4", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-5", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-0", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-1", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-2", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-3", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-4", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-5", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-0", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-1", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-2", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-3", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-4", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-5", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-0", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-1", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-2", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-3", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-4", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-5", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-0", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-1", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-2", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-3", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-4", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-5", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-0", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-1", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-2", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-3", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-4", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-5", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-0", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-1", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-2", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-3", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-4", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-5", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-0", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-1", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-2", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-3", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-4", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-5", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-0", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-1", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-2", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-3", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-4", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-5", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-0", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-1", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-2", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-3", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-4", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-5", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-0", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-1", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-2", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-3", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-4", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-5", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-0", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-1", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-2", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-3", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-4", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-5", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-0", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-1", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-2", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-3", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-4", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-5", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-0", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-1", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-2", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-3", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-4", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-5", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-0", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-1", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-2", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-3", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-4", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-5", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-0", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-1", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-2", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-3", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-4", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-5", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-0", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-1", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-2", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-3", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-4", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-5", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-0", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-1", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-2", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-3", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-4", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-5", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-0", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-1", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-2", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-3", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-4", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-5", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-0", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-1", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-2", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-3", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-4", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-5", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-0", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-1", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-2", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-3", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-4", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-5", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-0", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-1", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-2", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "survived", + "boolean": "survived", + "detailed_pytest": "survived", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-3", + "profile": "accept-all", + "relation": "violation" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-4", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "preserved", + "boolean": "preserved", + "detailed_pytest": "preserved", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-5", + "profile": "accept-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-0", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-1", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-2", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-3", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-4", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-0", + "id": "refund-0-5", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-0", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-1", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-2", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-3", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-4", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-1", + "id": "refund-1-5", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-0", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-1", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-2", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-3", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-4", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-2", + "id": "refund-2-5", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-0", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-1", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-2", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-3", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-4", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-3", + "id": "refund-3-5", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-0", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-1", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-2", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-3", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-4", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-4", + "id": "refund-4-5", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-0", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-1", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-2", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-3", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-4", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-5", + "id": "refund-5-5", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-0", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-1", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-2", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-3", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-4", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-6", + "id": "refund-6-5", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-0", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-1", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-2", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-3", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-4", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "refund", + "group": "refund-7", + "id": "refund-7-5", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-0", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-1", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-2", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-3", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-4", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-0", + "id": "shipping-0-5", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-0", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-1", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-2", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-3", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-4", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-1", + "id": "shipping-1-5", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-0", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-1", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-2", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-3", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-4", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-2", + "id": "shipping-2-5", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-0", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-1", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-2", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-3", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-4", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-3", + "id": "shipping-3-5", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-0", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-1", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-2", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-3", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-4", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-4", + "id": "shipping-4-5", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-0", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-1", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-2", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-3", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-4", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-5", + "id": "shipping-5-5", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-0", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-1", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-2", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-3", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-4", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-6", + "id": "shipping-6-5", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-0", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-1", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-2", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-3", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-4", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "shipping", + "group": "shipping-7", + "id": "shipping-7-5", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-0", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-1", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-2", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-3", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-4", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-0", + "id": "reservation-0-5", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-0", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-1", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-2", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-3", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-4", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-1", + "id": "reservation-1-5", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-0", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-1", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-2", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-3", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-4", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-2", + "id": "reservation-2-5", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-0", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-1", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-2", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-3", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-4", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-3", + "id": "reservation-3-5", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-0", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-1", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-2", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-3", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-4", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-4", + "id": "reservation-4-5", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-0", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-1", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-2", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-3", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-4", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-5", + "id": "reservation-5-5", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-0", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-1", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-2", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-3", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-4", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-6", + "id": "reservation-6-5", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-0", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-1", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-2", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-3", + "profile": "reject-all", + "relation": "violation" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-4", + "profile": "reject-all", + "relation": "preserve" + }, + { + "audit": "baseline_failed", + "boolean": "baseline_failed", + "detailed_pytest": "baseline_failed", + "domain": "reservation", + "group": "reservation-7", + "id": "reservation-7-5", + "profile": "reject-all", + "relation": "preserve" + } +] diff --git a/benchmarks/validator-study/evaluation/environment.json b/benchmarks/validator-study/evaluation/environment.json new file mode 100644 index 0000000..16cc6bf --- /dev/null +++ b/benchmarks/validator-study/evaluation/environment.json @@ -0,0 +1,4 @@ +{ + "library_version": "0.1.0a3", + "python": "3.12.13" +} diff --git a/benchmarks/validator-study/evaluation/manifest.json b/benchmarks/validator-study/evaluation/manifest.json new file mode 100644 index 0000000..1a4933c --- /dev/null +++ b/benchmarks/validator-study/evaluation/manifest.json @@ -0,0 +1,4 @@ +{ + "observations_sha256": "e65627be00ab2b214bb362d0d7d513ef137e4b878cae9b58ac32edd68ba7b737", + "prepared_manifest_sha256": "eefacd6f3a10da366f2556f0009191b482d32d944c0137eeb017669fae034c62" +} diff --git a/benchmarks/validator-study/evaluation/observations.json b/benchmarks/validator-study/evaluation/observations.json new file mode 100644 index 0000000..031f896 --- /dev/null +++ b/benchmarks/validator-study/evaluation/observations.json @@ -0,0 +1,17106 @@ +{ + "accept-all": { + "refund-0-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-0-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-0-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-0-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-0-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-0-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-1-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-1-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-1-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-1-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-1-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-1-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-2-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-2-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-2-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-2-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-2-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-2-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-3-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-3-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-3-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-3-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-3-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-3-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-4-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-4-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-4-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-4-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-4-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-4-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-5-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-5-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-5-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-5-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-5-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-5-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-6-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-6-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-6-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-6-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-6-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-6-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-7-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-7-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-7-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-7-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-7-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-7-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-0-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-0-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-0-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-0-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-0-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-0-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-1-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-1-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-1-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-1-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-1-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-1-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-2-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-2-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-2-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-2-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-2-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-2-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-3-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-3-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-3-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-3-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-3-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-3-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-4-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-4-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-4-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-4-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-4-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-4-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-5-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-5-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-5-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-5-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-5-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-5-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-6-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-6-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-6-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-6-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-6-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-6-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-7-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-7-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-7-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-7-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-7-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-7-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-0-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-0-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-0-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-0-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-0-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-0-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-1-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-1-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-1-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-1-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-1-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-1-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-2-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-2-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-2-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-2-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-2-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-2-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-3-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-3-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-3-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-3-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-3-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-3-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-4-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-4-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-4-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-4-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-4-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-4-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-5-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-5-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-5-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-5-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-5-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-5-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-6-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-6-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-6-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-6-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-6-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-6-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-7-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-7-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-7-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-7-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-7-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-7-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + } + }, + "correct": { + "refund-0-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-0-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-0-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-0-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "receipt_request" + ] + } + }, + "refund-0-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-0-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-1-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-1-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-1-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-1-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "receipt_request" + ] + } + }, + "refund-1-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-1-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-2-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-2-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-2-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-2-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "receipt_request" + ] + } + }, + "refund-2-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-2-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-3-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-3-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-3-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-3-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "receipt_request" + ] + } + }, + "refund-3-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-3-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-4-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-4-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-4-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-4-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "receipt_request" + ] + } + }, + "refund-4-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-4-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-5-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-5-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-5-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-5-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "receipt_request" + ] + } + }, + "refund-5-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-5-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-6-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-6-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-6-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-6-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "receipt_request" + ] + } + }, + "refund-6-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-6-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-7-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-7-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-7-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-7-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "receipt_request" + ] + } + }, + "refund-7-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-7-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-0-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-0-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-0-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-0-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "resource" + ] + } + }, + "reservation-0-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-0-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-1-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-1-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-1-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-1-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "resource" + ] + } + }, + "reservation-1-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-1-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-2-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-2-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-2-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-2-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "resource" + ] + } + }, + "reservation-2-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-2-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-3-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-3-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-3-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-3-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "resource" + ] + } + }, + "reservation-3-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-3-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-4-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-4-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-4-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-4-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "resource" + ] + } + }, + "reservation-4-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-4-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-5-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-5-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-5-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-5-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "resource" + ] + } + }, + "reservation-5-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-5-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-6-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-6-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-6-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-6-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "resource" + ] + } + }, + "reservation-6-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-6-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-7-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-7-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-7-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-7-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "resource" + ] + } + }, + "reservation-7-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-7-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-0-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-0-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-0-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-0-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "customer" + ] + } + }, + "shipping-0-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-0-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-1-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-1-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-1-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-1-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "customer" + ] + } + }, + "shipping-1-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-1-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-2-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-2-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-2-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-2-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "customer" + ] + } + }, + "shipping-2-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-2-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-3-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-3-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-3-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-3-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "customer" + ] + } + }, + "shipping-3-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-3-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-4-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-4-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-4-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-4-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "customer" + ] + } + }, + "shipping-4-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-4-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-5-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-5-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-5-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-5-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "customer" + ] + } + }, + "shipping-5-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-5-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-6-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-6-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-6-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-6-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "customer" + ] + } + }, + "shipping-6-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-6-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-7-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-7-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-7-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-7-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "customer" + ] + } + }, + "shipping-7-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-7-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + } + }, + "crash-on-fault": { + "refund-0-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-0-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-0-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-0-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-0-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-0-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-1-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-1-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-1-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-1-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-1-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-1-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-2-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-2-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-2-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-2-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-2-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-2-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-3-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-3-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-3-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-3-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-3-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-3-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-4-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-4-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-4-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-4-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-4-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-4-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-5-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-5-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-5-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-5-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-5-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-5-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-6-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-6-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-6-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-6-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-6-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-6-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-7-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-7-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-7-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-7-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "refund-7-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-7-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-0-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-0-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-0-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-0-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-0-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-0-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-1-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-1-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-1-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-1-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-1-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-1-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-2-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-2-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-2-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-2-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-2-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-2-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-3-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-3-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-3-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-3-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-3-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-3-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-4-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-4-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-4-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-4-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-4-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-4-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-5-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-5-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-5-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-5-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-5-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-5-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-6-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-6-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-6-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-6-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-6-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-6-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-7-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-7-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-7-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-7-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "reservation-7-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-7-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-0-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-0-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-0-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-0-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-0-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-0-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-1-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-1-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-1-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-1-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-1-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-1-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-2-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-2-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-2-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-2-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-2-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-2-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-3-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-3-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-3-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-3-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-3-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-3-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-4-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-4-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-4-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-4-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-4-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-4-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-5-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-5-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-5-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-5-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-5-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-5-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-6-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-6-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-6-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-6-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-6-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-6-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-7-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-7-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-7-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-7-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "error": "RuntimeError: authored validator crash" + } + }, + "shipping-7-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-7-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + } + }, + "omit-first-obligation": { + "refund-0-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-0-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-0-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-0-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "receipt_request" + ] + } + }, + "refund-0-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-0-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-1-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-1-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-1-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-1-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "receipt_request" + ] + } + }, + "refund-1-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-1-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-2-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-2-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-2-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-2-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "receipt_request" + ] + } + }, + "refund-2-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-2-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-3-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-3-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-3-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-3-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "receipt_request" + ] + } + }, + "refund-3-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-3-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-4-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-4-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-4-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-4-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "receipt_request" + ] + } + }, + "refund-4-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-4-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-5-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-5-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-5-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-5-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "receipt_request" + ] + } + }, + "refund-5-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-5-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-6-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-6-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-6-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-6-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "receipt_request" + ] + } + }, + "refund-6-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-6-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-7-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-7-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-7-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-7-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "receipt_request" + ] + } + }, + "refund-7-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-7-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-0-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-0-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-0-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-0-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "resource" + ] + } + }, + "reservation-0-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-0-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-1-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-1-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-1-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-1-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "resource" + ] + } + }, + "reservation-1-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-1-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-2-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-2-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-2-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-2-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "resource" + ] + } + }, + "reservation-2-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-2-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-3-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-3-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-3-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-3-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "resource" + ] + } + }, + "reservation-3-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-3-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-4-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-4-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-4-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-4-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "resource" + ] + } + }, + "reservation-4-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-4-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-5-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-5-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-5-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-5-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "resource" + ] + } + }, + "reservation-5-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-5-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-6-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-6-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-6-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-6-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "resource" + ] + } + }, + "reservation-6-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-6-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-7-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-7-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-7-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-7-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "resource" + ] + } + }, + "reservation-7-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-7-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-0-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-0-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-0-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-0-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "customer" + ] + } + }, + "shipping-0-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-0-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-1-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-1-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-1-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-1-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "customer" + ] + } + }, + "shipping-1-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-1-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-2-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-2-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-2-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-2-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "customer" + ] + } + }, + "shipping-2-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-2-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-3-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-3-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-3-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-3-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "customer" + ] + } + }, + "shipping-3-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-3-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-4-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-4-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-4-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-4-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "customer" + ] + } + }, + "shipping-4-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-4-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-5-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-5-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-5-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-5-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "customer" + ] + } + }, + "shipping-5-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-5-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-6-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-6-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-6-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-6-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "customer" + ] + } + }, + "shipping-6-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-6-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-7-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-7-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-7-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-7-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "customer" + ] + } + }, + "shipping-7-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-7-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + } + }, + "reject-all": { + "refund-0-0": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-0-1": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-0-2": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-0-3": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-0-4": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-0-5": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-1-0": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-1-1": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-1-2": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-1-3": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-1-4": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-1-5": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-2-0": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-2-1": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-2-2": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-2-3": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-2-4": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-2-5": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-3-0": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-3-1": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-3-2": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-3-3": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-3-4": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-3-5": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-4-0": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-4-1": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-4-2": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-4-3": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-4-4": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-4-5": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-5-0": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-5-1": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-5-2": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-5-3": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-5-4": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-5-5": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-6-0": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-6-1": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-6-2": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-6-3": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-6-4": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-6-5": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-7-0": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-7-1": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-7-2": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-7-3": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-7-4": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "refund-7-5": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-0-0": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-0-1": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-0-2": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-0-3": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-0-4": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-0-5": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-1-0": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-1-1": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-1-2": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-1-3": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-1-4": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-1-5": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-2-0": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-2-1": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-2-2": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-2-3": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-2-4": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-2-5": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-3-0": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-3-1": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-3-2": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-3-3": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-3-4": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-3-5": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-4-0": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-4-1": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-4-2": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-4-3": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-4-4": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-4-5": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-5-0": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-5-1": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-5-2": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-5-3": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-5-4": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-5-5": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-6-0": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-6-1": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-6-2": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-6-3": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-6-4": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-6-5": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-7-0": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-7-1": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-7-2": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-7-3": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-7-4": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "reservation-7-5": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-0-0": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-0-1": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-0-2": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-0-3": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-0-4": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-0-5": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-1-0": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-1-1": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-1-2": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-1-3": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-1-4": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-1-5": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-2-0": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-2-1": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-2-2": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-2-3": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-2-4": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-2-5": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-3-0": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-3-1": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-3-2": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-3-3": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-3-4": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-3-5": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-4-0": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-4-1": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-4-2": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-4-3": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-4-4": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-4-5": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-5-0": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-5-1": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-5-2": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-5-3": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-5-4": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-5-5": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-6-0": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-6-1": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-6-2": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-6-3": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-6-4": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-6-5": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-7-0": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-7-1": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-7-2": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-7-3": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-7-4": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + }, + "shipping-7-5": { + "baseline": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "reject_all" + ] + } + } + }, + "reject-valid-variation": { + "refund-0-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-0-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-0-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-0-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "receipt_request" + ] + } + }, + "refund-0-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "wording" + ] + } + }, + "refund-0-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-1-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-1-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-1-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-1-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "receipt_request" + ] + } + }, + "refund-1-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "wording" + ] + } + }, + "refund-1-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-2-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-2-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-2-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-2-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "receipt_request" + ] + } + }, + "refund-2-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "wording" + ] + } + }, + "refund-2-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-3-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-3-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-3-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-3-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "receipt_request" + ] + } + }, + "refund-3-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "wording" + ] + } + }, + "refund-3-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-4-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-4-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-4-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-4-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "receipt_request" + ] + } + }, + "refund-4-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "wording" + ] + } + }, + "refund-4-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-5-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-5-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-5-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-5-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "receipt_request" + ] + } + }, + "refund-5-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "wording" + ] + } + }, + "refund-5-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-6-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-6-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-6-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-6-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "receipt_request" + ] + } + }, + "refund-6-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "wording" + ] + } + }, + "refund-6-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-7-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-7-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-7-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "completion_status" + ] + } + }, + "refund-7-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "receipt_request" + ] + } + }, + "refund-7-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "wording" + ] + } + }, + "refund-7-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-0-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-0-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-0-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-0-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "resource" + ] + } + }, + "reservation-0-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "wording" + ] + } + }, + "reservation-0-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-1-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-1-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-1-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-1-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "resource" + ] + } + }, + "reservation-1-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "wording" + ] + } + }, + "reservation-1-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-2-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-2-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-2-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-2-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "resource" + ] + } + }, + "reservation-2-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "wording" + ] + } + }, + "reservation-2-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-3-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-3-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-3-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-3-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "resource" + ] + } + }, + "reservation-3-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "wording" + ] + } + }, + "reservation-3-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-4-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-4-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-4-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-4-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "resource" + ] + } + }, + "reservation-4-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "wording" + ] + } + }, + "reservation-4-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-5-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-5-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-5-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-5-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "resource" + ] + } + }, + "reservation-5-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "wording" + ] + } + }, + "reservation-5-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-6-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-6-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-6-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-6-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "resource" + ] + } + }, + "reservation-6-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "wording" + ] + } + }, + "reservation-6-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-7-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-7-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-7-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "quantity" + ] + } + }, + "reservation-7-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "resource" + ] + } + }, + "reservation-7-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "wording" + ] + } + }, + "reservation-7-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-0-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-0-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-0-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-0-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "customer" + ] + } + }, + "shipping-0-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "wording" + ] + } + }, + "shipping-0-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-1-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-1-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-1-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-1-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "customer" + ] + } + }, + "shipping-1-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "wording" + ] + } + }, + "shipping-1-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-2-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-2-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-2-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-2-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "customer" + ] + } + }, + "shipping-2-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "wording" + ] + } + }, + "shipping-2-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-3-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-3-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-3-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-3-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "customer" + ] + } + }, + "shipping-3-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "wording" + ] + } + }, + "shipping-3-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-4-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-4-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-4-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-4-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "customer" + ] + } + }, + "shipping-4-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "wording" + ] + } + }, + "shipping-4-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-5-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-5-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-5-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-5-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "customer" + ] + } + }, + "shipping-5-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "wording" + ] + } + }, + "shipping-5-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-6-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-6-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-6-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-6-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "customer" + ] + } + }, + "shipping-6-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "wording" + ] + } + }, + "shipping-6-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-7-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-7-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-7-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "transition" + ] + } + }, + "shipping-7-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "customer" + ] + } + }, + "shipping-7-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "wording" + ] + } + }, + "shipping-7-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + } + }, + "unknown-on-fault": { + "refund-0-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-0-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-0-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-0-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-0-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-0-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-1-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-1-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-1-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-1-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-1-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-1-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-2-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-2-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-2-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-2-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-2-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-2-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-3-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-3-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-3-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-3-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-3-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-3-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-4-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-4-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-4-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-4-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-4-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-4-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-5-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-5-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-5-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-5-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-5-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-5-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-6-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-6-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-6-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-6-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-6-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-6-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-7-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-7-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-7-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-7-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "refund-7-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-7-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-0-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-0-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-0-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-0-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-0-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-0-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-1-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-1-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-1-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-1-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-1-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-1-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-2-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-2-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-2-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-2-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-2-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-2-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-3-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-3-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-3-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-3-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-3-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-3-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-4-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-4-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-4-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-4-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-4-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-4-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-5-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-5-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-5-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-5-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-5-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-5-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-6-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-6-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-6-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-6-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-6-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-6-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-7-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-7-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-7-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-7-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "reservation-7-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-7-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-0-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-0-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-0-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-0-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-0-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-0-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-1-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-1-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-1-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-1-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-1-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-1-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-2-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-2-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-2-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-2-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-2-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-2-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-3-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-3-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-3-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-3-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-3-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-3-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-4-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-4-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-4-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-4-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-4-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-4-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-5-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-5-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-5-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-5-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-5-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-5-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-6-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-6-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-6-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-6-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-6-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-6-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-7-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-7-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-7-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-7-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": false, + "error": null, + "violations": [] + } + }, + "shipping-7-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-7-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + } + }, + "wrong-diagnostic-ID": { + "refund-0-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-0-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-0-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-0-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-0-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-0-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-1-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-1-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-1-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-1-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-1-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-1-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-2-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-2-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-2-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-2-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-2-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-2-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-3-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-3-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-3-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-3-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-3-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-3-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-4-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-4-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-4-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-4-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-4-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-4-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-5-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-5-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-5-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-5-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-5-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-5-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-6-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-6-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-6-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-6-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-6-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-6-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-7-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-7-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-7-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-7-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "refund-7-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "refund-7-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-0-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-0-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-0-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-0-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-0-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-0-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-1-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-1-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-1-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-1-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-1-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-1-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-2-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-2-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-2-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-2-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-2-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-2-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-3-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-3-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-3-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-3-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-3-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-3-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-4-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-4-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-4-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-4-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-4-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-4-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-5-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-5-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-5-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-5-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-5-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-5-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-6-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-6-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-6-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-6-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-6-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-6-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-7-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-7-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-7-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-7-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "reservation-7-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "reservation-7-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-0-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-0-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-0-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-0-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-0-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-0-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-1-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-1-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-1-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-1-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-1-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-1-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-2-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-2-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-2-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-2-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-2-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-2-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-3-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-3-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-3-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-3-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-3-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-3-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-4-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-4-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-4-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-4-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-4-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-4-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-5-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-5-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-5-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-5-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-5-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-5-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-6-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-6-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-6-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-6-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-6-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-6-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-7-0": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-7-1": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-7-2": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-7-3": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": false, + "complete": true, + "error": null, + "violations": [ + "unrelated_reason" + ] + } + }, + "shipping-7-4": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + }, + "shipping-7-5": { + "baseline": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + }, + "variant": { + "accepted": true, + "complete": true, + "error": null, + "violations": [] + } + } + } +} diff --git a/benchmarks/validator-study/evaluation/summary.json b/benchmarks/validator-study/evaluation/summary.json new file mode 100644 index 0000000..68b0d53 --- /dev/null +++ b/benchmarks/validator-study/evaluation/summary.json @@ -0,0 +1,218 @@ +{ + "disagreements": 0, + "groups": 24, + "library_version": "0.1.0a3", + "pairs": 144, + "profiles": { + "accept-all": { + "ablations": { + "invalid_error_credit_score": 0.0, + "invalid_hide_unknowns": { + "denominator": 96, + "score": 0.0 + }, + "without_controls": { + "detection_score": 0.0, + "preservation_rate": null + } + }, + "boolean_detections": 0, + "boolean_success_without_target": 0, + "counts": { + "preserved": 48, + "survived": 96 + }, + "detailed_disagreements": 0, + "detection_score": 0.0, + "eligible_controls": 48, + "eligible_faults": 96, + "preservation_rate": 1.0, + "raw_fault_rejections_ignoring_baseline": 0, + "targeted_detections": 0 + }, + "correct": { + "ablations": { + "invalid_error_credit_score": 1.0, + "invalid_hide_unknowns": { + "denominator": 96, + "score": 1.0 + }, + "without_controls": { + "detection_score": 1.0, + "preservation_rate": null + } + }, + "boolean_detections": 96, + "boolean_success_without_target": 0, + "counts": { + "detected": 96, + "preserved": 48 + }, + "detailed_disagreements": 0, + "detection_score": 1.0, + "eligible_controls": 48, + "eligible_faults": 96, + "preservation_rate": 1.0, + "raw_fault_rejections_ignoring_baseline": 96, + "targeted_detections": 96 + }, + "crash-on-fault": { + "ablations": { + "invalid_error_credit_score": 1.0, + "invalid_hide_unknowns": { + "denominator": 96, + "score": 0.0 + }, + "without_controls": { + "detection_score": 0.0, + "preservation_rate": null + } + }, + "boolean_detections": 0, + "boolean_success_without_target": 0, + "counts": { + "error": 96, + "preserved": 48 + }, + "detailed_disagreements": 0, + "detection_score": 0.0, + "eligible_controls": 48, + "eligible_faults": 96, + "preservation_rate": 1.0, + "raw_fault_rejections_ignoring_baseline": 0, + "targeted_detections": 0 + }, + "omit-first-obligation": { + "ablations": { + "invalid_error_credit_score": 0.25, + "invalid_hide_unknowns": { + "denominator": 96, + "score": 0.25 + }, + "without_controls": { + "detection_score": 0.25, + "preservation_rate": null + } + }, + "boolean_detections": 24, + "boolean_success_without_target": 0, + "counts": { + "detected": 24, + "preserved": 48, + "survived": 72 + }, + "detailed_disagreements": 0, + "detection_score": 0.25, + "eligible_controls": 48, + "eligible_faults": 96, + "preservation_rate": 1.0, + "raw_fault_rejections_ignoring_baseline": 24, + "targeted_detections": 24 + }, + "reject-all": { + "ablations": { + "invalid_error_credit_score": null, + "invalid_hide_unknowns": { + "denominator": 0, + "score": null + }, + "without_controls": { + "detection_score": null, + "preservation_rate": null + } + }, + "boolean_detections": 0, + "boolean_success_without_target": 0, + "counts": { + "baseline_failed": 144 + }, + "detailed_disagreements": 0, + "detection_score": null, + "eligible_controls": 0, + "eligible_faults": 0, + "preservation_rate": null, + "raw_fault_rejections_ignoring_baseline": 96, + "targeted_detections": 0 + }, + "reject-valid-variation": { + "ablations": { + "invalid_error_credit_score": 1.0, + "invalid_hide_unknowns": { + "denominator": 96, + "score": 1.0 + }, + "without_controls": { + "detection_score": 1.0, + "preservation_rate": null + } + }, + "boolean_detections": 96, + "boolean_success_without_target": 0, + "counts": { + "detected": 96, + "preserved": 24, + "regressed": 24 + }, + "detailed_disagreements": 0, + "detection_score": 1.0, + "eligible_controls": 48, + "eligible_faults": 96, + "preservation_rate": 0.5, + "raw_fault_rejections_ignoring_baseline": 96, + "targeted_detections": 96 + }, + "unknown-on-fault": { + "ablations": { + "invalid_error_credit_score": 0.0, + "invalid_hide_unknowns": { + "denominator": 0, + "score": null + }, + "without_controls": { + "detection_score": 0.0, + "preservation_rate": null + } + }, + "boolean_detections": 0, + "boolean_success_without_target": 0, + "counts": { + "preserved": 48, + "undetermined": 96 + }, + "detailed_disagreements": 0, + "detection_score": 0.0, + "eligible_controls": 48, + "eligible_faults": 96, + "preservation_rate": 1.0, + "raw_fault_rejections_ignoring_baseline": 0, + "targeted_detections": 0 + }, + "wrong-diagnostic-ID": { + "ablations": { + "invalid_error_credit_score": 0.0, + "invalid_hide_unknowns": { + "denominator": 96, + "score": 0.0 + }, + "without_controls": { + "detection_score": 0.0, + "preservation_rate": null + } + }, + "boolean_detections": 96, + "boolean_success_without_target": 96, + "counts": { + "preserved": 48, + "unattributed_rejection": 96 + }, + "detailed_disagreements": 0, + "detection_score": 0.0, + "eligible_controls": 48, + "eligible_faults": 96, + "preservation_rate": 1.0, + "raw_fault_rejections_ignoring_baseline": 96, + "targeted_detections": 0 + } + }, + "schema_version": 1 +} diff --git a/benchmarks/validator-study/prepared/corpus.json b/benchmarks/validator-study/prepared/corpus.json new file mode 100644 index 0000000..57cf228 --- /dev/null +++ b/benchmarks/validator-study/prepared/corpus.json @@ -0,0 +1,3434 @@ +[ + { + "baseline": { + "receipt": { + "request": "refund-0", + "status": "completed" + }, + "request": "refund-0", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "completion_status" + ], + "group": "refund-0", + "id": "refund-0-0", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": null, + "request": "refund-0", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-0", + "status": "completed" + }, + "request": "refund-0", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "completion_status" + ], + "group": "refund-0", + "id": "refund-0-1", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": { + "request": "refund-0", + "status": "pending" + }, + "request": "refund-0", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-0", + "status": "completed" + }, + "request": "refund-0", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "completion_status" + ], + "group": "refund-0", + "id": "refund-0-2", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": { + "request": "refund-0", + "status": "accepted" + }, + "request": "refund-0", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-0", + "status": "completed" + }, + "request": "refund-0", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "receipt_request" + ], + "group": "refund-0", + "id": "refund-0-3", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": { + "request": "refund-0-other", + "status": "completed" + }, + "request": "refund-0", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-0", + "status": "completed" + }, + "request": "refund-0", + "text": "Recorded." + }, + "domain": "refund", + "expected": [], + "group": "refund-0", + "id": "refund-0-4", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "receipt": { + "request": "refund-0", + "status": "completed" + }, + "request": "refund-0", + "text": "Acknowledged." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-0", + "status": "completed" + }, + "request": "refund-0", + "text": "Recorded." + }, + "domain": "refund", + "expected": [], + "group": "refund-0", + "id": "refund-0-5", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "receipt": { + "request": "refund-0-next", + "status": "completed" + }, + "request": "refund-0-next", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-1", + "status": "completed" + }, + "request": "refund-1", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "completion_status" + ], + "group": "refund-1", + "id": "refund-1-0", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": null, + "request": "refund-1", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-1", + "status": "completed" + }, + "request": "refund-1", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "completion_status" + ], + "group": "refund-1", + "id": "refund-1-1", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": { + "request": "refund-1", + "status": "pending" + }, + "request": "refund-1", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-1", + "status": "completed" + }, + "request": "refund-1", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "completion_status" + ], + "group": "refund-1", + "id": "refund-1-2", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": { + "request": "refund-1", + "status": "accepted" + }, + "request": "refund-1", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-1", + "status": "completed" + }, + "request": "refund-1", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "receipt_request" + ], + "group": "refund-1", + "id": "refund-1-3", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": { + "request": "refund-1-other", + "status": "completed" + }, + "request": "refund-1", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-1", + "status": "completed" + }, + "request": "refund-1", + "text": "Recorded." + }, + "domain": "refund", + "expected": [], + "group": "refund-1", + "id": "refund-1-4", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "receipt": { + "request": "refund-1", + "status": "completed" + }, + "request": "refund-1", + "text": "Acknowledged." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-1", + "status": "completed" + }, + "request": "refund-1", + "text": "Recorded." + }, + "domain": "refund", + "expected": [], + "group": "refund-1", + "id": "refund-1-5", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "receipt": { + "request": "refund-1-next", + "status": "completed" + }, + "request": "refund-1-next", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-2", + "status": "completed" + }, + "request": "refund-2", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "completion_status" + ], + "group": "refund-2", + "id": "refund-2-0", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": null, + "request": "refund-2", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-2", + "status": "completed" + }, + "request": "refund-2", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "completion_status" + ], + "group": "refund-2", + "id": "refund-2-1", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": { + "request": "refund-2", + "status": "pending" + }, + "request": "refund-2", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-2", + "status": "completed" + }, + "request": "refund-2", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "completion_status" + ], + "group": "refund-2", + "id": "refund-2-2", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": { + "request": "refund-2", + "status": "accepted" + }, + "request": "refund-2", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-2", + "status": "completed" + }, + "request": "refund-2", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "receipt_request" + ], + "group": "refund-2", + "id": "refund-2-3", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": { + "request": "refund-2-other", + "status": "completed" + }, + "request": "refund-2", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-2", + "status": "completed" + }, + "request": "refund-2", + "text": "Recorded." + }, + "domain": "refund", + "expected": [], + "group": "refund-2", + "id": "refund-2-4", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "receipt": { + "request": "refund-2", + "status": "completed" + }, + "request": "refund-2", + "text": "Acknowledged." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-2", + "status": "completed" + }, + "request": "refund-2", + "text": "Recorded." + }, + "domain": "refund", + "expected": [], + "group": "refund-2", + "id": "refund-2-5", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "receipt": { + "request": "refund-2-next", + "status": "completed" + }, + "request": "refund-2-next", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-3", + "status": "completed" + }, + "request": "refund-3", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "completion_status" + ], + "group": "refund-3", + "id": "refund-3-0", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": null, + "request": "refund-3", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-3", + "status": "completed" + }, + "request": "refund-3", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "completion_status" + ], + "group": "refund-3", + "id": "refund-3-1", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": { + "request": "refund-3", + "status": "pending" + }, + "request": "refund-3", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-3", + "status": "completed" + }, + "request": "refund-3", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "completion_status" + ], + "group": "refund-3", + "id": "refund-3-2", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": { + "request": "refund-3", + "status": "accepted" + }, + "request": "refund-3", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-3", + "status": "completed" + }, + "request": "refund-3", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "receipt_request" + ], + "group": "refund-3", + "id": "refund-3-3", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": { + "request": "refund-3-other", + "status": "completed" + }, + "request": "refund-3", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-3", + "status": "completed" + }, + "request": "refund-3", + "text": "Recorded." + }, + "domain": "refund", + "expected": [], + "group": "refund-3", + "id": "refund-3-4", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "receipt": { + "request": "refund-3", + "status": "completed" + }, + "request": "refund-3", + "text": "Acknowledged." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-3", + "status": "completed" + }, + "request": "refund-3", + "text": "Recorded." + }, + "domain": "refund", + "expected": [], + "group": "refund-3", + "id": "refund-3-5", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "receipt": { + "request": "refund-3-next", + "status": "completed" + }, + "request": "refund-3-next", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-4", + "status": "completed" + }, + "request": "refund-4", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "completion_status" + ], + "group": "refund-4", + "id": "refund-4-0", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": null, + "request": "refund-4", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-4", + "status": "completed" + }, + "request": "refund-4", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "completion_status" + ], + "group": "refund-4", + "id": "refund-4-1", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": { + "request": "refund-4", + "status": "pending" + }, + "request": "refund-4", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-4", + "status": "completed" + }, + "request": "refund-4", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "completion_status" + ], + "group": "refund-4", + "id": "refund-4-2", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": { + "request": "refund-4", + "status": "accepted" + }, + "request": "refund-4", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-4", + "status": "completed" + }, + "request": "refund-4", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "receipt_request" + ], + "group": "refund-4", + "id": "refund-4-3", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": { + "request": "refund-4-other", + "status": "completed" + }, + "request": "refund-4", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-4", + "status": "completed" + }, + "request": "refund-4", + "text": "Recorded." + }, + "domain": "refund", + "expected": [], + "group": "refund-4", + "id": "refund-4-4", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "receipt": { + "request": "refund-4", + "status": "completed" + }, + "request": "refund-4", + "text": "Acknowledged." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-4", + "status": "completed" + }, + "request": "refund-4", + "text": "Recorded." + }, + "domain": "refund", + "expected": [], + "group": "refund-4", + "id": "refund-4-5", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "receipt": { + "request": "refund-4-next", + "status": "completed" + }, + "request": "refund-4-next", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-5", + "status": "completed" + }, + "request": "refund-5", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "completion_status" + ], + "group": "refund-5", + "id": "refund-5-0", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": null, + "request": "refund-5", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-5", + "status": "completed" + }, + "request": "refund-5", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "completion_status" + ], + "group": "refund-5", + "id": "refund-5-1", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": { + "request": "refund-5", + "status": "pending" + }, + "request": "refund-5", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-5", + "status": "completed" + }, + "request": "refund-5", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "completion_status" + ], + "group": "refund-5", + "id": "refund-5-2", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": { + "request": "refund-5", + "status": "accepted" + }, + "request": "refund-5", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-5", + "status": "completed" + }, + "request": "refund-5", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "receipt_request" + ], + "group": "refund-5", + "id": "refund-5-3", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": { + "request": "refund-5-other", + "status": "completed" + }, + "request": "refund-5", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-5", + "status": "completed" + }, + "request": "refund-5", + "text": "Recorded." + }, + "domain": "refund", + "expected": [], + "group": "refund-5", + "id": "refund-5-4", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "receipt": { + "request": "refund-5", + "status": "completed" + }, + "request": "refund-5", + "text": "Acknowledged." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-5", + "status": "completed" + }, + "request": "refund-5", + "text": "Recorded." + }, + "domain": "refund", + "expected": [], + "group": "refund-5", + "id": "refund-5-5", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "receipt": { + "request": "refund-5-next", + "status": "completed" + }, + "request": "refund-5-next", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-6", + "status": "completed" + }, + "request": "refund-6", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "completion_status" + ], + "group": "refund-6", + "id": "refund-6-0", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": null, + "request": "refund-6", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-6", + "status": "completed" + }, + "request": "refund-6", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "completion_status" + ], + "group": "refund-6", + "id": "refund-6-1", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": { + "request": "refund-6", + "status": "pending" + }, + "request": "refund-6", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-6", + "status": "completed" + }, + "request": "refund-6", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "completion_status" + ], + "group": "refund-6", + "id": "refund-6-2", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": { + "request": "refund-6", + "status": "accepted" + }, + "request": "refund-6", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-6", + "status": "completed" + }, + "request": "refund-6", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "receipt_request" + ], + "group": "refund-6", + "id": "refund-6-3", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": { + "request": "refund-6-other", + "status": "completed" + }, + "request": "refund-6", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-6", + "status": "completed" + }, + "request": "refund-6", + "text": "Recorded." + }, + "domain": "refund", + "expected": [], + "group": "refund-6", + "id": "refund-6-4", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "receipt": { + "request": "refund-6", + "status": "completed" + }, + "request": "refund-6", + "text": "Acknowledged." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-6", + "status": "completed" + }, + "request": "refund-6", + "text": "Recorded." + }, + "domain": "refund", + "expected": [], + "group": "refund-6", + "id": "refund-6-5", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "receipt": { + "request": "refund-6-next", + "status": "completed" + }, + "request": "refund-6-next", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-7", + "status": "completed" + }, + "request": "refund-7", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "completion_status" + ], + "group": "refund-7", + "id": "refund-7-0", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": null, + "request": "refund-7", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-7", + "status": "completed" + }, + "request": "refund-7", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "completion_status" + ], + "group": "refund-7", + "id": "refund-7-1", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": { + "request": "refund-7", + "status": "pending" + }, + "request": "refund-7", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-7", + "status": "completed" + }, + "request": "refund-7", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "completion_status" + ], + "group": "refund-7", + "id": "refund-7-2", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": { + "request": "refund-7", + "status": "accepted" + }, + "request": "refund-7", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-7", + "status": "completed" + }, + "request": "refund-7", + "text": "Recorded." + }, + "domain": "refund", + "expected": [ + "receipt_request" + ], + "group": "refund-7", + "id": "refund-7-3", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "receipt": { + "request": "refund-7-other", + "status": "completed" + }, + "request": "refund-7", + "text": "Recorded." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-7", + "status": "completed" + }, + "request": "refund-7", + "text": "Recorded." + }, + "domain": "refund", + "expected": [], + "group": "refund-7", + "id": "refund-7-4", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "receipt": { + "request": "refund-7", + "status": "completed" + }, + "request": "refund-7", + "text": "Acknowledged." + } + }, + { + "baseline": { + "receipt": { + "request": "refund-7", + "status": "completed" + }, + "request": "refund-7", + "text": "Recorded." + }, + "domain": "refund", + "expected": [], + "group": "refund-7", + "id": "refund-7-5", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "receipt": { + "request": "refund-7-next", + "status": "completed" + }, + "request": "refund-7-next", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-0", + "owner": "shipping-0", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "transition" + ], + "group": "shipping-0", + "id": "shipping-0-0", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "delivered", + "before": "queued", + "customer": "shipping-0", + "owner": "shipping-0", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-0", + "owner": "shipping-0", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "transition" + ], + "group": "shipping-0", + "id": "shipping-0-1", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "cancelled", + "before": "queued", + "customer": "shipping-0", + "owner": "shipping-0", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-0", + "owner": "shipping-0", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "transition" + ], + "group": "shipping-0", + "id": "shipping-0-2", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "queued", + "before": "queued", + "customer": "shipping-0", + "owner": "shipping-0", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-0", + "owner": "shipping-0", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "customer" + ], + "group": "shipping-0", + "id": "shipping-0-3", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "packed", + "before": "queued", + "customer": "shipping-0-other", + "owner": "shipping-0", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-0", + "owner": "shipping-0", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [], + "group": "shipping-0", + "id": "shipping-0-4", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "after": "packed", + "before": "queued", + "customer": "shipping-0", + "owner": "shipping-0", + "text": "Acknowledged." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-0", + "owner": "shipping-0", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [], + "group": "shipping-0", + "id": "shipping-0-5", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "after": "shipped", + "before": "packed", + "customer": "shipping-0", + "owner": "shipping-0", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-1", + "owner": "shipping-1", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "transition" + ], + "group": "shipping-1", + "id": "shipping-1-0", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "delivered", + "before": "queued", + "customer": "shipping-1", + "owner": "shipping-1", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-1", + "owner": "shipping-1", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "transition" + ], + "group": "shipping-1", + "id": "shipping-1-1", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "cancelled", + "before": "queued", + "customer": "shipping-1", + "owner": "shipping-1", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-1", + "owner": "shipping-1", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "transition" + ], + "group": "shipping-1", + "id": "shipping-1-2", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "queued", + "before": "queued", + "customer": "shipping-1", + "owner": "shipping-1", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-1", + "owner": "shipping-1", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "customer" + ], + "group": "shipping-1", + "id": "shipping-1-3", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "packed", + "before": "queued", + "customer": "shipping-1-other", + "owner": "shipping-1", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-1", + "owner": "shipping-1", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [], + "group": "shipping-1", + "id": "shipping-1-4", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "after": "packed", + "before": "queued", + "customer": "shipping-1", + "owner": "shipping-1", + "text": "Acknowledged." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-1", + "owner": "shipping-1", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [], + "group": "shipping-1", + "id": "shipping-1-5", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "after": "shipped", + "before": "packed", + "customer": "shipping-1", + "owner": "shipping-1", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-2", + "owner": "shipping-2", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "transition" + ], + "group": "shipping-2", + "id": "shipping-2-0", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "delivered", + "before": "queued", + "customer": "shipping-2", + "owner": "shipping-2", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-2", + "owner": "shipping-2", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "transition" + ], + "group": "shipping-2", + "id": "shipping-2-1", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "cancelled", + "before": "queued", + "customer": "shipping-2", + "owner": "shipping-2", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-2", + "owner": "shipping-2", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "transition" + ], + "group": "shipping-2", + "id": "shipping-2-2", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "queued", + "before": "queued", + "customer": "shipping-2", + "owner": "shipping-2", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-2", + "owner": "shipping-2", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "customer" + ], + "group": "shipping-2", + "id": "shipping-2-3", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "packed", + "before": "queued", + "customer": "shipping-2-other", + "owner": "shipping-2", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-2", + "owner": "shipping-2", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [], + "group": "shipping-2", + "id": "shipping-2-4", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "after": "packed", + "before": "queued", + "customer": "shipping-2", + "owner": "shipping-2", + "text": "Acknowledged." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-2", + "owner": "shipping-2", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [], + "group": "shipping-2", + "id": "shipping-2-5", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "after": "shipped", + "before": "packed", + "customer": "shipping-2", + "owner": "shipping-2", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-3", + "owner": "shipping-3", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "transition" + ], + "group": "shipping-3", + "id": "shipping-3-0", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "delivered", + "before": "queued", + "customer": "shipping-3", + "owner": "shipping-3", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-3", + "owner": "shipping-3", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "transition" + ], + "group": "shipping-3", + "id": "shipping-3-1", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "cancelled", + "before": "queued", + "customer": "shipping-3", + "owner": "shipping-3", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-3", + "owner": "shipping-3", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "transition" + ], + "group": "shipping-3", + "id": "shipping-3-2", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "queued", + "before": "queued", + "customer": "shipping-3", + "owner": "shipping-3", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-3", + "owner": "shipping-3", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "customer" + ], + "group": "shipping-3", + "id": "shipping-3-3", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "packed", + "before": "queued", + "customer": "shipping-3-other", + "owner": "shipping-3", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-3", + "owner": "shipping-3", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [], + "group": "shipping-3", + "id": "shipping-3-4", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "after": "packed", + "before": "queued", + "customer": "shipping-3", + "owner": "shipping-3", + "text": "Acknowledged." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-3", + "owner": "shipping-3", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [], + "group": "shipping-3", + "id": "shipping-3-5", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "after": "shipped", + "before": "packed", + "customer": "shipping-3", + "owner": "shipping-3", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-4", + "owner": "shipping-4", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "transition" + ], + "group": "shipping-4", + "id": "shipping-4-0", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "delivered", + "before": "queued", + "customer": "shipping-4", + "owner": "shipping-4", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-4", + "owner": "shipping-4", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "transition" + ], + "group": "shipping-4", + "id": "shipping-4-1", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "cancelled", + "before": "queued", + "customer": "shipping-4", + "owner": "shipping-4", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-4", + "owner": "shipping-4", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "transition" + ], + "group": "shipping-4", + "id": "shipping-4-2", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "queued", + "before": "queued", + "customer": "shipping-4", + "owner": "shipping-4", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-4", + "owner": "shipping-4", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "customer" + ], + "group": "shipping-4", + "id": "shipping-4-3", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "packed", + "before": "queued", + "customer": "shipping-4-other", + "owner": "shipping-4", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-4", + "owner": "shipping-4", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [], + "group": "shipping-4", + "id": "shipping-4-4", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "after": "packed", + "before": "queued", + "customer": "shipping-4", + "owner": "shipping-4", + "text": "Acknowledged." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-4", + "owner": "shipping-4", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [], + "group": "shipping-4", + "id": "shipping-4-5", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "after": "shipped", + "before": "packed", + "customer": "shipping-4", + "owner": "shipping-4", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-5", + "owner": "shipping-5", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "transition" + ], + "group": "shipping-5", + "id": "shipping-5-0", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "delivered", + "before": "queued", + "customer": "shipping-5", + "owner": "shipping-5", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-5", + "owner": "shipping-5", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "transition" + ], + "group": "shipping-5", + "id": "shipping-5-1", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "cancelled", + "before": "queued", + "customer": "shipping-5", + "owner": "shipping-5", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-5", + "owner": "shipping-5", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "transition" + ], + "group": "shipping-5", + "id": "shipping-5-2", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "queued", + "before": "queued", + "customer": "shipping-5", + "owner": "shipping-5", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-5", + "owner": "shipping-5", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "customer" + ], + "group": "shipping-5", + "id": "shipping-5-3", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "packed", + "before": "queued", + "customer": "shipping-5-other", + "owner": "shipping-5", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-5", + "owner": "shipping-5", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [], + "group": "shipping-5", + "id": "shipping-5-4", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "after": "packed", + "before": "queued", + "customer": "shipping-5", + "owner": "shipping-5", + "text": "Acknowledged." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-5", + "owner": "shipping-5", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [], + "group": "shipping-5", + "id": "shipping-5-5", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "after": "shipped", + "before": "packed", + "customer": "shipping-5", + "owner": "shipping-5", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-6", + "owner": "shipping-6", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "transition" + ], + "group": "shipping-6", + "id": "shipping-6-0", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "delivered", + "before": "queued", + "customer": "shipping-6", + "owner": "shipping-6", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-6", + "owner": "shipping-6", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "transition" + ], + "group": "shipping-6", + "id": "shipping-6-1", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "cancelled", + "before": "queued", + "customer": "shipping-6", + "owner": "shipping-6", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-6", + "owner": "shipping-6", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "transition" + ], + "group": "shipping-6", + "id": "shipping-6-2", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "queued", + "before": "queued", + "customer": "shipping-6", + "owner": "shipping-6", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-6", + "owner": "shipping-6", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "customer" + ], + "group": "shipping-6", + "id": "shipping-6-3", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "packed", + "before": "queued", + "customer": "shipping-6-other", + "owner": "shipping-6", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-6", + "owner": "shipping-6", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [], + "group": "shipping-6", + "id": "shipping-6-4", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "after": "packed", + "before": "queued", + "customer": "shipping-6", + "owner": "shipping-6", + "text": "Acknowledged." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-6", + "owner": "shipping-6", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [], + "group": "shipping-6", + "id": "shipping-6-5", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "after": "shipped", + "before": "packed", + "customer": "shipping-6", + "owner": "shipping-6", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-7", + "owner": "shipping-7", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "transition" + ], + "group": "shipping-7", + "id": "shipping-7-0", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "delivered", + "before": "queued", + "customer": "shipping-7", + "owner": "shipping-7", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-7", + "owner": "shipping-7", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "transition" + ], + "group": "shipping-7", + "id": "shipping-7-1", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "cancelled", + "before": "queued", + "customer": "shipping-7", + "owner": "shipping-7", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-7", + "owner": "shipping-7", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "transition" + ], + "group": "shipping-7", + "id": "shipping-7-2", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "queued", + "before": "queued", + "customer": "shipping-7", + "owner": "shipping-7", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-7", + "owner": "shipping-7", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [ + "customer" + ], + "group": "shipping-7", + "id": "shipping-7-3", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "after": "packed", + "before": "queued", + "customer": "shipping-7-other", + "owner": "shipping-7", + "text": "Recorded." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-7", + "owner": "shipping-7", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [], + "group": "shipping-7", + "id": "shipping-7-4", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "after": "packed", + "before": "queued", + "customer": "shipping-7", + "owner": "shipping-7", + "text": "Acknowledged." + } + }, + { + "baseline": { + "after": "packed", + "before": "queued", + "customer": "shipping-7", + "owner": "shipping-7", + "text": "Recorded." + }, + "domain": "shipping", + "expected": [], + "group": "shipping-7", + "id": "shipping-7-5", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "after": "shipped", + "before": "packed", + "customer": "shipping-7", + "owner": "shipping-7", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-0", + "capacity": 10, + "quantity": 2, + "resource": "reservation-0", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "quantity" + ], + "group": "reservation-0", + "id": "reservation-0-0", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-0", + "capacity": 10, + "quantity": 0, + "resource": "reservation-0", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-0", + "capacity": 10, + "quantity": 2, + "resource": "reservation-0", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "quantity" + ], + "group": "reservation-0", + "id": "reservation-0-1", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-0", + "capacity": 10, + "quantity": 11, + "resource": "reservation-0", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-0", + "capacity": 10, + "quantity": 2, + "resource": "reservation-0", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "quantity" + ], + "group": "reservation-0", + "id": "reservation-0-2", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-0", + "capacity": 10, + "quantity": -1, + "resource": "reservation-0", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-0", + "capacity": 10, + "quantity": 2, + "resource": "reservation-0", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "resource" + ], + "group": "reservation-0", + "id": "reservation-0-3", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-0", + "capacity": 10, + "quantity": 2, + "resource": "reservation-0-other", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-0", + "capacity": 10, + "quantity": 2, + "resource": "reservation-0", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [], + "group": "reservation-0", + "id": "reservation-0-4", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "authoritative_resource": "reservation-0", + "capacity": 10, + "quantity": 2, + "resource": "reservation-0", + "text": "Acknowledged." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-0", + "capacity": 10, + "quantity": 2, + "resource": "reservation-0", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [], + "group": "reservation-0", + "id": "reservation-0-5", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "authoritative_resource": "reservation-0", + "capacity": 10, + "quantity": 1, + "resource": "reservation-0", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-1", + "capacity": 11, + "quantity": 3, + "resource": "reservation-1", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "quantity" + ], + "group": "reservation-1", + "id": "reservation-1-0", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-1", + "capacity": 11, + "quantity": 0, + "resource": "reservation-1", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-1", + "capacity": 11, + "quantity": 3, + "resource": "reservation-1", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "quantity" + ], + "group": "reservation-1", + "id": "reservation-1-1", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-1", + "capacity": 11, + "quantity": 12, + "resource": "reservation-1", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-1", + "capacity": 11, + "quantity": 3, + "resource": "reservation-1", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "quantity" + ], + "group": "reservation-1", + "id": "reservation-1-2", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-1", + "capacity": 11, + "quantity": -1, + "resource": "reservation-1", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-1", + "capacity": 11, + "quantity": 3, + "resource": "reservation-1", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "resource" + ], + "group": "reservation-1", + "id": "reservation-1-3", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-1", + "capacity": 11, + "quantity": 3, + "resource": "reservation-1-other", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-1", + "capacity": 11, + "quantity": 3, + "resource": "reservation-1", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [], + "group": "reservation-1", + "id": "reservation-1-4", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "authoritative_resource": "reservation-1", + "capacity": 11, + "quantity": 3, + "resource": "reservation-1", + "text": "Acknowledged." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-1", + "capacity": 11, + "quantity": 3, + "resource": "reservation-1", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [], + "group": "reservation-1", + "id": "reservation-1-5", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "authoritative_resource": "reservation-1", + "capacity": 11, + "quantity": 1, + "resource": "reservation-1", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-2", + "capacity": 12, + "quantity": 4, + "resource": "reservation-2", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "quantity" + ], + "group": "reservation-2", + "id": "reservation-2-0", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-2", + "capacity": 12, + "quantity": 0, + "resource": "reservation-2", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-2", + "capacity": 12, + "quantity": 4, + "resource": "reservation-2", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "quantity" + ], + "group": "reservation-2", + "id": "reservation-2-1", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-2", + "capacity": 12, + "quantity": 13, + "resource": "reservation-2", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-2", + "capacity": 12, + "quantity": 4, + "resource": "reservation-2", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "quantity" + ], + "group": "reservation-2", + "id": "reservation-2-2", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-2", + "capacity": 12, + "quantity": -1, + "resource": "reservation-2", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-2", + "capacity": 12, + "quantity": 4, + "resource": "reservation-2", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "resource" + ], + "group": "reservation-2", + "id": "reservation-2-3", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-2", + "capacity": 12, + "quantity": 4, + "resource": "reservation-2-other", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-2", + "capacity": 12, + "quantity": 4, + "resource": "reservation-2", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [], + "group": "reservation-2", + "id": "reservation-2-4", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "authoritative_resource": "reservation-2", + "capacity": 12, + "quantity": 4, + "resource": "reservation-2", + "text": "Acknowledged." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-2", + "capacity": 12, + "quantity": 4, + "resource": "reservation-2", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [], + "group": "reservation-2", + "id": "reservation-2-5", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "authoritative_resource": "reservation-2", + "capacity": 12, + "quantity": 1, + "resource": "reservation-2", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-3", + "capacity": 13, + "quantity": 5, + "resource": "reservation-3", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "quantity" + ], + "group": "reservation-3", + "id": "reservation-3-0", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-3", + "capacity": 13, + "quantity": 0, + "resource": "reservation-3", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-3", + "capacity": 13, + "quantity": 5, + "resource": "reservation-3", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "quantity" + ], + "group": "reservation-3", + "id": "reservation-3-1", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-3", + "capacity": 13, + "quantity": 14, + "resource": "reservation-3", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-3", + "capacity": 13, + "quantity": 5, + "resource": "reservation-3", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "quantity" + ], + "group": "reservation-3", + "id": "reservation-3-2", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-3", + "capacity": 13, + "quantity": -1, + "resource": "reservation-3", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-3", + "capacity": 13, + "quantity": 5, + "resource": "reservation-3", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "resource" + ], + "group": "reservation-3", + "id": "reservation-3-3", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-3", + "capacity": 13, + "quantity": 5, + "resource": "reservation-3-other", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-3", + "capacity": 13, + "quantity": 5, + "resource": "reservation-3", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [], + "group": "reservation-3", + "id": "reservation-3-4", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "authoritative_resource": "reservation-3", + "capacity": 13, + "quantity": 5, + "resource": "reservation-3", + "text": "Acknowledged." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-3", + "capacity": 13, + "quantity": 5, + "resource": "reservation-3", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [], + "group": "reservation-3", + "id": "reservation-3-5", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "authoritative_resource": "reservation-3", + "capacity": 13, + "quantity": 1, + "resource": "reservation-3", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-4", + "capacity": 14, + "quantity": 6, + "resource": "reservation-4", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "quantity" + ], + "group": "reservation-4", + "id": "reservation-4-0", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-4", + "capacity": 14, + "quantity": 0, + "resource": "reservation-4", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-4", + "capacity": 14, + "quantity": 6, + "resource": "reservation-4", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "quantity" + ], + "group": "reservation-4", + "id": "reservation-4-1", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-4", + "capacity": 14, + "quantity": 15, + "resource": "reservation-4", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-4", + "capacity": 14, + "quantity": 6, + "resource": "reservation-4", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "quantity" + ], + "group": "reservation-4", + "id": "reservation-4-2", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-4", + "capacity": 14, + "quantity": -1, + "resource": "reservation-4", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-4", + "capacity": 14, + "quantity": 6, + "resource": "reservation-4", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "resource" + ], + "group": "reservation-4", + "id": "reservation-4-3", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-4", + "capacity": 14, + "quantity": 6, + "resource": "reservation-4-other", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-4", + "capacity": 14, + "quantity": 6, + "resource": "reservation-4", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [], + "group": "reservation-4", + "id": "reservation-4-4", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "authoritative_resource": "reservation-4", + "capacity": 14, + "quantity": 6, + "resource": "reservation-4", + "text": "Acknowledged." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-4", + "capacity": 14, + "quantity": 6, + "resource": "reservation-4", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [], + "group": "reservation-4", + "id": "reservation-4-5", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "authoritative_resource": "reservation-4", + "capacity": 14, + "quantity": 1, + "resource": "reservation-4", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-5", + "capacity": 15, + "quantity": 7, + "resource": "reservation-5", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "quantity" + ], + "group": "reservation-5", + "id": "reservation-5-0", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-5", + "capacity": 15, + "quantity": 0, + "resource": "reservation-5", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-5", + "capacity": 15, + "quantity": 7, + "resource": "reservation-5", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "quantity" + ], + "group": "reservation-5", + "id": "reservation-5-1", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-5", + "capacity": 15, + "quantity": 16, + "resource": "reservation-5", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-5", + "capacity": 15, + "quantity": 7, + "resource": "reservation-5", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "quantity" + ], + "group": "reservation-5", + "id": "reservation-5-2", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-5", + "capacity": 15, + "quantity": -1, + "resource": "reservation-5", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-5", + "capacity": 15, + "quantity": 7, + "resource": "reservation-5", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "resource" + ], + "group": "reservation-5", + "id": "reservation-5-3", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-5", + "capacity": 15, + "quantity": 7, + "resource": "reservation-5-other", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-5", + "capacity": 15, + "quantity": 7, + "resource": "reservation-5", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [], + "group": "reservation-5", + "id": "reservation-5-4", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "authoritative_resource": "reservation-5", + "capacity": 15, + "quantity": 7, + "resource": "reservation-5", + "text": "Acknowledged." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-5", + "capacity": 15, + "quantity": 7, + "resource": "reservation-5", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [], + "group": "reservation-5", + "id": "reservation-5-5", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "authoritative_resource": "reservation-5", + "capacity": 15, + "quantity": 1, + "resource": "reservation-5", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-6", + "capacity": 16, + "quantity": 8, + "resource": "reservation-6", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "quantity" + ], + "group": "reservation-6", + "id": "reservation-6-0", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-6", + "capacity": 16, + "quantity": 0, + "resource": "reservation-6", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-6", + "capacity": 16, + "quantity": 8, + "resource": "reservation-6", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "quantity" + ], + "group": "reservation-6", + "id": "reservation-6-1", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-6", + "capacity": 16, + "quantity": 17, + "resource": "reservation-6", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-6", + "capacity": 16, + "quantity": 8, + "resource": "reservation-6", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "quantity" + ], + "group": "reservation-6", + "id": "reservation-6-2", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-6", + "capacity": 16, + "quantity": -1, + "resource": "reservation-6", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-6", + "capacity": 16, + "quantity": 8, + "resource": "reservation-6", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "resource" + ], + "group": "reservation-6", + "id": "reservation-6-3", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-6", + "capacity": 16, + "quantity": 8, + "resource": "reservation-6-other", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-6", + "capacity": 16, + "quantity": 8, + "resource": "reservation-6", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [], + "group": "reservation-6", + "id": "reservation-6-4", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "authoritative_resource": "reservation-6", + "capacity": 16, + "quantity": 8, + "resource": "reservation-6", + "text": "Acknowledged." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-6", + "capacity": 16, + "quantity": 8, + "resource": "reservation-6", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [], + "group": "reservation-6", + "id": "reservation-6-5", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "authoritative_resource": "reservation-6", + "capacity": 16, + "quantity": 1, + "resource": "reservation-6", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-7", + "capacity": 17, + "quantity": 9, + "resource": "reservation-7", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "quantity" + ], + "group": "reservation-7", + "id": "reservation-7-0", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-7", + "capacity": 17, + "quantity": 0, + "resource": "reservation-7", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-7", + "capacity": 17, + "quantity": 9, + "resource": "reservation-7", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "quantity" + ], + "group": "reservation-7", + "id": "reservation-7-1", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-7", + "capacity": 17, + "quantity": 18, + "resource": "reservation-7", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-7", + "capacity": 17, + "quantity": 9, + "resource": "reservation-7", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "quantity" + ], + "group": "reservation-7", + "id": "reservation-7-2", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-7", + "capacity": 17, + "quantity": -1, + "resource": "reservation-7", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-7", + "capacity": 17, + "quantity": 9, + "resource": "reservation-7", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [ + "resource" + ], + "group": "reservation-7", + "id": "reservation-7-3", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "violation", + "variant": { + "authoritative_resource": "reservation-7", + "capacity": 17, + "quantity": 9, + "resource": "reservation-7-other", + "text": "Recorded." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-7", + "capacity": 17, + "quantity": 9, + "resource": "reservation-7", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [], + "group": "reservation-7", + "id": "reservation-7-4", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "authoritative_resource": "reservation-7", + "capacity": 17, + "quantity": 9, + "resource": "reservation-7", + "text": "Acknowledged." + } + }, + { + "baseline": { + "authoritative_resource": "reservation-7", + "capacity": 17, + "quantity": 9, + "resource": "reservation-7", + "text": "Recorded." + }, + "domain": "reservation", + "expected": [], + "group": "reservation-7", + "id": "reservation-7-5", + "provenance": "Authored from protocol v1; mechanically checked by specification.", + "relation": "preserve", + "variant": { + "authoritative_resource": "reservation-7", + "capacity": 17, + "quantity": 1, + "resource": "reservation-7", + "text": "Recorded." + } + } +] diff --git a/benchmarks/validator-study/prepared/manifest.json b/benchmarks/validator-study/prepared/manifest.json new file mode 100644 index 0000000..f8a8b26 --- /dev/null +++ b/benchmarks/validator-study/prepared/manifest.json @@ -0,0 +1,10 @@ +{ + "files": { + "corpus.json": "e731e2bb6352e9d09f6d73eef3861a4579a6dd07d31f8deb0647bb9378a208eb", + "protocol.md": "802d2e075447a73587f1ee7bad699bd5b735261fde913eec471c08af8722a438", + "runner.py.txt": "66247c5e34606fc6d73dfb1d006560c69501e43771853d546a701b16e0c14fad" + }, + "pairs": 144, + "schema_version": 1, + "stage": "prepared-before-capture" +} diff --git a/benchmarks/validator-study/prepared/protocol.md b/benchmarks/validator-study/prepared/protocol.md new file mode 100644 index 0000000..abd4815 --- /dev/null +++ b/benchmarks/validator-study/prepared/protocol.md @@ -0,0 +1,89 @@ +# Validator audit study protocol, version 1 + +Frozen before executing the new controlled experiment. This is an author-run, +prospective engineering experiment, not independent preregistration, a user study, +or an evaluation of natural-language truth. Historical reproduction has its own +protocol and candidate ledger; its results must not be pooled with this corpus. + +## Questions and fixed scope + +1. Which failures of specified validator policies are visible to a boolean + rejection test, a detailed ordinary-Python/pytest comparator, and the a3 audit? +2. How do targeted attribution, valid controls, and explicit execution/unknown + accounting affect the reported evidence on the same paired inputs? +3. Can historical public validator fixes be reproduced without an external team? + +No claim of superiority over well-written pytest is hypothesized. The detailed +comparator should agree with the library on per-case classifications; disagreement +is a study defect to investigate and disclose, not an observation to suppress. +No inference calls, new model downloads, or human annotation are needed. + +## Controlled corpus + +Three authored policies have mechanically checkable labels: + +- Refund completion requires a completed receipt for the same request. +- Shipment transitions must belong to a fixed application transition relation, + and the order must belong to the authenticated customer. +- A reservation must have a positive integer quantity no greater than available + capacity, and refer to the authoritative resource. + +For each policy construct eight baseline groups, four single-obligation faults +and two valid variations per group: 144 paired cases in 24 groups. Variations in +numeric values and identifiers exercise plumbing; they are not independent +semantic phenomena. There is no held-out natural corpus. No precision/recall on +natural errors, population confidence interval, usability or effort claim follows. + +Case labels and expected diagnostic IDs are fixed by the policy construction, +before evaluating a validator. A separate specification oracle verifies each +baseline and relation and identifies the intended violation. The implementation +under audit never receives the case label or expected IDs. Keep valid fields and +schema intact when mutating a domain obligation. Controls change irrelevant text +or another field while preserving the full specified policy. + +## Validator profiles + +Each policy has the same eight authored profiles: correct, omit-first-obligation, +wrong-diagnostic-ID, reject-valid-variation, crash-on-fault, unknown-on-fault, +accept-all, reject-all. Profiles are illustrative transformations of validators, +not discovered production bugs. All comparators observe the same recorded +baseline/variant outcomes; each callable is executed once per input for capture. + +The boolean comparator checks accepted, complete baselines, then accepts any +completed rejection as detection and requires acceptance of valid controls. +It retains crashes and unknowns. The detailed comparator independently implements +explicit pytest-style checks for the expected finding IDs, completeness, valid +controls and valid baselines. The a3 audit consumes the same normalized recorded +outcomes, with one declared but untested obligation per policy to exercise the +inventory. This inventory is caller-supplied, not inferred coverage. + +## Ablations and reporting + +- No attribution: report the boolean comparator next to targeted results. +- No controls: retain fault counts and mark preservation undefined; never invent + a perfect preservation score when the denominator is empty. +- Count errors as detection (deliberately invalid accounting): expose how much + this would increase the apparent score; retain the proper result alongside it. +- Hide unknowns from the denominator (deliberately invalid accounting): show both + the reduced denominator and undefined values if it removes every eligible fault. + +Publish every pair, normalized baseline/variant outcome, detailed comparison, +library audit, profile counts, and disagreements. Broken baselines are counted +explicitly and excluded from eligible mutation denominators, never counted as +detections. A separate raw variant-rejection count explains why reject-all cannot +be trusted. Report counts by policy and profile; pooled counts are descriptive. +No runtime or authoring-effort advantage is inferred from this experiment. + +## Freeze, reproduction and deviations + +Commit this protocol before running the corpus. The prepare command writes the +corpus and records hashes of this protocol and generator/comparator source before +capture/evaluation. Evaluation checks those hashes and refuses changed inputs. +Publish results with dependency/runtime metadata and a separate stable replay +output. Offline replay must reproduce classifications byte-for-byte. Changes +after first execution require a disclosed deviation; never relabel a case or +alter a library rule just to improve a score. Existing a2 research stays frozen. + +The study can support a tool demonstration with explicit limitations. External +adoption, commercial demand, time savings and arbitrary-prose correctness remain +unmeasured regardless of the results. diff --git a/benchmarks/validator-study/prepared/runner.py.txt b/benchmarks/validator-study/prepared/runner.py.txt new file mode 100644 index 0000000..9f7d6cd --- /dev/null +++ b/benchmarks/validator-study/prepared/runner.py.txt @@ -0,0 +1,437 @@ +"""Prepared, offline comparison of paired validator audit methods (protocol v1).""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import platform +import shutil +from collections import Counter +from copy import deepcopy +from pathlib import Path + +from narrative_contracts import __version__ +from narrative_contracts.model import digest +from narrative_contracts.mutations import Relation, Validity +from narrative_contracts.validator_audit import AuditCase, Obligation, Verdict, audit_validator + +ROOT = Path(__file__).resolve().parents[1] +PROTOCOL = ROOT / "paper" / "validator-study-protocol.md" +PROFILES = ( + "correct", + "omit-first-obligation", + "wrong-diagnostic-ID", + "reject-valid-variation", + "crash-on-fault", + "unknown-on-fault", + "accept-all", + "reject-all", +) +TARGETS = { + "refund": ("completion_status", "receipt_request"), + "shipping": ("transition", "customer"), + "reservation": ("quantity", "resource"), +} + + +def write_json(path, value): + path.write_text(json.dumps(value, sort_keys=True, indent=2, allow_nan=False) + "\n") + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def specification(domain, sample): + """Independent policy oracle; no calls to validator implementations.""" + if domain == "refund": + receipt = sample["receipt"] + return { + "completion_status": receipt is not None and receipt["status"] == "completed", + "receipt_request": receipt is None or receipt["request"] == sample["request"], + } + if domain == "shipping": + successor = {"queued": "packed", "packed": "shipped", "shipped": "delivered"} + return { + "transition": successor.get(sample["before"]) == sample["after"], + "customer": sample["customer"] == sample["owner"], + } + return { + "quantity": type(sample["quantity"]) is int + and sample["quantity"] in range(1, sample["capacity"] + 1), + "resource": sample["resource"] == sample["authoritative_resource"], + } + + +def make_corpus(): + rows = [] + for domain, targets in TARGETS.items(): + for group in range(8): + name = f"{domain}-{group}" + if domain == "refund": + baseline = { + "request": name, + "receipt": {"request": name, "status": "completed"}, + "text": "Recorded.", + } + faults = [ + {**baseline, "receipt": None}, + {**baseline, "receipt": {"request": name, "status": "pending"}}, + {**baseline, "receipt": {"request": name, "status": "accepted"}}, + {**baseline, "receipt": {"request": name + "-other", "status": "completed"}}, + ] + control = { + **baseline, + "request": name + "-next", + "receipt": {"request": name + "-next", "status": "completed"}, + } + elif domain == "shipping": + baseline = { + "before": "queued", + "after": "packed", + "customer": name, + "owner": name, + "text": "Recorded.", + } + faults = [ + {**baseline, "after": state} for state in ("delivered", "cancelled", "queued") + ] + [{**baseline, "customer": name + "-other"}] + control = {**baseline, "before": "packed", "after": "shipped"} + else: + baseline = { + "capacity": 10 + group, + "quantity": 2 + group, + "resource": name, + "authoritative_resource": name, + "text": "Recorded.", + } + faults = [{**baseline, "quantity": value} for value in (0, 11 + group, -1)] + [ + {**baseline, "resource": name + "-other"} + ] + control = {**baseline, "quantity": 1} + variants = faults + [{**baseline, "text": "Acknowledged."}, control] + for index, variant in enumerate(variants): + row = { + "id": f"{name}-{index}", + "domain": domain, + "group": name, + "baseline": baseline, + "variant": variant, + "relation": "violation" if index < 4 else "preserve", + "expected": [targets[0 if index < 3 else 1]] if index < 4 else [], + "provenance": "Authored from protocol v1; mechanically checked by specification.", + } + verify_row(row) + rows.append(row) + return rows + + +def verify_row(row): + if not all(specification(row["domain"], row["baseline"]).values()): + raise ValueError(f"Invalid baseline: {row['id']}") + broken = { + key for key, holds in specification(row["domain"], row["variant"]).items() if not holds + } + if broken != set(row["expected"]) or (bool(broken) != (row["relation"] == "violation")): + raise ValueError(f"Invalid relation/target: {row['id']}") + if row["baseline"] == row["variant"]: + raise ValueError(f"No-op: {row['id']}") + + +def validate(domain, sample, profile): + """Candidate implementations see samples and profile only, never case labels.""" + failures = [] + if domain == "refund": + receipt = sample["receipt"] + if receipt is None or receipt["status"] != "completed": + failures.append("completion_status") + if receipt is not None and receipt["request"] != sample["request"]: + failures.append("receipt_request") + elif domain == "shipping": + if (sample["before"], sample["after"]) not in ( + ("queued", "packed"), + ("packed", "shipped"), + ("shipped", "delivered"), + ): + failures.append("transition") + if sample["customer"] != sample["owner"]: + failures.append("customer") + else: + if type(sample["quantity"]) is not int or not 0 < sample["quantity"] <= sample["capacity"]: + failures.append("quantity") + if sample["resource"] != sample["authoritative_resource"]: + failures.append("resource") + if profile == "accept-all": + failures = [] + elif profile == "reject-all": + failures = ["reject_all"] + elif profile == "omit-first-obligation": + failures = [item for item in failures if item != TARGETS[domain][0]] + elif profile == "wrong-diagnostic-ID" and failures: + failures = ["unrelated_reason"] + elif profile == "reject-valid-variation" and sample["text"] == "Acknowledged.": + failures.append("wording") + elif profile == "crash-on-fault" and failures: + raise RuntimeError("authored validator crash") + elif profile == "unknown-on-fault" and failures: + return Verdict(False, complete=False) + return Verdict(not failures, tuple(failures)) + + +def capture(domain, sample, profile): + try: + result = validate(domain, deepcopy(sample), profile) + return { + "accepted": result.accepted, + "complete": result.complete, + "violations": list(result.violations), + "error": None, + } + except Exception as exc: + return {"error": f"{type(exc).__name__}: {exc}"} + + +def detailed_comparator(row, original, changed, *, attribute=True): + """Ordinary assertion-based comparator, independent of audit classification code.""" + if original["error"] or not original["accepted"] or not original["complete"]: + return "baseline_failed" + if changed["error"]: + return "error" + if not changed["complete"]: + return "undetermined" + if row["relation"] == "preserve": + return "preserved" if changed["accepted"] else "regressed" + if changed["accepted"]: + return "survived" + if not attribute: + return "detected" + try: + for finding_id in row["expected"]: + assert finding_id in changed["violations"], finding_id + except AssertionError: + return "unattributed_rejection" + return "detected" + + +def restore(record): + if record["error"]: + raise RuntimeError(record["error"]) + return Verdict(record["accepted"], tuple(record["violations"]), record["complete"]) + + +def case(row): + return AuditCase( + row["id"], + row["domain"], + row["baseline"], + row["variant"], + Relation(row["relation"]), + tuple(row["expected"]), + Validity.VALID, + row["provenance"], + ) + + +def analyze(rows, observations): + comparisons, reports, summary = [], {}, {} + for profile in PROFILES: + records = observations[profile] + for domain in TARGETS: + selected = [row for row in rows if row["domain"] == domain] + lookup = {} + for row in selected: + for side in ("baseline", "variant"): + key = digest(row[side]) + record = records[row["id"]][side] + if key in lookup and lookup[key] != record: + raise ValueError( + "Recorded validator is not deterministic for the same input" + ) + lookup[key] = record + report = audit_validator( + lambda sample: restore(lookup[digest(sample)]), + tuple(case(row) for row in selected), + obligations=( + Obligation(domain, f"Authored {domain} policy."), + Obligation(domain + "-untested", "An inventoried untested obligation."), + ), + validator_id=f"controlled/{domain}/{profile}/1", + ) + reports[f"{domain}/{profile}"] = report.to_dict() + outcomes = {item.id: item.outcome for item in report.cases} + for row in selected: + original, changed = (records[row["id"]][side] for side in ("baseline", "variant")) + comparisons.append( + { + "id": row["id"], + "domain": domain, + "group": row["group"], + "profile": profile, + "relation": row["relation"], + "boolean": detailed_comparator(row, original, changed, attribute=False), + "detailed_pytest": detailed_comparator(row, original, changed), + "audit": outcomes[row["id"]], + } + ) + selected = [r for r in comparisons if r["profile"] == profile] + eligible_faults = [ + r for r in selected if r["relation"] == "violation" and r["audit"] != "baseline_failed" + ] + eligible_controls = [ + r for r in selected if r["relation"] == "preserve" and r["audit"] != "baseline_failed" + ] + counts = Counter(r["audit"] for r in selected) + denominator = len(eligible_faults) + hidden_unknowns = denominator - counts["undetermined"] + summary[profile] = { + "counts": dict(sorted(counts.items())), + "eligible_faults": denominator, + "eligible_controls": len(eligible_controls), + "boolean_detections": sum(r["boolean"] == "detected" for r in eligible_faults), + "targeted_detections": counts["detected"], + "detection_score": counts["detected"] / denominator if denominator else None, + "preservation_rate": counts["preserved"] / len(eligible_controls) + if eligible_controls + else None, + "boolean_success_without_target": sum( + r["boolean"] == "detected" and r["audit"] != "detected" for r in selected + ), + "detailed_disagreements": sum(r["detailed_pytest"] != r["audit"] for r in selected), + "raw_fault_rejections_ignoring_baseline": sum( + row["relation"] == "violation" + and not records[row["id"]]["variant"]["error"] + and not records[row["id"]]["variant"]["accepted"] + and records[row["id"]]["variant"]["complete"] + for row in rows + ), + "ablations": { + "without_controls": { + "detection_score": counts["detected"] / denominator if denominator else None, + "preservation_rate": None, + }, + "invalid_error_credit_score": (counts["detected"] + counts["error"]) / denominator + if denominator + else None, + "invalid_hide_unknowns": { + "denominator": hidden_unknowns, + "score": counts["detected"] / hidden_unknowns if hidden_unknowns else None, + }, + }, + } + return ( + { + "schema_version": 1, + "library_version": __version__, + "pairs": len(rows), + "groups": len({row["group"] for row in rows}), + "profiles": summary, + "disagreements": sum(s["detailed_disagreements"] for s in summary.values()), + }, + comparisons, + reports, + ) + + +def prepare(destination): + destination.mkdir(parents=True, exist_ok=False) + rows = make_corpus() + write_json(destination / "corpus.json", rows) + shutil.copyfile(PROTOCOL, destination / "protocol.md") + shutil.copyfile(__file__, destination / "runner.py.txt") + write_json( + destination / "manifest.json", + { + "schema_version": 1, + "stage": "prepared-before-capture", + "pairs": len(rows), + "files": { + name: sha(destination / name) + for name in ("corpus.json", "protocol.md", "runner.py.txt") + }, + }, + ) + + +def load_prepared(prepared): + manifest = json.loads((prepared / "manifest.json").read_text()) + expected_files = {"corpus.json", "protocol.md", "runner.py.txt"} + if set(manifest["files"]) != expected_files: + raise ValueError("Unexpected prepared manifest members") + for name, expected in manifest["files"].items(): + if sha(prepared / name) != expected: + raise ValueError(f"Prepared content changed: {name}") + if ( + sha(Path(__file__)) != manifest["files"]["runner.py.txt"] + or sha(PROTOCOL) != manifest["files"]["protocol.md"] + ): + raise ValueError("Runner/protocol differ from prepared snapshot") + rows = json.loads((prepared / "corpus.json").read_text()) + if len(rows) != manifest["pairs"]: + raise ValueError("Incorrect pair count") + for row in rows: + verify_row(row) + return rows + + +def run(prepared, output, captured=None): + rows = load_prepared(prepared) + output.mkdir(parents=True, exist_ok=False) + if captured is None: + observations = { + profile: { + row["id"]: { + side: capture(row["domain"], row[side], profile) + for side in ("baseline", "variant") + } + for row in rows + } + for profile in PROFILES + } + write_json( + output / "environment.json", + {"python": platform.python_version(), "library_version": __version__}, + ) + else: + recorded_manifest = json.loads((captured / "manifest.json").read_text()) + if recorded_manifest["prepared_manifest_sha256"] != sha(prepared / "manifest.json"): + raise ValueError("Capture belongs to another prepared corpus") + if sha(captured / "observations.json") != recorded_manifest["observations_sha256"]: + raise ValueError("Captured outcomes changed") + observations = json.loads((captured / "observations.json").read_text()) + summary, comparisons, reports = analyze(rows, observations) + write_json(output / "observations.json", observations) + write_json(output / "summary.json", summary) + write_json(output / "comparisons.json", comparisons) + write_json(output / "audit-reports.json", reports) + write_json( + output / "manifest.json", + { + "prepared_manifest_sha256": sha(prepared / "manifest.json"), + "observations_sha256": sha(output / "observations.json"), + }, + ) + if summary["disagreements"]: + raise AssertionError( + "Detailed comparator disagrees: retained outputs require investigation" + ) + print(json.dumps(summary, indent=2)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("prepare", "evaluate", "replay")) + parser.add_argument("--prepared", type=Path, required=True) + parser.add_argument("--output", type=Path) + parser.add_argument("--captured", type=Path) + args = parser.parse_args() + if args.action == "prepare": + prepare(args.prepared) + else: + if args.output is None or (args.action == "replay" and args.captured is None): + parser.error("output is required; replay also requires captured") + run(args.prepared, args.output, args.captured if args.action == "replay" else None) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/validator_study.py b/benchmarks/validator_study.py new file mode 100644 index 0000000..9f7d6cd --- /dev/null +++ b/benchmarks/validator_study.py @@ -0,0 +1,437 @@ +"""Prepared, offline comparison of paired validator audit methods (protocol v1).""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import platform +import shutil +from collections import Counter +from copy import deepcopy +from pathlib import Path + +from narrative_contracts import __version__ +from narrative_contracts.model import digest +from narrative_contracts.mutations import Relation, Validity +from narrative_contracts.validator_audit import AuditCase, Obligation, Verdict, audit_validator + +ROOT = Path(__file__).resolve().parents[1] +PROTOCOL = ROOT / "paper" / "validator-study-protocol.md" +PROFILES = ( + "correct", + "omit-first-obligation", + "wrong-diagnostic-ID", + "reject-valid-variation", + "crash-on-fault", + "unknown-on-fault", + "accept-all", + "reject-all", +) +TARGETS = { + "refund": ("completion_status", "receipt_request"), + "shipping": ("transition", "customer"), + "reservation": ("quantity", "resource"), +} + + +def write_json(path, value): + path.write_text(json.dumps(value, sort_keys=True, indent=2, allow_nan=False) + "\n") + + +def sha(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def specification(domain, sample): + """Independent policy oracle; no calls to validator implementations.""" + if domain == "refund": + receipt = sample["receipt"] + return { + "completion_status": receipt is not None and receipt["status"] == "completed", + "receipt_request": receipt is None or receipt["request"] == sample["request"], + } + if domain == "shipping": + successor = {"queued": "packed", "packed": "shipped", "shipped": "delivered"} + return { + "transition": successor.get(sample["before"]) == sample["after"], + "customer": sample["customer"] == sample["owner"], + } + return { + "quantity": type(sample["quantity"]) is int + and sample["quantity"] in range(1, sample["capacity"] + 1), + "resource": sample["resource"] == sample["authoritative_resource"], + } + + +def make_corpus(): + rows = [] + for domain, targets in TARGETS.items(): + for group in range(8): + name = f"{domain}-{group}" + if domain == "refund": + baseline = { + "request": name, + "receipt": {"request": name, "status": "completed"}, + "text": "Recorded.", + } + faults = [ + {**baseline, "receipt": None}, + {**baseline, "receipt": {"request": name, "status": "pending"}}, + {**baseline, "receipt": {"request": name, "status": "accepted"}}, + {**baseline, "receipt": {"request": name + "-other", "status": "completed"}}, + ] + control = { + **baseline, + "request": name + "-next", + "receipt": {"request": name + "-next", "status": "completed"}, + } + elif domain == "shipping": + baseline = { + "before": "queued", + "after": "packed", + "customer": name, + "owner": name, + "text": "Recorded.", + } + faults = [ + {**baseline, "after": state} for state in ("delivered", "cancelled", "queued") + ] + [{**baseline, "customer": name + "-other"}] + control = {**baseline, "before": "packed", "after": "shipped"} + else: + baseline = { + "capacity": 10 + group, + "quantity": 2 + group, + "resource": name, + "authoritative_resource": name, + "text": "Recorded.", + } + faults = [{**baseline, "quantity": value} for value in (0, 11 + group, -1)] + [ + {**baseline, "resource": name + "-other"} + ] + control = {**baseline, "quantity": 1} + variants = faults + [{**baseline, "text": "Acknowledged."}, control] + for index, variant in enumerate(variants): + row = { + "id": f"{name}-{index}", + "domain": domain, + "group": name, + "baseline": baseline, + "variant": variant, + "relation": "violation" if index < 4 else "preserve", + "expected": [targets[0 if index < 3 else 1]] if index < 4 else [], + "provenance": "Authored from protocol v1; mechanically checked by specification.", + } + verify_row(row) + rows.append(row) + return rows + + +def verify_row(row): + if not all(specification(row["domain"], row["baseline"]).values()): + raise ValueError(f"Invalid baseline: {row['id']}") + broken = { + key for key, holds in specification(row["domain"], row["variant"]).items() if not holds + } + if broken != set(row["expected"]) or (bool(broken) != (row["relation"] == "violation")): + raise ValueError(f"Invalid relation/target: {row['id']}") + if row["baseline"] == row["variant"]: + raise ValueError(f"No-op: {row['id']}") + + +def validate(domain, sample, profile): + """Candidate implementations see samples and profile only, never case labels.""" + failures = [] + if domain == "refund": + receipt = sample["receipt"] + if receipt is None or receipt["status"] != "completed": + failures.append("completion_status") + if receipt is not None and receipt["request"] != sample["request"]: + failures.append("receipt_request") + elif domain == "shipping": + if (sample["before"], sample["after"]) not in ( + ("queued", "packed"), + ("packed", "shipped"), + ("shipped", "delivered"), + ): + failures.append("transition") + if sample["customer"] != sample["owner"]: + failures.append("customer") + else: + if type(sample["quantity"]) is not int or not 0 < sample["quantity"] <= sample["capacity"]: + failures.append("quantity") + if sample["resource"] != sample["authoritative_resource"]: + failures.append("resource") + if profile == "accept-all": + failures = [] + elif profile == "reject-all": + failures = ["reject_all"] + elif profile == "omit-first-obligation": + failures = [item for item in failures if item != TARGETS[domain][0]] + elif profile == "wrong-diagnostic-ID" and failures: + failures = ["unrelated_reason"] + elif profile == "reject-valid-variation" and sample["text"] == "Acknowledged.": + failures.append("wording") + elif profile == "crash-on-fault" and failures: + raise RuntimeError("authored validator crash") + elif profile == "unknown-on-fault" and failures: + return Verdict(False, complete=False) + return Verdict(not failures, tuple(failures)) + + +def capture(domain, sample, profile): + try: + result = validate(domain, deepcopy(sample), profile) + return { + "accepted": result.accepted, + "complete": result.complete, + "violations": list(result.violations), + "error": None, + } + except Exception as exc: + return {"error": f"{type(exc).__name__}: {exc}"} + + +def detailed_comparator(row, original, changed, *, attribute=True): + """Ordinary assertion-based comparator, independent of audit classification code.""" + if original["error"] or not original["accepted"] or not original["complete"]: + return "baseline_failed" + if changed["error"]: + return "error" + if not changed["complete"]: + return "undetermined" + if row["relation"] == "preserve": + return "preserved" if changed["accepted"] else "regressed" + if changed["accepted"]: + return "survived" + if not attribute: + return "detected" + try: + for finding_id in row["expected"]: + assert finding_id in changed["violations"], finding_id + except AssertionError: + return "unattributed_rejection" + return "detected" + + +def restore(record): + if record["error"]: + raise RuntimeError(record["error"]) + return Verdict(record["accepted"], tuple(record["violations"]), record["complete"]) + + +def case(row): + return AuditCase( + row["id"], + row["domain"], + row["baseline"], + row["variant"], + Relation(row["relation"]), + tuple(row["expected"]), + Validity.VALID, + row["provenance"], + ) + + +def analyze(rows, observations): + comparisons, reports, summary = [], {}, {} + for profile in PROFILES: + records = observations[profile] + for domain in TARGETS: + selected = [row for row in rows if row["domain"] == domain] + lookup = {} + for row in selected: + for side in ("baseline", "variant"): + key = digest(row[side]) + record = records[row["id"]][side] + if key in lookup and lookup[key] != record: + raise ValueError( + "Recorded validator is not deterministic for the same input" + ) + lookup[key] = record + report = audit_validator( + lambda sample: restore(lookup[digest(sample)]), + tuple(case(row) for row in selected), + obligations=( + Obligation(domain, f"Authored {domain} policy."), + Obligation(domain + "-untested", "An inventoried untested obligation."), + ), + validator_id=f"controlled/{domain}/{profile}/1", + ) + reports[f"{domain}/{profile}"] = report.to_dict() + outcomes = {item.id: item.outcome for item in report.cases} + for row in selected: + original, changed = (records[row["id"]][side] for side in ("baseline", "variant")) + comparisons.append( + { + "id": row["id"], + "domain": domain, + "group": row["group"], + "profile": profile, + "relation": row["relation"], + "boolean": detailed_comparator(row, original, changed, attribute=False), + "detailed_pytest": detailed_comparator(row, original, changed), + "audit": outcomes[row["id"]], + } + ) + selected = [r for r in comparisons if r["profile"] == profile] + eligible_faults = [ + r for r in selected if r["relation"] == "violation" and r["audit"] != "baseline_failed" + ] + eligible_controls = [ + r for r in selected if r["relation"] == "preserve" and r["audit"] != "baseline_failed" + ] + counts = Counter(r["audit"] for r in selected) + denominator = len(eligible_faults) + hidden_unknowns = denominator - counts["undetermined"] + summary[profile] = { + "counts": dict(sorted(counts.items())), + "eligible_faults": denominator, + "eligible_controls": len(eligible_controls), + "boolean_detections": sum(r["boolean"] == "detected" for r in eligible_faults), + "targeted_detections": counts["detected"], + "detection_score": counts["detected"] / denominator if denominator else None, + "preservation_rate": counts["preserved"] / len(eligible_controls) + if eligible_controls + else None, + "boolean_success_without_target": sum( + r["boolean"] == "detected" and r["audit"] != "detected" for r in selected + ), + "detailed_disagreements": sum(r["detailed_pytest"] != r["audit"] for r in selected), + "raw_fault_rejections_ignoring_baseline": sum( + row["relation"] == "violation" + and not records[row["id"]]["variant"]["error"] + and not records[row["id"]]["variant"]["accepted"] + and records[row["id"]]["variant"]["complete"] + for row in rows + ), + "ablations": { + "without_controls": { + "detection_score": counts["detected"] / denominator if denominator else None, + "preservation_rate": None, + }, + "invalid_error_credit_score": (counts["detected"] + counts["error"]) / denominator + if denominator + else None, + "invalid_hide_unknowns": { + "denominator": hidden_unknowns, + "score": counts["detected"] / hidden_unknowns if hidden_unknowns else None, + }, + }, + } + return ( + { + "schema_version": 1, + "library_version": __version__, + "pairs": len(rows), + "groups": len({row["group"] for row in rows}), + "profiles": summary, + "disagreements": sum(s["detailed_disagreements"] for s in summary.values()), + }, + comparisons, + reports, + ) + + +def prepare(destination): + destination.mkdir(parents=True, exist_ok=False) + rows = make_corpus() + write_json(destination / "corpus.json", rows) + shutil.copyfile(PROTOCOL, destination / "protocol.md") + shutil.copyfile(__file__, destination / "runner.py.txt") + write_json( + destination / "manifest.json", + { + "schema_version": 1, + "stage": "prepared-before-capture", + "pairs": len(rows), + "files": { + name: sha(destination / name) + for name in ("corpus.json", "protocol.md", "runner.py.txt") + }, + }, + ) + + +def load_prepared(prepared): + manifest = json.loads((prepared / "manifest.json").read_text()) + expected_files = {"corpus.json", "protocol.md", "runner.py.txt"} + if set(manifest["files"]) != expected_files: + raise ValueError("Unexpected prepared manifest members") + for name, expected in manifest["files"].items(): + if sha(prepared / name) != expected: + raise ValueError(f"Prepared content changed: {name}") + if ( + sha(Path(__file__)) != manifest["files"]["runner.py.txt"] + or sha(PROTOCOL) != manifest["files"]["protocol.md"] + ): + raise ValueError("Runner/protocol differ from prepared snapshot") + rows = json.loads((prepared / "corpus.json").read_text()) + if len(rows) != manifest["pairs"]: + raise ValueError("Incorrect pair count") + for row in rows: + verify_row(row) + return rows + + +def run(prepared, output, captured=None): + rows = load_prepared(prepared) + output.mkdir(parents=True, exist_ok=False) + if captured is None: + observations = { + profile: { + row["id"]: { + side: capture(row["domain"], row[side], profile) + for side in ("baseline", "variant") + } + for row in rows + } + for profile in PROFILES + } + write_json( + output / "environment.json", + {"python": platform.python_version(), "library_version": __version__}, + ) + else: + recorded_manifest = json.loads((captured / "manifest.json").read_text()) + if recorded_manifest["prepared_manifest_sha256"] != sha(prepared / "manifest.json"): + raise ValueError("Capture belongs to another prepared corpus") + if sha(captured / "observations.json") != recorded_manifest["observations_sha256"]: + raise ValueError("Captured outcomes changed") + observations = json.loads((captured / "observations.json").read_text()) + summary, comparisons, reports = analyze(rows, observations) + write_json(output / "observations.json", observations) + write_json(output / "summary.json", summary) + write_json(output / "comparisons.json", comparisons) + write_json(output / "audit-reports.json", reports) + write_json( + output / "manifest.json", + { + "prepared_manifest_sha256": sha(prepared / "manifest.json"), + "observations_sha256": sha(output / "observations.json"), + }, + ) + if summary["disagreements"]: + raise AssertionError( + "Detailed comparator disagrees: retained outputs require investigation" + ) + print(json.dumps(summary, indent=2)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("prepare", "evaluate", "replay")) + parser.add_argument("--prepared", type=Path, required=True) + parser.add_argument("--output", type=Path) + parser.add_argument("--captured", type=Path) + args = parser.parse_args() + if args.action == "prepare": + prepare(args.prepared) + else: + if args.output is None or (args.action == "replay" and args.captured is None): + parser.error("output is required; replay also requires captured") + run(args.prepared, args.output, args.captured if args.action == "replay" else None) + + +if __name__ == "__main__": + main() diff --git a/docs/project-name.md b/docs/project-name.md new file mode 100644 index 0000000..f963e76 --- /dev/null +++ b/docs/project-name.md @@ -0,0 +1,15 @@ +# MateProbe and historical package names + +**MateProbe by Mate4B** is the current project. Since `0.1.0a4`, install +`mateprobe` and optionally `pytest-mateprobe`, import `mateprobe`, and use the +`mateprobe` CLI and pytest fixture with `--mateprobe-report`. +See the [migration guide](migration-mateprobe.md) for complete commands. + +The controlled comparison and historical-validator studies were recorded with +`narrative-contracts==0.1.0a3`. Their source snapshots, protocol files and reports +retain that distribution name and `narrative_contracts` imports. These are +historical identifiers, not instructions for current application integrations. + +The [study reproduction guide](validator-study.md#offline-reproduction) separates +byte-for-byte a3 replay from a4 compatibility replay. No frozen corpus, observation, +label or report is renamed or regenerated to make migration checks pass. diff --git a/docs/validator-study.md b/docs/validator-study.md new file mode 100644 index 0000000..aa1fb81 --- /dev/null +++ b/docs/validator-study.md @@ -0,0 +1,144 @@ +# Controlled comparisons and historical reproductions + +This author-run a3 study tests validator-audit accounting and reproduces public +historical defects. It does not evaluate natural-language truth, independent +adoption, user effort, or commercial demand. It uses no LLM calls or new model +weights. Historical examples are software-validator bugs, not natural LLM errors. + +## Controlled comparison + +The [protocol](../paper/validator-study-protocol.md) was committed before execution +(`5de4a06`); the corpus and source were frozen at `97005d6`. Three authored policies +(refund receipts, shipment transitions/ownership, and reservation capacity/resource) +produce 144 pairs in 24 shared baseline groups: 96 faults and 48 valid controls. +Each pair is evaluated under eight authored validator profiles, giving 1,152 paired +classifications. Numeric/identifier repetitions do not create independent domains. + +Comparators consume identical recorded verdicts: boolean rejection, detailed +ordinary Python assertions exercised through pytest, and `audit_validator`. +The detailed comparator includes the same baseline, completeness and target checks +as the audit. It does not call the audit's classifier. + +| Validator profile | Boolean detections | Attributed detections | Preserved controls | Other observed results | +|---|---:|---:|---:|---| +| Correct | 96/96 | 96/96 | 48/48 | None | +| Missing first obligation | 24/96 | 24/96 | 48/48 | 72 survivors | +| Wrong finding ID | 96/96 | 0/96 | 48/48 | 96 unattributed rejections | +| Reject valid wording | 96/96 | 96/96 | 24/48 | 24 false rejections | +| Crash on faults | 0/96 | 0/96 | 48/48 | 96 execution errors | +| Unknown on faults | 0/96 | 0/96 | 48/48 | 96 incomplete verdicts | +| Accept all | 0/96 | 0/96 | 48/48 | 96 survivors | +| Reject all | Undefined | Undefined | Undefined | All 144 baselines fail | + +**Detailed assertions and the audit agree on all 1,152 classifications.** This +supports equivalent accounting on these cases, not greater detection power than +well-written pytest. A wrong diagnostic ID does not prove the rejection was unsafe; +it fails to establish the particular diagnostic obligation under test. + +Ablations expose the information lost by weaker accounting: removing controls hides +24 false rejections; counting crashes as detections would falsely change 0/96 to +96/96; dropping all unknowns produces an empty denominator, not 100%. Ignoring +baselines makes reject-all appear to reject all 96 faulty variants. These examples +are constructed demonstrations of failure modes, not estimates of their prevalence. +The declared inventory also retains one untested obligation per policy. That does +not discover obligations the author failed to list. + +[Stable results](../benchmarks/validator-study/evaluation/summary.json) · +[All comparisons](../benchmarks/validator-study/evaluation/comparisons.json) · +[Prepared corpus](../benchmarks/validator-study/prepared/corpus.json). + +## Historical package execution + +A purposive screening ledger contains ten public candidates; four were included +and six excluded with reasons. Before collection, upstream reports were read and +minimal inputs/relations were frozen. Initial preparation errors, environment +failures and subsequent compatibility corrections are documented in the +[historical protocol](../paper/historical-validator-protocol.md) and the +[follow-up protocol](../paper/historical-followup-protocol.md). This is retrospective +reproduction, not independent preregistration or discovery of previously unknown bugs. + +Each included issue has a trigger pair and a separate changed valid-control pair. +The packages execute in isolated environments with exact versions, installed +dependency locks, downloaded artifact hashes and raw normalized outcomes retained. +Only the enum trigger is a violation; the other triggers are valid inputs which +should remain accepted. Exceptions are retained as errors, not detections. + +| Upstream issue | Versions finally exercised | Trigger before → after | Valid control | Result | +|---|---|---|---|---| +| [jsonschema #575](https://github.com/python-jsonschema/jsonschema/issues/575), enum `0` vs `False` | 3.0.1 → 3.0.2 | Survived → detected | Preserved in both | Reproduced | +| [jsonschema #1018](https://github.com/python-jsonschema/jsonschema/issues/1018), cached boolean schema | 4.17.1 → 4.17.3 | Error → preserved | Preserved in both | Reproduced | +| [Marshmallow #2891](https://github.com/marshmallow-code/marshmallow/issues/2891), uppercase FILE | 4.2.0 → 4.2.1 | Regressed → preserved | Preserved in both | Reproduced | +| [Marshmallow #2936](https://github.com/marshmallow-code/marshmallow/issues/2936), IDN email | 4.2.3 → 4.2.4 | Preserved → preserved | Preserved in both | Not reproduced by selected input | + +The first run reproduced only FILE: jsonschema 3.0.x could not import +`pkg_resources`, 4.17.2 was unavailable from the package index, and the chosen IDN +input already passed. An explicit follow-up pinned setuptools 70.3.0 for both old +jsonschema versions and substituted available 4.17.1 for 4.17.2. A first follow-up +lockfile conflict is also retained. The corrected follow-up reproduces three fixes +across two packages; the IDN case remains unchanged and unsuccessful. There is no +claim about the unavailable 4.17.2 artifact. Three of four purposively selected +cases is not a population success rate or evidence of three external adopters. + +[Initial results](../benchmarks/historical-validator-results/evaluation/summary.json) · +[Final compatibility results](../benchmarks/historical-validator-results/compatibility-followup-v2/evaluation/summary.json) · +[Screening ledger](../benchmarks/historical-validator-results/prepared/inputs.json). + +## Offline reproduction + +From a repository checkout with MateProbe installed (`pip install -e '.[dev]'`), +replay all three studies offline with: + +```sh +python benchmarks/study_replay.py --output /tmp/mateprobe-study-replay +``` + +This wrapper temporarily maps historical imports to MateProbe inside its own +process. The original drivers and their source hashes remain unchanged. It compares +all study results with the frozen evidence, allowing only the explicit transition +from `library_version: 0.1.0a3` to the installed MateProbe version in reports and the +controlled summary. It never rewrites archived reports or removes version fields. +The output includes `compatibility.json` with the actual replay version. This is a +compatibility check, not new historical package execution. + +To reproduce the original a3 JSON byte-for-byte, use a **separate environment**: + +```sh +python3.12 -m venv /tmp/mateprobe-historical-a3 +. /tmp/mateprobe-historical-a3/bin/activate +python -m pip install narrative-contracts==0.1.0a3 +``` + +Then run the unchanged historical drivers from the repository root: + +```sh +python benchmarks/validator_study.py replay --prepared benchmarks/validator-study/prepared --captured benchmarks/validator-study/evaluation --output /tmp/controlled-replay +python benchmarks/historical_validator_study.py replay --captured benchmarks/historical-validator-results/captured --output /tmp/historical-initial-replay +python benchmarks/historical_followup.py replay --captured benchmarks/historical-validator-results/compatibility-followup-v2/captured --output /tmp/historical-final-replay +``` + +Output directories must be new. Replay verifies hashes, reclassifies recorded +verdicts and requires no historical third-party validator packages, model or network +once MateProbe or the a3 library is installed. CI compares original a3 JSON outputs +byte-for-byte and separately checks replay under published MateProbe a4. Replay does +not independently re-execute third-party packages. In the historical a3 environment, fresh execution of the final version +pairs requires explicitly permitting downloads: + +```sh +python benchmarks/historical_followup.py collect --network --work /tmp/historical-envs --output /tmp/historical-fresh +python benchmarks/historical_followup.py replay --captured /tmp/historical-fresh --output /tmp/historical-fresh-evaluation +``` + +Use Python 3.12 for the recorded historical environment. Installation requires +package-index availability and uses the frozen dependency versions; captured pip +reports record the actual downloaded artifact hashes. Install commands pin versions, +not wheel hashes, so compare manifests if verifying download identity. Operating +system/build differences may matter. Failed setup is not a successful reproduction. + +## Interpretation + +The study adds independently originated bug reports, actual old/new package runs, +a detailed comparator with zero disagreements, and explicit component ablations. +It does not establish a new mutation algorithm, natural-output accuracy, adoption, +or reduced engineering effort. The main demonstrated benefit is a reusable protocol +and report for paired evidence; careful handwritten tests can express the same checks. +The original a2 synthetic/model evidence remains separate and unchanged. diff --git a/llms.txt b/llms.txt index 8bed4fa..603a6fb 100644 --- a/llms.txt +++ b/llms.txt @@ -25,6 +25,8 @@ Supply authoritative state and branch selection from trusted application code. V - [Contract semantics](https://mate4b.github.io/mateprobe/docs/contracts.md): typed equality, unknowns, errors, policies, and mutation denominators. - [Real-output mutations](https://mate4b.github.io/mateprobe/docs/real-output-mutations.md): 32 captured responses, 384 authored variants, and 64 accepted prose-only challenges; no semantic accuracy claim. +- [Controlled and historical validator studies](https://mate4b.github.io/mateprobe/docs/validator-study.md): 1,152 paired classifications agree with detailed assertions; three historical fixes reproduced across two packages, with failures and selection limits retained. Includes historical a3 and current MateProbe replay instructions. + ## Audit existing validators You do not need to change your validator to start. These APIs are available in a4. diff --git a/mkdocs.yml b/mkdocs.yml index e3f7aea..508577d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -23,6 +23,7 @@ validation: nav: - Start: - Overview: index.md + - MateProbe and package names: docs/project-name.md - Test AI output validators: docs/testing-ai-output-validators.md - Try a validator audit: docs/launch.md - First audit recipe: docs/first-audit.md @@ -37,6 +38,7 @@ nav: - Contract semantics: docs/contracts.md - Architecture: docs/architecture.md - Evidence: + - Validator comparison study: docs/validator-study.md - Discovery and a3 integration: docs/agent-readiness.md - Agent integration trial: docs/agent-adoption.md - Real-output mutations: docs/real-output-mutations.md diff --git a/paper/draft.md b/paper/draft.md index 282a32e..63a4e86 100644 --- a/paper/draft.md +++ b/paper/draft.md @@ -1,16 +1,35 @@ -# MateProbe: Deterministic State-Conditioned Validation with Mutation and Invariance Audits +# MateProbe: Attributable Mutation Audits for State-Aware Validators -**Technical report, alpha artifact v0.1.0a2 with follow-up evidence — 29 September 2026.** +**Technical report, a3 validator study with preserved a2 evidence — 29 September 2026.** The project was renamed to **MateProbe by Mate4B** in `0.1.0a4`. Reported experiments retain their original package versions, names and frozen artifacts; the rename is not a new evaluation. -Status: public engineering report, not submitted or peer reviewed. The artifact includes synthetic author-constructed cases, a second independent software integration, and a small frozen corpus of actual model generations. There are no independent human labels or LLM-judge labels. Natural-output acceptance is not semantic accuracy. +Status: public engineering report, not submitted or peer reviewed. The artifact includes synthetic author-constructed cases, a second author-implemented application integration, and a small frozen corpus of actual model generations. There are no independent human labels or LLM-judge labels. Natural-output acceptance is not semantic accuracy. ## Abstract -Language-generation systems that maintain authoritative structured state can evaluate a useful subset of output obligations without another language model. We present MateProbe (released as Narrative Contracts for the experiments reported here), a dependency-free Python library that separates exact predicates over structured state and declared claims from lexical heuristics over prose. Contracts produce four-valued outcomes with branch-local scope, versioned configuration and diagnostic evidence. An accompanying mutation audit requires attributable detections and valid-variation controls, and exposes equivalent, unreviewed and baseline-invalid cases rather than counting them as successful tests. A synthetic feasibility corpus contains 720 paired cases in 48 shared scenario groups. The strict profile detects 384 of 480 authored faults and accepts 192 of 240 valid variations. Retained failures expose three boundaries: semantic restatement without lexical overlap, contradictions outside supplied claim annotations and negated phrases rejected by regex. An additional 21-case mutation campaign retains five survivors and one valid-control regression. A frozen 32-request pilot with two local model families records structural failures and contract findings without assigning semantic-accuracy labels. A follow-up 32-response corpus from larger user-installed models supports a 384-case authored mutation audit: 64 declaration faults, 96 lexical faults and 96 schema faults are detected; 48 changed controls are preserved and 16 no-ops excluded. All 64 inserted prose-only contradictions pass and are reported separately from in-scope scores. A separate customer-support workflow demonstrates trusted branch selection outside LifeCard. These artifacts illustrate behavior and limitations; they do not establish natural-output accuracy, superiority to LLM judges or broad cross-domain generalization. Independent evaluation is future work, not a prerequisite for releasing the engineering artifact. +MateProbe by Mate4B (formerly Narrative Contracts; the study used alpha a3 +as `narrative-contracts`) provides a dependency-free Python core and an optional +pytest plugin for +attributable mutation audits of existing validators and bounded state-conditioned +checks. An audit pairs accepted baselines with authored faults and valid variations, +requires the intended diagnostic findings, and preserves crashes, incomplete evidence, +false rejections and untested obligations. A new controlled study evaluates 144 pairs +from three exact policies under eight authored validator profiles. Detailed ordinary +Python assertions and the audit agree on all 1,152 classifications. A boolean-rejection +comparator counts 96 wrong-ID rejections as detections, while a fault-only view hides +24 rejected valid controls. These are constructed accounting examples, not a measured +frequency of defects or evidence of superiority over careful pytest tests. A separate +retrospective study screens ten public candidates and executes four historical package +comparisons. After disclosed environment-compatibility corrections, three fixes are +reproduced across jsonschema and Marshmallow; the fourth selected input passes before +and after, and remains a non-reproduction. Earlier a2 artifacts include 720 synthetic +pairs, an expanded mutation campaign, and frozen model outputs, with prose-grounding +failures retained. The evidence supports a reusable engineering protocol and diagnostic +artifact. It does not establish natural-output accuracy, independent adoption, lower +authoring effort, a novel general mutation algorithm, or replacement of LLM judges. ## 1. Problem and intended contribution @@ -18,7 +37,7 @@ Consider an application where a trusted engine decides state transitions and a t Deterministic checks are attractive when the application already possesses the necessary evidence. Their reproducibility does not by itself establish construct validity: a length threshold can be implemented perfectly and remain a poor measure of informative content. We therefore treat the evaluator as an object requiring testing. -The artifact contributes (1) an explicit boundary between structured invariants and textual heuristics, (2) branch-scoped checks and four-valued results, and (3) an audit connecting labelled violations to individual rule/code/scope targets while also measuring preservation of valid variants. The present contribution is an engineering artifact and experimental protocol. A claim of research novelty or broad effectiveness awaits the study below. +The original a2 artifact contributes (1) an explicit boundary between structured invariants and textual heuristics, (2) branch-scoped checks and four-valued results, and (3) an audit connecting labelled violations to individual rule/code/scope targets while also measuring preservation of valid variants. The a3 contribution additionally packages independent-validator adapters, obligation inventories and auditable outcome accounting. The controlled and historical experiments below evaluate this engineering artifact; broad effectiveness and research novelty beyond this combination remain unestablished. ## 2. Related work @@ -26,7 +45,7 @@ IFEval operationalizes verifiable instructions using reproducible checks [1]. It Mutation-based meta-evaluation also has close antecedents. Breaking Models to Test the Judge introduces controlled semantic defects in domain class diagrams to evaluate semantic judges [3]. When Knowledge Changes uses mutations and metamorphic relations to assess RAG under evolving corpora [4]. We do not claim to introduce mutation testing for semantic evaluators. -LLM-as-a-judge work documents both useful agreement with human preferences and systematic biases [5]. Those results motivate careful evaluator calibration, not a blanket claim that judges are ineffective. Existing tools such as Promptfoo already support deterministic assertions and custom code [6]. The proposed study must therefore investigate the value of state-conditioned obligations, diagnostic attribution and audit methodology beyond ordinary assertions. +LLM-as-a-judge work documents both useful agreement with human preferences and systematic biases [5]. Those results motivate careful evaluator calibration, not a blanket claim that judges are ineffective. Existing tools such as Promptfoo already support deterministic assertions and custom code [6]. The controlled study below therefore compares diagnostic attribution and audit accounting with ordinary assertions; it does not establish greater detection power or lower authoring cost. ## 3. Model and semantics @@ -57,7 +76,7 @@ The separate source-mutation experiment changes selected evaluator implementatio ## 5. Implementation -The core uses Python dataclasses and the standard library. It exposes a contract protocol, eight built-in rule classes, a strict allowlisted JSON loader, a CLI, a LifeCard-shaped adapter and a mutation runner. The optional pytest package provides fixtures, threshold assertions and JSON reports. Neither package calls an LLM. +The core uses Python dataclasses and the standard library. The original a2 core exposes a contract protocol, eight built-in rule classes, a strict allowlisted JSON loader, a CLI, a LifeCard-shaped adapter and a mutation runner. The optional pytest package provides fixtures, threshold assertions and JSON reports. Neither package calls an LLM. Reports include SHA-256 digests of canonical inputs and configured contracts, rule and package versions, and policy. They omit timestamps. Configuration digests identify declared settings, not executable source; release hashes and environment records complement them. The adapter maps each outcome to its own state reference and leaves trusted state execution in the application. @@ -181,19 +200,110 @@ reproduces deterministic reports; regenerating outputs is not promised to be byt across runtime versions and hardware. The evaluation was replayed under a2 package metadata, with unchanged rule semantics. Source-mutation checks still kill all eight selected mutants. -## 8. Threats to validity +## 8. Independent validator audits in a3 + +The a3 `audit_validator` interface adapts plain application samples and a small +`Verdict` rather than requiring the library's document/claim model. Stable finding +IDs encode diagnostic targets and optional scope. A rejection for an unrelated ID +is recorded as `unattributed_rejection`; exceptions and incomplete verdicts remain +errors and unknowns. An obligation inventory reports untested items without claiming +to discover missing obligations. Field bindings and bounded relational rules support +explicit equality, numeric limits and allowed transitions; they do not establish +arbitrary-prose entailment or full state-machine reachability. + +The controlled protocol was committed at `5de4a06` before executing prepared +cases/source (`97005d6`). Three policies concern receipt-supported refund completion, +shipment transitions/ownership, and reservation capacity/resource identity. Eight +baseline groups per policy receive four faults and two controls: 144 pairs in 24 +groups, evaluated under eight authored validator profiles (1,152 classifications). +A separately implemented policy oracle verifies relations before validator execution. +All comparison methods consume the same recorded outcomes. The ordinary-assertion +reference has the same inputs, completeness information and expected diagnostic IDs. + +| Profile | Boolean detections | Attributed detections | Preserved controls | +|---|---:|---:|---:| +| Correct | 96/96 | 96/96 | 48/48 | +| Omit first obligation | 24/96 | 24/96 | 48/48 | +| Wrong diagnostic ID | 96/96 | 0/96 | 48/48 | +| Reject valid variation | 96/96 | 96/96 | 24/48 | +| Crash on faults | 0/96 | 0/96 | 48/48 | +| Unknown on faults | 0/96 | 0/96 | 48/48 | +| Accept all | 0/96 | 0/96 | 48/48 | +| Reject all | Undefined | Undefined | Undefined | + +There are zero classification disagreements between detailed assertions and the +audit. Reject-all fails all 144 baselines, leaving no eligible denominator. Removing +controls hides 24 regressions in the wording-sensitive profile. Deliberately crediting +crashes would change 0/96 to 96/96; discarding every unknown produces an empty +denominator. One inventoried obligation per policy remains untested. These ablations +illustrate information requirements, not the prevalence of poor testing practices. +The wrong-ID profile rejects actual faults; its failure is diagnostic attribution, +not proof that the application accepted unsafe output. + +The detailed comparator's agreement is evidence against a claim of additional +detection power over correctly implemented pytest checks. The artifact packages +paired execution and evidence accounting. Authoring-effort savings were not measured. +There is no held-out natural corpus; values within a policy reuse few templates. +See [protocol](validator-study-protocol.md) and +[complete data and reproduction](../docs/validator-study.md). + +## 9. Retrospective historical reproduction + +Ten purposively selected public candidates were screened using primary issue and +release records. Four were included, with two changed pairs each: a trigger and a +valid control. Six were excluded for scope or insufficient before/after evidence. +The labels originate in documented upstream behavior, while adapters and minimal +reproducers are author-written. Two preflight label/configuration mistakes were +corrected after reading original reports and before executing historical packages; +original preparation and deviations are retained. This is not independent evaluation. + +| Public issue | Final version comparison | Trigger before → after | Reproduced | +|---|---|---|---| +| jsonschema #575: enum boolean/numeric equality | 3.0.1 → 3.0.2 | Survived → detected | Yes | +| jsonschema #1018: cached boolean schemas | 4.17.1 → 4.17.3 | Error → preserved | Yes | +| Marshmallow #2891: uppercase FILE scheme | 4.2.0 → 4.2.1 | Regressed → preserved | Yes | +| Marshmallow #2936: IDN email | 4.2.3 → 4.2.4 | Preserved → preserved | No | + +All final preservation controls pass before and after. A first run reproduced only +the FILE case: old jsonschema imports failed on `pkg_resources`, version 4.17.2 was +unavailable, and the selected IDN input passed both releases. An explicitly +exploratory compatibility follow-up pinned setuptools 70.3.0 and substituted 4.17.1. +A conflicting first follow-up lockfile was also retained and corrected before a +further run. Inputs and labels were never changed in response to these outcomes. +The final run reproduces three fixes in two packages; unsuccessful IDN reproduction +is not evidence against the upstream fix, only against this chosen trigger. + +Exact package/dependency versions, installed-artifact hashes, raw outcomes, labels +and adapters are archived. Offline replay reclassifies those outcomes; it is not a +fresh run of old packages. Fresh execution uses isolated environments and explicit +package downloads. The historical and controlled datasets are not pooled. This +small retrospective sample demonstrates applicability to real validator defects; +it does not estimate natural LLM error detection, bug-discovery rate, usability, +or independently adopted deployments. Ordinary tests could detect these same bugs. +See the [screening protocol](historical-validator-protocol.md), +[compatibility follow-up](historical-followup-protocol.md), and +[primary links and results](../docs/validator-study.md). + +The frozen studies retain their a3 package names, runner sources and report metadata. +The current MateProbe checkout provides `benchmarks/study_replay.py`, which verifies +unchanged hashes and compares reclassified results while permitting only the explicit +library-version metadata transition. CI also replays the original drivers with the +published a3 package and checks exact outputs. Neither replay is a new third-party +package execution; see [the reproduction guide](../docs/validator-study.md). + +## 10. Threats to validity **Construction bias:** the authors designed both rules and most transformations. Covered-family success is expected and cannot validate real-world semantic accuracy. Challenge labels are author judgments and have not been independently adjudicated. **Limited diversity:** labels for three domains and two languages share few templates and the same schema. Numeric variations do not create independent semantic phenomena. The natural pilot contains small local models and authored scenarios, not production traffic. No held-out fault taxonomy or external adopter was evaluated; generation failures can dominate the observed pipeline. -**Comparator limitations:** the excerpt baseline omits parts of LifeCard and has fewer capabilities. No result here establishes superiority to a full existing tool or LLM judge. A future comparison must give competitors equivalent state, history and obligations. +**Comparator limitations:** the a2 excerpt baseline omits parts of LifeCard and has fewer capabilities. The a3 detailed-assertion reference agrees with the audit on the controlled corpus; it was implemented by the same authors, not an independently developed comparator. No result establishes superiority to a full existing tool or LLM judge. Historical failures are software-validator defects, not spontaneous LLM errors. **Oracle gap:** declaration consistency is not textual entailment. Grounding, annotation completeness and authoritative context remain application responsibilities. A complete report is complete only for executed checks. **Operational scope:** bounded snapshots and supplied history do not establish global reachability, liveness or future narrative quality. Regex configuration and normalization are language-specific maintenance obligations. -## 9. Optional independent study and community validation +## 11. Optional independent study and community validation The accompanying protocol specifies frozen research questions, corpus separation, independent annotations, judge calibration, matched-input comparisons, ablations, cluster-aware statistics and cost accounting. The highest-priority question is whether attributable synthetic mutation detection predicts recall on independently labelled natural errors while maintaining an acceptable valid-output rejection rate. This question remains unanswered. The public alpha ships with issue templates for reproducible counterexamples and contribution guidance; community publication invites scrutiny but does not itself constitute independent validation. Human review and calibrated model-judge comparisons are optional follow-up studies rather than release gates. diff --git a/paper/historical-followup-protocol.md b/paper/historical-followup-protocol.md new file mode 100644 index 0000000..ee4ac78 --- /dev/null +++ b/paper/historical-followup-protocol.md @@ -0,0 +1,42 @@ +# Historical compatibility follow-up protocol + +Prepared after the first historical run and before executing this follow-up. +This is explicitly exploratory compatibility recovery, not a preplanned success. +Original ledger, inputs, failures and results remain available unchanged. + +The first run reproduced the uppercase FILE-scheme fix, but both jsonschema 3.0.x +versions failed importing pkg_resources, jsonschema 4.17.2 was unavailable from the +package index, and the chosen IDN email was accepted by both Marshmallow versions. + +Two changes are fixed in advance of the new run: + +1. Add setuptools 70.3.0 (which supplies pkg_resources) to both jsonschema 3.0.x + environments. Keep all other captured dependencies pinned. +2. Use jsonschema 4.17.1 as the available predecessor to 4.17.3. Issue #1018 describes + the regression starting in 4.16.0; the pinned changelog associates the fix with + 4.17.3. This is not evidence about the unavailable 4.17.2 package. + +All eight paired inputs, labels, policies and adapters remain identical. The IDN +case is rerun unchanged; no search for a different successful reproducer is made. +The same two Marshmallow version comparisons act as checks on repeat execution. + +The follow-up driver, base collector, protocol, dependency requirements and parent +capture identity are frozen before new collection. The driver hash is included in +the prepared inputs; requirements hashes are verified before installing. Actual +installation reports retain artifact URLs and hashes, along with pip-freeze locks. +Offline replay and fresh package execution are distinct operations. Both the +initial and follow-up tables must be reported, including failed reproduction. + +A successful reproduction requires the issue-triggering outcome to change from +incorrect to correct and the preservation control to pass in both versions. +A fixed example is a historical regression reproduced, not a newly discovered bug, +an external adopter, or evidence of a sensitivity advantage over detailed pytest. + +## Lockfile correction after the first follow-up attempt + +The first follow-up accidentally retained setuptools 84.0.0 while also requesting +70.3.0. Pip refused these two environments before executing their samples. Other +candidates ran; that capture and replay remain under `compatibility-followup/`. +Version 2 replaces the existing setuptools pin instead of appending a conflicting +one. The corrected driver and requirements were frozen again before new execution. +Case inputs and labels remain unchanged. Results of all attempts are retained. diff --git a/paper/historical-validator-protocol.md b/paper/historical-validator-protocol.md new file mode 100644 index 0000000..3bb1a4a --- /dev/null +++ b/paper/historical-validator-protocol.md @@ -0,0 +1,80 @@ +# Historical validator study protocol + +Status: version 2 prepared 2026-09-29 (America/Argentina/Buenos_Aires), before execution. + +This is a retrospective reproduction study, not an independent preregistration. +The unit is a documented validator defect with a public primary issue, pull +request, advisory, or release note naming the fix. A candidate is eligible only +when the upstream record describes an observable validation or validation-adjacent +behavior change and identifies both a before and after release (or commits). +Configuration omissions, documented defaults, feature requests, type-checking +only changes, and regressions without a later fix are excluded. + +Before any reproduction, the candidate ledger, minimal inputs, expected behavior, +adapter source, and this protocol are hashed. Reproductions use fresh virtual +environments below `work/`; no package is installed into the study checkout or +global interpreter. Network access is explicit (`collect --network`). The offline +replay consumes only the captured normalized verdicts and validates their hashes. + +Each included candidate has one valid baseline control and one invalid or newly +accepted variant. The control must remain accepted in both versions. The expected +finding is authored from the primary record and kept separate from adapter logic. +`audit_validator` outcomes are reported verbatim; errors, unsupported cases, and +failed baselines remain in the ledger and do not count as fixes. This protocol +does not tune library rules and makes no claim that the harness outperforms pytest. + +The frozen target is 8--12 candidates, with at least three reproduced fixes when +the available Python/runtime/package constraints permit. Fewer successful cases +are reported honestly. For every candidate we retain package/version/runtime, +source URL and ID, rationale, input digest, adapter hash, raw before/after output, +normalized audit report, and environment `pip freeze` hash. + +Primary sources are upstream GitHub issues, pull requests, release notes, or +security advisories. Minimal reproducers are authored for this study and do not +copy upstream test files. + +## Preflight corrections and selection limitations + +Version 1 is retained under `preflight-v1/` and in Git commit `5035606`. +No package reproduction was executed with that preparation. Before collection, +review of the original upstream reports found: uppercase file schemes and IDN +email addresses are valid variations, not faults; the file example needs explicit +`schemes={"file"}`; the boolean-ref issue requires a cached boolean schema in a +RefResolver store rather than a plain local reference; and the IDN case should +exercise a Unicode TLD. Those inputs/adapters and relations are corrected in v2. +Unsupported release guesses for excluded candidates are replaced with null. +The original serialization description for Pydantic #12348 was also corrected +after reading its actual ModuleType reproducer. These are pre-execution corrections, +not findings about the third-party libraries. Original flawed inputs are not evidence. + +Every included issue now links to its upstream report and a commit-pinned changelog +establishing the fixed version. The ten candidates were a purposive, bounded +screening sample, not an exhaustive or randomly sampled review of issues. +Exclusion means outside this experiment or insufficient documented evidence; it +does not mean an upstream report is invalid. No population discovery rate follows. + +Each included issue has two actual changed pairs: the issue-triggering variant +and a valid preservation control. The enum issue is a violation; the other three +are preservation issues, including a valid input triggering an exception. +Replay compares the full recorded normalized results against the same a3 audit; +exceptions remain errors rather than validation rejections. Baseline verdicts +and child failures are retained. The summary calls a fix reproduced only when +the before-version misbehaves on the trigger, the after-version satisfies it, +and the control is preserved in both versions. + +Freeze writes the prepared ledger and source/protocol hashes once. Setup never +rewrites it. Collection verifies prepared hashes, uses isolated version-specific +environments, and records exact installed dependency versions plus downloaded +artifact hashes from pip's installation report. Re-collection installs those +recorded dependency versions when supplied via `--lock-from`. Offline replay +does not reinstall packages and is distinguished from fresh package execution. +Hashes detect accidental changes; mutable manifests are not tamper-proof evidence. + +## Bootstrap-only execution deviation + +Two initial collection attempts stopped while creating the first virtual environment, +before installing a historical package or executing any sample. This macOS Python +distribution aborts when its executable is copied by EnvBuilder; the collector now +uses symlinks, matching the working command-line venv behavior. The source/protocol +were re-frozen before sample execution. The preceding prepared snapshot is retained +as `prepared-v2-bootstrap/`; case selection, labels and adapter logic are unchanged. diff --git a/paper/validator-study-protocol.md b/paper/validator-study-protocol.md new file mode 100644 index 0000000..abd4815 --- /dev/null +++ b/paper/validator-study-protocol.md @@ -0,0 +1,89 @@ +# Validator audit study protocol, version 1 + +Frozen before executing the new controlled experiment. This is an author-run, +prospective engineering experiment, not independent preregistration, a user study, +or an evaluation of natural-language truth. Historical reproduction has its own +protocol and candidate ledger; its results must not be pooled with this corpus. + +## Questions and fixed scope + +1. Which failures of specified validator policies are visible to a boolean + rejection test, a detailed ordinary-Python/pytest comparator, and the a3 audit? +2. How do targeted attribution, valid controls, and explicit execution/unknown + accounting affect the reported evidence on the same paired inputs? +3. Can historical public validator fixes be reproduced without an external team? + +No claim of superiority over well-written pytest is hypothesized. The detailed +comparator should agree with the library on per-case classifications; disagreement +is a study defect to investigate and disclose, not an observation to suppress. +No inference calls, new model downloads, or human annotation are needed. + +## Controlled corpus + +Three authored policies have mechanically checkable labels: + +- Refund completion requires a completed receipt for the same request. +- Shipment transitions must belong to a fixed application transition relation, + and the order must belong to the authenticated customer. +- A reservation must have a positive integer quantity no greater than available + capacity, and refer to the authoritative resource. + +For each policy construct eight baseline groups, four single-obligation faults +and two valid variations per group: 144 paired cases in 24 groups. Variations in +numeric values and identifiers exercise plumbing; they are not independent +semantic phenomena. There is no held-out natural corpus. No precision/recall on +natural errors, population confidence interval, usability or effort claim follows. + +Case labels and expected diagnostic IDs are fixed by the policy construction, +before evaluating a validator. A separate specification oracle verifies each +baseline and relation and identifies the intended violation. The implementation +under audit never receives the case label or expected IDs. Keep valid fields and +schema intact when mutating a domain obligation. Controls change irrelevant text +or another field while preserving the full specified policy. + +## Validator profiles + +Each policy has the same eight authored profiles: correct, omit-first-obligation, +wrong-diagnostic-ID, reject-valid-variation, crash-on-fault, unknown-on-fault, +accept-all, reject-all. Profiles are illustrative transformations of validators, +not discovered production bugs. All comparators observe the same recorded +baseline/variant outcomes; each callable is executed once per input for capture. + +The boolean comparator checks accepted, complete baselines, then accepts any +completed rejection as detection and requires acceptance of valid controls. +It retains crashes and unknowns. The detailed comparator independently implements +explicit pytest-style checks for the expected finding IDs, completeness, valid +controls and valid baselines. The a3 audit consumes the same normalized recorded +outcomes, with one declared but untested obligation per policy to exercise the +inventory. This inventory is caller-supplied, not inferred coverage. + +## Ablations and reporting + +- No attribution: report the boolean comparator next to targeted results. +- No controls: retain fault counts and mark preservation undefined; never invent + a perfect preservation score when the denominator is empty. +- Count errors as detection (deliberately invalid accounting): expose how much + this would increase the apparent score; retain the proper result alongside it. +- Hide unknowns from the denominator (deliberately invalid accounting): show both + the reduced denominator and undefined values if it removes every eligible fault. + +Publish every pair, normalized baseline/variant outcome, detailed comparison, +library audit, profile counts, and disagreements. Broken baselines are counted +explicitly and excluded from eligible mutation denominators, never counted as +detections. A separate raw variant-rejection count explains why reject-all cannot +be trusted. Report counts by policy and profile; pooled counts are descriptive. +No runtime or authoring-effort advantage is inferred from this experiment. + +## Freeze, reproduction and deviations + +Commit this protocol before running the corpus. The prepare command writes the +corpus and records hashes of this protocol and generator/comparator source before +capture/evaluation. Evaluation checks those hashes and refuses changed inputs. +Publish results with dependency/runtime metadata and a separate stable replay +output. Offline replay must reproduce classifications byte-for-byte. Changes +after first execution require a disclosed deviation; never relabel a case or +alter a library rule just to improve a score. Existing a2 research stays frozen. + +The study can support a tool demonstration with explicit limitations. External +adoption, commercial demand, time savings and arbitrary-prose correctness remain +unmeasured regardless of the results. diff --git a/scripts/build_docs.py b/scripts/build_docs.py index 72e3518..80dd75b 100644 --- a/scripts/build_docs.py +++ b/scripts/build_docs.py @@ -57,6 +57,7 @@ "docs/api-a4.md", "docs/natural-benchmark.md", "docs/policy-regression.md", + "docs/project-name.md", "docs/pydantic.md", "docs/pypi-publishing.md", "docs/quickstart.md", @@ -66,9 +67,13 @@ "docs/support-workflow.md", "docs/testing-ai-output-validators.md", "docs/validator-audit.md", + "docs/validator-study.md", "docs/verification.md", "paper/draft.md", "paper/protocol.md", + "paper/validator-study-protocol.md", + "paper/historical-validator-protocol.md", + "paper/historical-followup-protocol.md", "paper/real-mutation-protocol.md", "paper/release-protocol.md", ) diff --git a/tests/conftest.py b/tests/conftest.py index c6481d5..f9a811c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1 +1,13 @@ +from pathlib import Path +from runpy import run_path + +import pytest + pytest_plugins = ["pytester"] + + +@pytest.fixture +def study_runtime(): + runtime = run_path(str(Path(__file__).resolve().parents[1] / "benchmarks/study_replay.py")) + with runtime["legacy_imports"](): + yield runtime diff --git a/tests/test_historical_validator_study.py b/tests/test_historical_validator_study.py new file mode 100644 index 0000000..774d574 --- /dev/null +++ b/tests/test_historical_validator_study.py @@ -0,0 +1,69 @@ +import importlib.util +import json +import shutil +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +@pytest.fixture +def study(study_runtime): + return study_runtime["load_runner"]("historical_validator_study") + + +CAPTURED = ROOT / "benchmarks/historical-validator-results/captured" + + +def test_prepared_records_include_valid_trigger_variations(study, study_runtime): + data = study.load_prepared() + assert len(data["candidates"]) == 10 + pairs = data["authored_inputs"] + for id in ("marshmallow-file-url", "marshmallow-idn-email", "jsonschema-bool-ref-cache"): + assert pairs[id][0]["relation"] == "preserve" + assert pairs[id][0]["expected"] == [] + assert pairs["jsonschema-enum-bool-int"][0]["relation"] == "violation" + assert all( + json.dumps(p["baseline"], sort_keys=True) != json.dumps(p["variant"], sort_keys=True) + for rows in pairs.values() + for p in rows + ) + + +def test_historical_offline_replay_matches_frozen_evidence(tmp_path, study, study_runtime): + study.replay(CAPTURED, tmp_path / "out") + study_runtime["compare_outputs"]("historical", study.OUT / "evaluation", tmp_path / "out") + + +def test_historical_capture_tampering_is_rejected(tmp_path, study, study_runtime): + captured = tmp_path / "captured" + shutil.copytree(CAPTURED, captured) + (captured / "captured.json").write_text("{}") + with pytest.raises(ValueError, match="Capture changed"): + study.replay(captured, tmp_path / "out") + + +def test_historical_versions_and_paired_controls_are_retained(study, study_runtime): + summary = json.loads((study.OUT / "evaluation/summary.json").read_text()) + assert summary["included"] == 4 + assert summary["excluded"] == 6 + assert len(summary["results"]) == summary["included"] + for item in summary["results"]: + assert set(item["outcomes"]) == {item["before_version"], item["after_version"]} + + +def test_compatibility_followup_replays_and_keeps_nonreproduction(tmp_path, study, study_runtime): + spec = importlib.util.spec_from_file_location( + "historical_followup_test", ROOT / "benchmarks/historical_followup.py" + ) + followup = importlib.util.module_from_spec(spec) + spec.loader.exec_module(followup) + followup.verify() + summary = followup.base.replay(followup.OUT / "captured", tmp_path / "out") + assert summary["fixes_reproduced"] == 3 + assert ( + next(r for r in summary["results"] if r["id"] == "marshmallow-idn-email")["fix_reproduced"] + is False + ) + study_runtime["compare_outputs"]("followup", followup.OUT / "evaluation", tmp_path / "out") diff --git a/tests/test_study_replay.py b/tests/test_study_replay.py new file mode 100644 index 0000000..f167c02 --- /dev/null +++ b/tests/test_study_replay.py @@ -0,0 +1,33 @@ +import sys +from pathlib import Path +from runpy import run_path +from types import ModuleType + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +def test_historical_import_bridge_cleans_up_after_failure(): + runtime = run_path(str(ROOT / "benchmarks/study_replay.py")) + before = { + key: value for key, value in sys.modules.items() if key.startswith("narrative_contracts") + } + with pytest.raises(RuntimeError, match="study failure"): + with runtime["legacy_imports"](): + runtime["load_runner"]("validator_study") + raise RuntimeError("study failure") + after = { + key: value for key, value in sys.modules.items() if key.startswith("narrative_contracts") + } + assert after == before + + +def test_historical_import_bridge_refuses_mixed_runtime(monkeypatch): + runtime = run_path(str(ROOT / "benchmarks/study_replay.py")) + legacy = ModuleType("narrative_contracts") + monkeypatch.setitem(sys.modules, "narrative_contracts", legacy) + with pytest.raises(RuntimeError, match="already imported legacy"): + with runtime["legacy_imports"](): + pytest.fail("Must not mix installed library implementations") + assert sys.modules["narrative_contracts"] is legacy diff --git a/tests/test_validator_study.py b/tests/test_validator_study.py new file mode 100644 index 0000000..17c0013 --- /dev/null +++ b/tests/test_validator_study.py @@ -0,0 +1,88 @@ +import json +import shutil +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +@pytest.fixture +def study(study_runtime): + return study_runtime["load_runner"]("validator_study") + + +PREPARED = ROOT / "benchmarks/validator-study/prepared" +CAPTURED = ROOT / "benchmarks/validator-study/evaluation" + + +def test_specification_rejects_bad_label_before_running_a_validator(study, study_runtime): + row = study.make_corpus()[0] + row["expected"] = ["unrelated_reason"] + with pytest.raises(ValueError, match="Invalid relation/target"): + study.verify_row(row) + + +def test_prepared_corruption_fails_closed(tmp_path, study, study_runtime): + prepared = tmp_path / "prepared" + shutil.copytree(PREPARED, prepared) + corpus = prepared / "corpus.json" + corpus.write_text(corpus.read_text() + " ") + with pytest.raises(ValueError, match="Prepared content changed"): + study.load_prepared(prepared) + + +def test_replay_matches_frozen_evidence_with_explicit_version_transition( + tmp_path, study, study_runtime +): + output = tmp_path / "replay" + study.run(PREPARED, output, CAPTURED) + study_runtime["compare_outputs"]("controlled", CAPTURED, output) + + +def test_capture_corruption_fails_closed(tmp_path, study, study_runtime): + captured = tmp_path / "captured" + shutil.copytree(CAPTURED, captured) + (captured / "observations.json").write_text("{}") + with pytest.raises(ValueError, match="Captured outcomes changed"): + study.run(PREPARED, tmp_path / "out", captured) + + +def test_reexecution_matches_all_recorded_validator_observations(study, study_runtime): + observations = json.loads((CAPTURED / "observations.json").read_text()) + for profile, records in observations.items(): + for row in study.load_prepared(PREPARED): + for side in ("baseline", "variant"): + assert study.capture(row["domain"], row[side], profile) == records[row["id"]][side] + + +def test_metrics_do_not_reward_wrong_reasons_or_crashes(study, study_runtime): + profiles = json.loads((CAPTURED / "summary.json").read_text())["profiles"] + assert profiles["wrong-diagnostic-ID"]["boolean_success_without_target"] == 96 + assert profiles["crash-on-fault"]["targeted_detections"] == 0 + assert profiles["reject-valid-variation"]["preservation_rate"] == 0.5 + assert profiles["reject-all"]["detection_score"] is None + assert profiles["unknown-on-fault"]["ablations"]["invalid_hide_unknowns"]["score"] is None + assert all(p["detailed_disagreements"] == 0 for p in profiles.values()) + + +@pytest.mark.parametrize("change", ["outcome", "evidence", "version", "membership"]) +def test_compatibility_check_rejects_changed_results(tmp_path, study, study_runtime, change): + output = tmp_path / "replay" + study.run(PREPARED, output, CAPTURED) + path = output / "audit-reports.json" + reports = json.loads(path.read_text()) + key = next(iter(reports)) + if change == "outcome": + reports[key]["cases"][0]["outcome"] = "detected" + elif change == "evidence": + reports[key]["cases"][0]["variant"]["evidence"] = [{"library_version": "hidden-change"}] + elif change == "version": + reports[key]["library_version"] = "unverified-version" + else: + del reports[key] + path.write_text(json.dumps(reports)) + with pytest.raises( + ValueError, match="Replay report membership|Unexpected replay|differs beyond" + ): + study_runtime["compare_outputs"]("controlled", CAPTURED, output)