diff --git a/research/selected-json-utf8-replay/README.md b/research/selected-json-utf8-replay/README.md new file mode 100644 index 0000000000..f9bff442b9 --- /dev/null +++ b/research/selected-json-utf8-replay/README.md @@ -0,0 +1,128 @@ +# Selected JSON: a source-preserving UTF-8 replay + +The pinned `scripts/verify_selected_result.py` reads the selected record, +certificate and source manifest with the platform default codec. On native +Windows CPython 3.12.10 (`cp1252`, UTF-8 mode off), a valid UTF-8 U+211D in +any one of those JSON documents raises `UnicodeDecodeError`. A valid UTF-8 +`supplier/preuve-é.txt` path instead becomes mojibake and fails file lookup. + +This package reproduces both failures, verifies **`-X utf8` on the unchanged +checker** as an immediate remedy, and tests the three explicit-encoding reads +in [UTF8-READS.patch](UTF8-READS.patch) only in private temporary copies. +It adds no mathematical claim or new kappa. + +## Result and source boundary + +The audit starts from canonical main +`721c03822c421b7f7fcbb2598993b947c4d88419`, priority revision 8. Its exact input +is SHA256 +`d6b305eac5b9bdb8172c469ad30a1655a60fa0aceac8b1bf610e1b73f51babd8`. +[SOURCES.json](SOURCES.json) pins the Git blob and task/selected-record provenance. + +The selected `certificates/frontier6030-source.json` itself pins this checker. +Applying the patch directly would break that source closure. This contribution +therefore leaves the checker, selected manifest, scientific sources and frontier +unchanged. The current selected metadata passed its normal binding-only baseline +in this session: **the demonstrated failures use complete synthetic metadata +fixtures, not a claim that the current selected result fails.** + +The tested unchanged-source invocation is: + +```sh +python3 -X utf8 -B scripts/verify_selected_result.py +# The current baseline also calls that checker in the same interpreter: +python3 -X utf8 -B scripts/research_selected_baseline.py +``` + +`-X utf8` selects the decoding mode; it does not disable source hashing or modify +any input. The three-line explicit-read candidate avoids changing the whole +interpreter's default encoding, but needs a maintainer-approved source transition +or a successor entry point before integration. + +## Reproduce the bounded audit + +Python 3.12 and the standard library suffice. From the repository root: + +```sh +python3 -B scripts/research_package.py --package research/selected-json-utf8-replay --check-only +python3 -B scripts/research_package.py --package research/selected-json-utf8-replay +``` + +For a Git-free export, run: + +```sh +python3 -B research/selected-json-utf8-replay/verify.py +``` + +Expect `PASS_SELECTED_JSON_UTF8_REPLAY tests=8` followed by the native-locale +flag and `no_new_kappa`. The runner registers ordinary and optimized audit +drivers. The child checker deliberately runs with assertions enabled in both +cases, because its CLI rejects `-O` by contract. + +The eight test methods cover: + +- An ASCII positive control under the legacy, unchanged UTF-8 and candidate modes. +- Separate U+211D failures at the record, certificate and manifest read boundaries. +- Successful replay of each boundary using both remedies. +- Exact Unicode inventory-path spelling and the legacy lookup failure. +- Invalid UTF-8 byte `0xff`: the legacy codec accepts it as U+00FF; both remedies + reject it at every boundary. Certificate digests are recomputed for this + fixture, so rejection tests encoding rather than a stale hash. +- Source corruption and a whitespace-only certificate byte change: both remedies + still reject, with the original inventory and certificate diagnostics. +- Native cp1252 CLI execution without any encoding mock, when that host is + available. This method is explicitly skipped on other native locales. + +The portable codec control changes `io.text_encoding` only in an isolated child +process, models cp1252 when UTF-8 mode is off, and respects the explicitly enabled +UTF-8 mode. The native run independently checks that model. All fixture files +are written as bytes; source copies and child stdout/stderr are checked as bytes. +No package or pinned input is modified during replay. + +## Task, overlap and scope + +The expertise-based `verifier-engineering` picker suggested `portable-replay` +with nonce `269949e813c9ca9d693f6a23b2d0d0c2`. [TASK.json](TASK.json) records the +refined scope, original card, stop rule and next questions. This is a bounded +interpreter mismatch, within the card's allowed variations. + +The newest visible handoffs in Discussion #1166, onboarding #1163, active +Discussions and targeted all-state PR searches were read. The repository's +`find_overlap.py` was also run, but its remote refresh could not run because +`gh` was unavailable; the sparse checkout omitted most local research notes. +The connector searches were bounded, and the visible Discussion page was not +a complete comment corpus. These checks do not establish exhaustive novelty. + +[PR #5973](https://github.com/CrocSwap/integer-mult-bounds/pull/5973) repairs +invoice stdout byte binding; it is a separate issue and is not duplicated here. +[PR #2531](https://github.com/CrocSwap/integer-mult-bounds/pull/2531) repairs a +Lean-literal decoder, [PR #1834](https://github.com/CrocSwap/integer-mult-bounds/pull/1834) +repairs status-test reads, and [PR #2554](https://github.com/CrocSwap/integer-mult-bounds/pull/2554) +handles console output labels. They establish the encoding-portability precedent; +this contribution adds a pinned three-boundary selected-checker audit and a +verified invocation that preserves its selected source hash. + +No native-operator replay, supplier admission, full multiplication theorem, +global locale audit or source-policy transition is established. The five +inherited framework assumptions remain. Local finite checks are author-checked +evidence, not independent human or proof-kernel acceptance. + +## Open next questions + +1. Integrate explicit UTF-8 reads without invalidating frozen source closures. + First inventory current and historical consumers of the checker, then choose + a successor entry point or an explicit source transition. The blocker is that + maintainer policy decision, not the three-line change itself. +2. Run these exact fixtures on a native UTF-8 POSIX host and another supported + Windows Python version. Report native-only skips separately. Those environments + were unavailable locally; the portable codec model is not a substitute for + claiming those native executions. + +## Credit + +The selected-checker logic and its source-binding contract are Douglas Colkitt's +with the retained contributors. The current productive-helper baseline is Mihir +Srivastava's; the codec precedents above retain their original authorship. +Prepared for `josecarlosgonzalezroque-design` with substantial OpenAI Codex +assistance in investigation, implementation, testing and writing. One agent, +no subagents. New work is Apache-2.0; original notices remain intact. diff --git a/research/selected-json-utf8-replay/SOURCES.json b/research/selected-json-utf8-replay/SOURCES.json new file mode 100644 index 0000000000..98b10a5335 --- /dev/null +++ b/research/selected-json-utf8-replay/SOURCES.json @@ -0,0 +1,24 @@ +{ + "schema_version": 1, + "base_commit": "721c03822c421b7f7fcbb2598993b947c4d88419", + "files": [ + { + "path": "certificates/selected-result.json", + "usage": "provenance", + "git_blob": "d59b5035e7b4cbaa9d688a1d32dd0d6eb79c466a", + "sha256": "9b8d0eea95b3bf0dfcb7938a732a6439b87a2685dec2a5ab9e525b3b34debbfa" + }, + { + "path": "docs/research/agent-task-catalog.json", + "usage": "provenance", + "git_blob": "3404fbdd5e7529a0674eef26114b5071e4586cec", + "sha256": "ead6cdc9d0af6d58e8a49744d8ec8c7c15e13d2ba93c276c434bcf9b51b93a81" + }, + { + "path": "scripts/verify_selected_result.py", + "usage": "input", + "git_blob": "6882a4700e410dec569b51ae2eaf1a19cc019195", + "sha256": "d6b305eac5b9bdb8172c469ad30a1655a60fa0aceac8b1bf610e1b73f51babd8" + } + ] +} diff --git a/research/selected-json-utf8-replay/TASK.json b/research/selected-json-utf8-replay/TASK.json new file mode 100644 index 0000000000..a8fe222034 --- /dev/null +++ b/research/selected-json-utf8-replay/TASK.json @@ -0,0 +1,72 @@ +{ + "schema_version": 1, + "task_id": "portable-replay", + "priority_revision": 8, + "base_commit": "721c03822c421b7f7fcbb2598993b947c4d88419", + "original_card": { + "id": "portable-replay", + "question": "Can one reproducible environment or artifact-comparison failure be repaired?", + "starting_points": [ + "research/gzip-container-reproducibility/README.md", + "docs/ci-verification.md" + ], + "methods": [ + "Reproduce one portability failure and add a minimal cross-environment fix.", + "Audit one canonical-payload comparison for omitted proof-relevant fields." + ], + "acceptance_gate": "Preserve immutable source hashes and proof-relevant content; a different compressed byte stream alone is not a mathematical mismatch.", + "first_step": { + "command": [ + "python3", + "-B", + "scripts/research_selected_baseline.py" + ], + "expected": "PASS_RESEARCH_BASELINE_BINDINGS_ONLY", + "inspect": "Resolve the current selected review, separate complex/ordinary BIT/admitted BIT/outer budgets, and compare the card against live handoffs before starting. Historical ceilings require exact source/model equivalence.", + "scope": "Source-binding orientation only; not a scientific replay or all-size proof. Materialize selected inputs if this checkout is sparse." + }, + "first_experiment": "Reproduce one specific mismatch or audit one canonical comparison for omitted proof-relevant fields.", + "bottleneck_link": "Portable exact checks prevent environment failures from blocking valid contributions.", + "allowed_variations": [ + "One interpreter/version mismatch or one payload field comparison." + ], + "scope_change_triggers": [ + "Removing immutable-source pins because generated compressed bytes differ." + ], + "overlap_terms": [ + "gzip container", + "reproducibility", + "canonical payload" + ] + }, + "scope_status": "refined", + "actual_scope": "Reproduce the three locale-dependent selected-checker JSON reads, verify a source-preserving UTF-8 invocation, and test an explicit-encoding candidate in private temporary copies.", + "scope_change_reason": "The portable-replay card permits one interpreter/environment mismatch. The selected checker is itself pinned in frontier6030-source.json, so this result preserves its bytes and supplies a tested invocation plus a candidate patch for future maintainer integration.", + "bottleneck_link": "Exact source-binding checks should accept valid UTF-8 metadata and exact Unicode input names on a native cp1252 host without relaxing byte hashes or repinning scientific sources.", + "budget_minutes": 60, + "stop_rule": "Stop after all three JSON boundaries, exact Unicode path spelling, malformed-encoding rejection, source corruption and byte-hash controls pass for the workaround and candidate. No scientific cold replay or wider codec sweep.", + "outcome": "completed", + "evidence": "verify.py: eight test methods exercise the real pinned CLI in temporary closures. Native Windows cp1252 corroborates the portable codec model. UTF8-READS.patch is tested only in temporary copies.", + "author": "josecarlosgonzalezroque-design", + "ai_assistance": "Substantial OpenAI Codex assistance for investigation, implementation, testing and writing; one agent, no subagents.", + "open_questions": [ + { + "question": "Can the explicit UTF-8 read patch be integrated through a successor checker without invalidating frozen selected and historical source closures?", + "why_it_matters": "Callers should not need to remember an interpreter flag; direct edits to the pinned checker invalidate selected source hashes.", + "first_step": "Inventory live and historical consumers of scripts/verify_selected_result.py, then propose a versioned entry point or explicit maintainer source transition using the tested three-line patch.", + "blocked_by": "Maintainer choice of source-transition policy; this contribution does not alter the selected manifest." + }, + { + "question": "Do the same exact fixtures pass on a native UTF-8 POSIX host and another supported Windows Python version?", + "why_it_matters": "The portable cp1252 model and native CPython 3.12.10 run cover different parts of the portability claim.", + "first_step": "Run the registered replay unchanged and report the interpreter, native locale, passed tests and native-only skip.", + "blocked_by": "Those environments were not available in this session." + } + ], + "assignment": { + "seed": "269949e813c9ca9d693f6a23b2d0d0c2", + "track": "verifier-engineering", + "scope_key": "831cc1a3c8f9ff01b197", + "selection": "Expertise-based track selection after reading newest handoffs; no suitable bounded request taken." + } +} diff --git a/research/selected-json-utf8-replay/UTF8-READS.patch b/research/selected-json-utf8-replay/UTF8-READS.patch new file mode 100644 index 0000000000..b9eef46c49 --- /dev/null +++ b/research/selected-json-utf8-replay/UTF8-READS.patch @@ -0,0 +1,28 @@ +--- a/scripts/verify_selected_result.py ++++ b/scripts/verify_selected_result.py +@@ -76,14 +76,14 @@ + def check(record, root=ROOT, *, historical=False): + root = root.resolve() + certificate_path = path(root, record['certificate']) +- certificate = json.loads(certificate_path.read_text()) ++ certificate = json.loads(certificate_path.read_text(encoding='utf-8')) + kappa = Fraction(record['kappa']) + require(kappa > 0 and kappa == Fraction(certificate['kappa']), 'Selected kappa differs from finite certificate') + require(kappa == Fraction(record['decimal']), 'Selected decimal differs from exact kappa') + require(sha256(certificate_path.read_bytes()).hexdigest() == record['certificate_sha256'], + 'Selected certificate bytes changed') + manifest_path = path(root, record['source_manifest']) +- manifest = json.loads(manifest_path.read_text()) ++ manifest = json.loads(manifest_path.read_text(encoding='utf-8')) + require(manifest.get('files'), 'Empty selected source inventory') + explicit = manifest.get('format') == 'repository-files-sha256-v1' + if explicit: +@@ -130,7 +130,7 @@ + parser.add_argument('--record', type=Path, default=ROOT / 'certificates/selected-result.json', + help='Selected or historical checkpoint record to verify') + args = parser.parse_args() +- record = json.loads(args.record.read_text()) ++ record = json.loads(args.record.read_text(encoding='utf-8')) + kappa = check(record, historical=args.record.resolve() != (ROOT / 'certificates/selected-result.json').resolve()) + print('PASS selected record, exact kappa, original source inventory and proof supplement: ' + str(kappa)) + diff --git a/research/selected-json-utf8-replay/VERIFY.json b/research/selected-json-utf8-replay/VERIFY.json new file mode 100644 index 0000000000..ca5441a08f --- /dev/null +++ b/research/selected-json-utf8-replay/VERIFY.json @@ -0,0 +1,31 @@ +{ + "schema_version": 1, + "status": "ready", + "python": "3.12", + "requirements": [], + "inputs": [ + "scripts/verify_selected_result.py" + ], + "commands": [ + { + "argv": [ + "{python}", + "-B", + "{package}/verify.py" + ], + "timeout_seconds": 90, + "checks": "Source-pinned selected JSON locale failure, unchanged-source UTF-8 workaround, temporary explicit-read repair, raw-byte corruption controls and native cp1252 corroboration where available." + }, + { + "argv": [ + "{python}", + "-I", + "-O", + "-B", + "{package}/verify.py" + ], + "timeout_seconds": 90, + "checks": "The same audit with an optimized driver; source-checker child processes deliberately retain assertions and explicitly selected UTF-8 modes." + } + ] +} diff --git a/research/selected-json-utf8-replay/verify.py b/research/selected-json-utf8-replay/verify.py new file mode 100644 index 0000000000..cbeeadee07 --- /dev/null +++ b/research/selected-json-utf8-replay/verify.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +"""Source-preserving CLI replay of the selected-checker JSON locale failure. + +Prepared for josecarlosgonzalezroque-design with substantial OpenAI Codex +assistance. Apache-2.0. Source-binding logic remains Douglas Colkitt's. +""" +from hashlib import sha256 +import difflib +import json +import locale +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[1] +SOURCE = ROOT / 'scripts/verify_selected_result.py' +SOURCE_SHA256 = 'd6b305eac5b9bdb8172c469ad30a1655a60fa0aceac8b1bf610e1b73f51babd8' +FORCE_CP1252 = """ +import io +import runpy +import sys +original = io.text_encoding +def text_encoding(encoding, stacklevel=2): + if encoding is None: + return 'utf-8' if sys.flags.utf8_mode else 'cp1252' + if encoding == 'locale': + return 'cp1252' + return original(encoding, stacklevel) +io.text_encoding = text_encoding +sys.argv = sys.argv[1:] +runpy.run_path(sys.argv[0], run_name='__main__') +""" + + +def source_bytes(): + raw = SOURCE.read_bytes() + if sha256(raw).hexdigest() != SOURCE_SHA256: + raise ValueError('Pinned selected checker bytes changed') + return raw + + +def proposed_repair(raw): + """Apply the three-line candidate only to a private temporary source copy.""" + for name in (b'certificate_path', b'manifest_path', b'args.record'): + old = name + b'.read_text()' + if raw.count(old) != 1: + raise ValueError('Candidate patch no longer matches its pinned source') + raw = raw.replace(old, name + b".read_text(encoding='utf-8')") + return raw + + +class SelectedJsonLocaleReplay(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + for name in ('scripts', 'certificates', 'supplier'): + (self.root / name).mkdir() + self.script = self.root / 'scripts/verify_selected_result.py' + self.original = source_bytes() + + def fixture(self, unicode_at=None, malformed_at=None, unicode_path=False): + proof = 'supplier/preuve-\u00e9.txt' if unicode_path else 'supplier/proof.txt' + (self.root / proof).write_bytes(b'immutable proof\n') + digest = sha256((self.root / proof).read_bytes()).hexdigest() + claim = {'kappa': '1/1000'} + manifest = {'format': 'repository-files-sha256-v1', 'files': {proof: digest}} + record = { + 'kappa': '1/1000', 'decimal': '0.001', + 'certificate': 'certificate.json', 'source_manifest': 'manifest.json', + 'proof': proof, 'review': proof, 'validation': 'certificate.json', + 'proof_supplements': {proof: digest}, + } + values = {'certificate': claim, 'manifest': manifest, 'record': record} + paths = {'certificate': 'certificate.json', 'manifest': 'manifest.json', + 'record': 'certificates/selected-result.json'} + if unicode_at: + # U+211D ends in UTF-8 byte 0x9d, which is undefined in cp1252. + values[unicode_at]['description'] = 'Coefficient field \u211d' + if malformed_at: + values[malformed_at]['description'] = 'INVALID_UTF8' + for key in ('certificate', 'manifest', 'record'): + if key == 'record': + record['certificate_sha256'] = sha256( + (self.root / 'certificate.json').read_bytes()).hexdigest() + raw = (json.dumps(values[key], ensure_ascii=False) + '\n').encode('utf-8') + if key == malformed_at: + raw = raw.replace(b'INVALID_UTF8', b'\xff') + (self.root / paths[key]).write_bytes(raw) + return proof + + def replay(self, mode, native=False): + raw = proposed_repair(self.original) if mode == 'patch' else self.original + self.script.write_bytes(raw) + flag = 'utf8' if mode == 'utf8' else 'utf8=0' + argv = [sys.executable, '-I', '-X', flag, '-B'] + if not native: + argv += ['-c', FORCE_CP1252] + argv += [str(self.script)] + result = subprocess.run(argv, cwd=self.root, capture_output=True, timeout=20) + self.assertEqual(self.script.read_bytes(), raw) + return result + + def assert_pass(self, result): + self.assertEqual(result.returncode, 0, result.stderr.decode('utf-8', errors='replace')) + self.assertIn(b'PASS selected record, exact kappa', result.stdout) + self.assertIn(b'1/1000', result.stdout) + + def assert_rejected(self, result, diagnostic): + self.assertNotEqual(result.returncode, 0) + self.assertIn(diagnostic, result.stderr) + self.assertNotIn(b'PASS selected record', result.stdout) + + def test_ascii_control_all_modes(self): + self.fixture() + for mode in ('legacy', 'utf8', 'patch'): + with self.subTest(mode=mode): + self.assert_pass(self.replay(mode)) + + def test_original_fails_at_each_utf8_json_boundary(self): + for key in ('record', 'certificate', 'manifest'): + with self.subTest(boundary=key): + self.fixture(unicode_at=key) + self.assert_rejected(self.replay('legacy'), b'UnicodeDecodeError') + + def test_utf8_mode_and_temporary_patch_repair_each_boundary(self): + for mode in ('utf8', 'patch'): + for key in ('record', 'certificate', 'manifest'): + with self.subTest(mode=mode, boundary=key): + self.fixture(unicode_at=key) + self.assert_pass(self.replay(mode)) + + def test_unicode_inventory_path_keeps_exact_spelling(self): + self.fixture(unicode_path=True) + self.assert_rejected(self.replay('legacy'), b'Invalid selected file') + for mode in ('utf8', 'patch'): + with self.subTest(mode=mode): + self.assert_pass(self.replay(mode)) + + def test_invalid_utf8_is_not_accepted_by_repairs(self): + for key in ('record', 'certificate', 'manifest'): + self.fixture(malformed_at=key) + # The legacy codec accepts 0xff as U+00ff, even with a valid byte hash. + with self.subTest(mode='legacy', boundary=key): + self.assert_pass(self.replay('legacy')) + for mode in ('utf8', 'patch'): + with self.subTest(mode=mode, boundary=key): + self.assert_rejected(self.replay(mode), b'UnicodeDecodeError') + + def test_source_corruption_still_rejected(self): + proof = self.fixture(unicode_path=True) + (self.root / proof).write_bytes(b'changed proof\n') + for mode in ('utf8', 'patch'): + with self.subTest(mode=mode): + self.assert_rejected(self.replay(mode), b'Selected package inventory or bytes changed') + + def test_certificate_hash_still_checks_raw_bytes(self): + self.fixture(unicode_at='certificate') + with (self.root / 'certificate.json').open('ab') as stream: + stream.write(b' ') + for mode in ('utf8', 'patch'): + with self.subTest(mode=mode): + self.assert_rejected(self.replay(mode), b'Selected certificate bytes changed') + + @unittest.skipUnless(locale.getencoding().lower() == 'cp1252', + 'native cp1252 host required; portable codec cases still run') + def test_native_cp1252_cli_without_encoding_mock(self): + for key in ('record', 'certificate', 'manifest'): + self.fixture(unicode_at=key, unicode_path=True) + with self.subTest(mode='legacy', boundary=key): + self.assert_rejected(self.replay('legacy', native=True), b'UnicodeDecodeError') + for mode in ('utf8', 'patch'): + with self.subTest(mode=mode, boundary=key): + self.assert_pass(self.replay(mode, native=True)) + + +def main(): + raw = source_bytes() + expected_patch = ''.join(difflib.unified_diff( + raw.decode('utf-8').splitlines(keepends=True), + proposed_repair(raw).decode('utf-8').splitlines(keepends=True), + fromfile='a/scripts/verify_selected_result.py', + tofile='b/scripts/verify_selected_result.py')).encode('utf-8') + if (HERE / 'UTF8-READS.patch').read_bytes() != expected_patch: + raise ValueError('Review patch differs from the tested temporary repair') + result = unittest.TextTestRunner(verbosity=2).run( + unittest.defaultTestLoader.loadTestsFromTestCase(SelectedJsonLocaleReplay)) + if not result.wasSuccessful(): + return 1 + print('PASS_SELECTED_JSON_UTF8_REPLAY tests=%d native_cp1252=%s no_new_kappa' + % (result.testsRun, locale.getencoding().lower() == 'cp1252')) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main())