diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml new file mode 100644 index 0000000..5324728 --- /dev/null +++ b/.github/workflows/linux.yml @@ -0,0 +1,53 @@ +name: Linux CI +on: + push: + branches: [main] + pull_request: + workflow_dispatch: +permissions: + contents: read +concurrency: + group: linux-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +jobs: + qualification: + name: Linux / ${{ matrix.architecture }} / ${{ matrix.lane }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + lane: [linux, linux-safety] + architecture: [x86_64, arm64] + include: + - architecture: x86_64 + runner: ubuntu-24.04 + - architecture: arm64 + runner: ubuntu-24.04-arm + container: + # Official Swift 6.4.0 Ubuntu 24.04 multi-architecture OCI index. + image: swift@sha256:31d14d727f4451f25e29bac9c42563a098c39ac66415e8bad970f48038e38aca + env: + PYTHONDONTWRITEBYTECODE: '1' + steps: + - name: Install the CI harness interpreter + run: | + apt-get update + apt-get install --yes --no-install-recommends python3 + python3 --version + dpkg-query --show python3 python3-minimal + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + fetch-depth: 0 + - run: python3 -B Scripts/ci.py --lane '${{ matrix.lane }}' + - name: Preserve Linux receipts + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: linux-${{ matrix.architecture }}-${{ matrix.lane }} + if-no-files-found: error + retention-days: 30 + path: | + .build/ci/ + !.build/ci/**/reproduction/extracted/** diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1a41e88..caf5413 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,6 +19,10 @@ python3 Scripts/ci.py --lane safety python3 Scripts/ci.py --lane docs ``` +For Linux changes also run `python3 Scripts/ci.py --lane linux` and +`--lane linux-safety` on an actual Swift 6.4+ Ubuntu 24.04/glibc runtime. +Linux results supplement the Apple gates; see the [platform contract](Documentation/Architecture/Linux.md). + Keep APIs neutral and exact, budgets cumulative and mutable state invocation-owned. Do not add platforms, permissive parsing, online schema resolution, fallback engines or runtime dependencies without an explicit design review. Benchmark and fixture diff --git a/Documentation/Architecture/ExtractionManifest.json b/Documentation/Architecture/ExtractionManifest.json index fd8f054..0ac51bf 100644 --- a/Documentation/Architecture/ExtractionManifest.json +++ b/Documentation/Architecture/ExtractionManifest.json @@ -193,7 +193,8 @@ "original_implementation_notes": [ "JSONNumber.swift reference reviewed; new bounded decimal storage and conversions written without copying upstream numeric implementation.", "Native codec, Codable, Pointer, Patch and streaming are original implementations from the integrated contract, not automatically attributed to candidates.", - "Schema graph, explicit evaluator, exact sets/assertions and scalar format recognizers are original implementations from Draft 2020-12, RFC3986/3987/3339/3492/5321/5891/5892/5893/6531/6570/6901 and fixed Unicode13 data." + "Schema graph, explicit evaluator, exact sets/assertions and scalar format recognizers are original implementations from Draft 2020-12, RFC3986/3987/3339/3492/5321/5891/5892/5893/6531/6570/6901 and fixed Unicode13 data.", + "Private FIPS 180-4 SHA-256 is an original Swift implementation with checked cumulative production accounting, bounded scratch and inner cancellation. The separate trusted fixture tool mirrors its primitive core byte-for-byte; public_check verifies parity. No CryptoKit fallback or runtime dependency is introduced." ], "policy": "Record actual adaptations and retain applicable upstream copyrights and licenses. Reference inputs and comparison experiments are not distributed runtime dependencies." } diff --git a/Documentation/Architecture/Linux.md b/Documentation/Architecture/Linux.md new file mode 100644 index 0000000..c0f2913 --- /dev/null +++ b/Documentation/Architecture/Linux.md @@ -0,0 +1,70 @@ +# Linux platform contract + +This additive development branch prepares all four public products for 64-bit +Linux with Swift tools/compiler 6.4+ and Swift language mode 6. Initial execution +qualification targets Ubuntu 24.04 with glibc on ARM64 and x86_64. A platform +preflight identifies the actual runtime; cross-compilation alone cannot qualify +behavior or safety. Architecture-specific receipts must pass before claiming +that architecture qualified. Apple minimums remain macOS/SDK 27 and iOS/SDK 27. + +The root manifest keeps an empty dependency graph and the same four products. +Linux does not need a SwiftPM deployment-platform entry. Foundation file/resource +access remains platform infrastructure; JSON semantics come from the package's +strict codec, exact decimal model and decoded scalar identity. + +## Portable schema integrity + +Packaged Draft 2020-12 resources retain their nine exact byte sizes and SHA-256 +pins. An original package-private FIPS 180-4 SHA-256 implementation replaces +CryptoKit. It exposes no cryptography API or product and has no external dependency. +There is one implementation on Apple and Linux, with no alternate engine. + +Schema loading passes its active cumulative compilation meter into hashing. +Byte ingestion, schedule expansion, compression rounds and hex output charge +work before execution and check cancellation at the configured quantum. +336 logical scratch bytes are admitted before state/schedule allocation, then +128 bytes cover the simultaneous hex buffer and returned string. Scratch is +released after success, operational rejection or cancellation; callers account +for the 64-byte returned digest until releasing it. These weights describe logical +retention, not process RSS, allocator metadata or cryptographic throughput. +Standard UInt32 wrapping arithmetic implements SHA-256; resource counters remain +checked UInt64 arithmetic. Integrity mismatch is a compilation failure; exhausted +quotas and cancellation remain operational errors. + +Independent Python hashlib vectors cover 24 deterministic lengths around padding +and block boundaries. Standard empty/abc/56-byte/million-a answers, cumulative +N/N-1 work, scratch admission, input quota, quantum, cancellation, corruption and +concurrent invocation cases supplement the unchanged schema and Unicode corpora. +The trusted qualification executable mirrors the core byte for byte, enforced +by the public inventory check. Its fixture hashing is harness work; library +operations always use the real checked meter. + +## Qualification boundary + +Run `python3 Scripts/ci.py --lane linux` and `--lane linux-safety` on the Linux +runtime. These preserve exact test discovery, all 49 cap probes, both pinned +independent suites, authored replay and the 20 explicitly rooted public consumer +cases. SwiftData examples and Apple SDK/compiler ownership probes remain in the +Apple lanes; they do not become Linux evaluated successes. DocC qualification +continues on the supported Mac toolchain. + +The pinned Swift 6.4 Linux runtime leaks from an empty XCTest runner and +Foundation's `Data(contentsOf:)` empty-file read. Isolated sanitizer probes +reproduce both with no JSON imports. Linux safety starts only Swift Testing +(all 180 functions and 173 invocations remain) and trusted fixture I/O uses +explicitly closed FileHandle reads, including the zero-byte JSON rejection case. +Address/Leak Sanitizer and Thread Sanitizer remain enabled without suppressions. +Diagnostic upstream failures are recorded separately from package behavior. + +On Linux the public consumer resolves its copied Resources directory relative to +bundleURL; Foundation can already point resourceURL into that directory. Apple +uses resourceURL so it retains the platform's bundle Contents layout. Both layouts +retain the same frozen definition/vector bytes and all 20 explicitly rooted cases. +Linux qualification also verifies a byte-equal source archive, extracts it into a +clean directory, builds all products and executes six integrity/resource cases +and the four-product public consumer from that extracted source. + +WebAssembly, browser hosting, other distributions, musl deployment and 32-bit +profile calibration require separate qualification. This change adds none of +those claims. The concurrent schema-builder accounting change is a separate +integration dependency; this branch preserves its base implementation untouched. diff --git a/Documentation/CI/README.md b/Documentation/CI/README.md index 70808b2..7a201eb 100644 --- a/Documentation/CI/README.md +++ b/Documentation/CI/README.md @@ -43,3 +43,18 @@ streaming regression is documented in ../Performance/README.md. Run locally with `python3 Scripts/ci.py --lane native`, `--lane safety` or `--lane docs`. The first hosted result must be inspected before marking CI qualified. + +## Linux development lanes + +The additive Linux workflow selects the pinned official Swift 6.4.0 Ubuntu 24.04 +multi-architecture image on ARM64 and x86_64 hosted runners. It runs `linux` and +`linux-safety` with separate runtime preflight, source receipts and exact test, +consumer, corpus and 49-cap discovery checks. SwiftData/Apple compiler probes +remain in Native CI. The existing required `Native / native`, `Native / safety` +and `Native / docs` names and protection settings are preserved. This workflow +does not publish artifacts to a release or deploy documentation. + +Local ARM64 VM execution qualifies its observed Linux environment; an unexecuted +x86_64 job is pending evidence. Workflow preparation does not establish hosted +success. Read the [Linux contract](../Architecture/Linux.md) before extending +distribution, architecture or libc claims. diff --git a/Documentation/Qualification/LimitsMatrix.json b/Documentation/Qualification/LimitsMatrix.json index 17cbcf8..54c926b 100644 --- a/Documentation/Qualification/LimitsMatrix.json +++ b/Documentation/Qualification/LimitsMatrix.json @@ -59,6 +59,10 @@ "path": "Sources/JSONStreaming/NDJSON/NDJSONValueSequence.swift", "line": 41, "code": "try input.meter.require(.inputBytes, UInt64(line.count) + 1); try input.meter.retain(1); line.append(byte)" + }, + { + "path": "Sources/JSON/Integrity/JSONSHA256Core.swift", + "owner": "SHA-256 production JSONWorkMeter context; bounded input, scratch, padding/schedule/compression and output" } ], "public_probe_outcome": "limitExceeded", @@ -834,6 +838,10 @@ "path": "Sources/JSONSchema/Formats/JSONSchemaFormats.swift", "line": 22, "code": "try meter.release(meter.used(.retainedBytes)-before); return true" + }, + { + "path": "Sources/JSON/Integrity/JSONSHA256Core.swift", + "owner": "SHA-256 production JSONWorkMeter context; bounded input, scratch, padding/schedule/compression and output" } ], "public_probe_outcome": "limitExceeded", @@ -864,6 +872,10 @@ "path": "Sources/JSON/Limits/JSONWorkMeter.swift", "line": 114, "code": "counters[JSONResourceMetric.workUnits.index] = total" + }, + { + "path": "Sources/JSON/Integrity/JSONSHA256Core.swift", + "owner": "SHA-256 production JSONWorkMeter context; bounded input, scratch, padding/schedule/compression and output" } ], "public_probe_outcome": "limitExceeded", @@ -1254,6 +1266,10 @@ "path": "Sources/JSON/Limits/JSONLimits.swift", "line": 138, "code": "Validate quantum in 1...1024 before invocation." + }, + { + "path": "Sources/JSON/Integrity/JSONSHA256Core.swift", + "owner": "SHA-256 production JSONWorkMeter context; bounded input, scratch, padding/schedule/compression and output" } ], "public_probe_outcome": "invalidLimits", @@ -1394,6 +1410,10 @@ "path": "Sources/JSON/Limits/JSONWorkMeter.swift", "line": 102, "code": "case .schemaCompilation: domainMetric = .compileWorkUnits" + }, + { + "path": "Sources/JSON/Integrity/JSONSHA256Core.swift", + "owner": "SHA-256 production JSONWorkMeter context; bounded input, scratch, padding/schedule/compression and output" } ], "public_probe_outcome": "limitExceeded", @@ -1429,6 +1449,10 @@ "path": "Sources/JSON/Limits/JSONWorkMeter.swift", "line": 145, "code": "counters[JSONResourceMetric.compiledPlanBytes.index] -= bytes" + }, + { + "path": "Sources/JSON/Integrity/JSONSHA256Core.swift", + "owner": "SHA-256 production JSONWorkMeter context; bounded input, scratch, padding/schedule/compression and output" } ], "public_probe_outcome": "limitExceeded", diff --git a/Documentation/Qualification/README.md b/Documentation/Qualification/README.md index de9f3a1..13618a0 100644 --- a/Documentation/Qualification/README.md +++ b/Documentation/Qualification/README.md @@ -5,6 +5,13 @@ Each receipt identifies the source content, command exits and environment. CI artifacts contain raw logs, consumer outcomes and per-case corpus results. Safety and DocC lanes run separately. Unsupported hosts fail preflight. +The Linux development increment adds four private integrity tests and one schema +integrity test: exactly 180 root functions (JSON 101, JSONSchema 42, regex 12, +streaming 14, Patch 11) with the 173 parameterized invocations retained. Targeted +shared-state safety adds the sixth function for concurrent hashing. Historical +counts below remain the receipts for their original stages. The +[Linux contract](../Architecture/Linux.md) defines the separate runtime lanes. + The 1.0 baseline has 122 Swift Testing functions across five bundles and 173 parameterized invocations. The first 1.1 qualification milestone adds seven authored prototype cases in JSONTests (54 to 61), for 129 functions, with the existing 173 parameterized invocations retained @@ -38,6 +45,13 @@ exact keys/numbers/order, separate compilation and callback replacement of the invocation ledger. The four-product external consumer also checks the 150-unit composed-budget regression. +Integrating Linux support with that fix retains all nine builder cases and all +five integrity cases: exactly 189 root functions (JSON 101, JSONSchema 51, +JSONRegexRuntime 12, JSONStreaming 14, JSONPatch 11), 173 parameterized invocations +and six shared-state safety functions. Both Linux architectures and the three +Apple lanes qualify this combined source; historical counts above remain tied to +their original snapshots. + | Independent input | Observed policy | | --- | --- | | JSONTestSuite, pinned in FixtureRegistry.json | 318 discovered: 101 accepted, 186 rejected, 31 strict/finite-policy rejections | diff --git a/Documentation/README.md b/Documentation/README.md index 917c5d5..b1c1010 100644 --- a/Documentation/README.md +++ b/Documentation/README.md @@ -29,6 +29,7 @@ Compiled archives and public symbol graphs remain available in CI artifacts. - [Strict streaming and certified completion](Architecture/Streaming.md) - [Pointer, JSON Patch and Merge Patch](Architecture/Patch.md) - [Concurrency, cancellation and caller-work boundaries](Architecture/Concurrency.md) +- [Linux platform and portable schema integrity](Architecture/Linux.md) ## Consumers, qualification and maintenance diff --git a/README.md b/README.md index b6535ea..12baf94 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,11 @@ Choose the products your application needs: SwiftPM dependency graph is empty. Qualification and benchmark tools are separate packages. +The Linux development branch adds portable schema integrity for all four products +without runtime dependencies. Its initial qualification scope is Swift 6.4+, +64-bit Ubuntu 24.04/glibc; see the [Linux contract](Documentation/Architecture/Linux.md) +for execution gates and limits. Apple platform requirements remain unchanged. + ## Installation Add the package to your `Package.swift`: diff --git a/Scripts/README.md b/Scripts/README.md index fee5edc..e54a11e 100644 --- a/Scripts/README.md +++ b/Scripts/README.md @@ -9,6 +9,9 @@ separate from the dependency-free library. Native lanes fail on unsupported host | `python3 Scripts/preflight.py` | Actual native OS/compiler/SDK receipt | | `python3 Scripts/ci.py --lane native` | Manifests, builds, Swift Testing, consumers and suites | | `python3 Scripts/ci.py --lane safety` | Address/Thread Sanitizer and strict memory safety | +| `python3 Scripts/ci.py --lane linux` | Actual Linux preflight, all four products, exact root/cap/corpus/consumer counts | +| `python3 Scripts/ci.py --lane linux-safety` | Linux Address/Thread Sanitizer and strict memory safety | +| `python3 Scripts/qualify_linux_reproduction.py --output PATH` | Actual source archive, byte-equal rebuild, clean extraction, build/tests and public consumer | | `python3 Scripts/ci.py --lane docs` | Four DocC catalogs, authored API/parameter coverage, executed README examples and verified static Pages site | | `python3 Scripts/build_documentation_site.py` | Transform already-qualified DocC archives and verify static routes/assets | | `python3 Scripts/qualification_watchdog.py --timeout 900 --output PATH -- COMMAND` | Process-group timeout and native receipt | diff --git a/Scripts/ci.py b/Scripts/ci.py index 2a260c0..4bb00de 100644 --- a/Scripts/ci.py +++ b/Scripts/ci.py @@ -3,10 +3,10 @@ """Execute public native CI lanes with command, source and environment receipts.""" import argparse, datetime, hashlib, json, os, pathlib, re, subprocess, sys from archive_candidate import files -from preflight import collect +from preflight import collect, collect_linux ROOT = pathlib.Path(__file__).resolve().parents[1] -SHARED_TESTS = 'independentCodecFacadesAndExactValuesCanBeSharedAcrossTasks|concurrentSchemaEvaluationsHaveExclusiveDynamicRegexAndAnnotationState|sharedRegexProgramHasExclusiveConcurrentSearchState|immutableIndexedObjectCanBeSharedAcrossTasks|qaSharedPayloadAndRootHaveIndependentConcurrentReadScopes' +SHARED_TESTS = 'independentCodecFacadesAndExactValuesCanBeSharedAcrossTasks|concurrentSchemaEvaluationsHaveExclusiveDynamicRegexAndAnnotationState|sharedRegexProgramHasExclusiveConcurrentSearchState|immutableIndexedObjectCanBeSharedAcrossTasks|qaSharedPayloadAndRootHaveIndependentConcurrentReadScopes|sha256InputCapacityAndConcurrentInvocationsRemainIndependent' R0_TESTS = 'readerProbe|indexProbes|duplicateRecognitionPrecedesValueAndClosingSyntax' def snapshot(): @@ -15,6 +15,18 @@ def snapshot(): def commands(lane, output): # The qualification executable resolves output paths from its repository root. output = output.relative_to(ROOT) + if lane == 'linux': + # Root behavior/corpora remain required. SwiftData and Apple SDK/lifetime + # probes are separate Apple acceptance; they never become Linux passes. + return [c for c in commands('native', ROOT / output) + if not any(x in c for x in ['Tools/MobileQualification', '--mode'])] + [ + ['python3', '-B', 'Scripts/qualify_linux_reproduction.py', '--output', str(output / 'reproduction')]] + if lane == 'linux-safety': + # Every behavior case uses Swift Testing. Do not start the generated + # zero-case XCTest runners, which leak in the pinned Linux runtime. + return [['python3', '-B', 'Scripts/linux_runtime_diagnostics.py', '--output', str(output / 'runtime-diagnostics')]] + [ + c + ['--disable-xctest'] if c[:2] == ['swift', 'test'] else c + for c in commands('safety', ROOT / output)] if lane == 'qa': return [ ['swift', 'test', '--filter', 'qa'], @@ -77,8 +89,10 @@ def check_result(command, code, log): counts = [int(n) for n in re.findall(r'Test run with (\d+) tests? in[^\n]*passed', log)] passed = code == 0 if command[:2] == ['swift', 'test']: - expected = [2, 3, 3, 11] if 'qa' in command else [7] if R0_TESTS in command else [3] if 'Tools/MCPReadiness' in command or 'Tools/MobileQualification' in command else [1] if 'Tools/Qualification' in command else [1, 1, 3] if 'thread' in command else [11, 12, 14, 50, 97] + expected = [2, 3, 3, 11] if 'qa' in command else [7] if R0_TESTS in command else [3] if 'Tools/MCPReadiness' in command or 'Tools/MobileQualification' in command else [1] if 'Tools/Qualification' in command else [1, 1, 4] if 'thread' in command else [11, 12, 14, 51, 101] passed = passed and sorted(counts) == expected + if '--filter' not in command and not any(x.startswith('Tools/') for x in command): + passed = passed and sum(int(n) for n in re.findall(r'with (\d+) test cases passed', log)) == 173 if 'Tools/Qualification' in command: probes = re.findall(r'^CAP_PROBE (\w+) (\S+)$', log, re.MULTILINE) matrix = json.loads((ROOT / 'Documentation/Qualification/LimitsMatrix.json').read_text()) @@ -95,14 +109,15 @@ def check_result(command, code, log): def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--lane', choices=['native', 'safety', 'docs', 'r0', 'qa'], required=True) + parser.add_argument('--lane', choices=['native', 'safety', 'docs', 'r0', 'qa', 'linux', 'linux-safety'], required=True) parser.add_argument('--output') args = parser.parse_args() output = ROOT / (args.output or '.build/ci/' + args.lane) output.mkdir(parents=True, exist_ok=True) inputs = snapshot() commit = subprocess.run(['git', 'rev-parse', 'HEAD'], cwd=ROOT, capture_output=True, text=True) - receipt = {'schema_version': 1, 'lane': args.lane, 'status': 'blocked', 'environment': collect(), 'started_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'git_commit': commit.stdout.strip() if commit.returncode == 0 else None, 'github_run_id': os.environ.get('GITHUB_RUN_ID'), 'source_inputs': inputs, 'source_manifest_sha256': hashlib.sha256(json.dumps(inputs, sort_keys=True).encode()).hexdigest(), 'commands': []} + environment = collect_linux() if args.lane.startswith('linux') else collect() + receipt = {'schema_version': 1, 'lane': args.lane, 'status': 'blocked', 'environment': environment, 'started_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'git_commit': commit.stdout.strip() if commit.returncode == 0 else None, 'github_run_id': os.environ.get('GITHUB_RUN_ID'), 'source_inputs': inputs, 'source_manifest_sha256': hashlib.sha256(json.dumps(inputs, sort_keys=True).encode()).hexdigest(), 'commands': []} def save(): (output / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') save() @@ -135,7 +150,7 @@ def save(): save() else: receipt['status'] = 'passed' - if receipt['status'] == 'passed' and args.lane == 'native': + if receipt['status'] == 'passed' and args.lane in ('native', 'linux'): try: receipt['materialized_outcomes'] = materialized_outcomes(output) except (OSError, KeyError, ValueError, TypeError) as error: diff --git a/Scripts/linux_runtime_diagnostics.py b/Scripts/linux_runtime_diagnostics.py new file mode 100644 index 0000000..3c1bfb3 --- /dev/null +++ b/Scripts/linux_runtime_diagnostics.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Isolate Linux runtime leaks; never suppress leak or address checks in package tests.""" +import argparse, hashlib, json, pathlib, subprocess, sys + +ROOT = pathlib.Path(__file__).resolve().parents[1] +READER = '''import Foundation +let url = URL(fileURLWithPath: CommandLine.arguments[2]) +let data: Data +if CommandLine.arguments[1] == "data" { + data = try Data(contentsOf: url) +} else { + let handle = try FileHandle(forReadingFrom: url) + data = try handle.readToEnd() ?? Data() + try handle.close() +} +precondition(data.isEmpty) +print("EMPTY_FILE_READ bytes=0") +''' + +def main(): + p = argparse.ArgumentParser(); p.add_argument('--output', required=True); a = p.parse_args() + out = ROOT / a.output; out.mkdir(parents=True, exist_ok=True) + (out / 'empty.json').write_bytes(b'') + report = {'status': 'running', 'probes': [], 'sanitizer_suppressions': False} + def run(name, command): + log = out / (name + '.log') + result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True, timeout=120) + text = (result.stdout + result.stderr).replace(str(ROOT), '.') + log.write_text(text) + report['probes'].append({'name': name, 'command': [x.replace(str(ROOT), '.') for x in command], + 'exit_code': result.returncode, 'log': log.relative_to(ROOT).as_posix(), + 'log_sha256': hashlib.sha256(log.read_bytes()).hexdigest()}) + return result.returncode, text + failed = False + for name, source in [('reader', READER), ('xctest', 'import XCTest\nXCTMain([])\n')]: + path = out / (name + '.swift'); path.write_text(source) + code, _ = run('compile-' + name, ['swiftc', '-sanitize=address', '-g', str(path), '-o', str(out / name)]) + failed |= code != 0 + if code: continue + modes = ['data', 'handle'] if name == 'reader' else ['empty-xctest'] + for mode in modes: + command = [str(out / name)] + ([mode, str(out / 'empty.json')] if name == 'reader' else []) + code, text = run(mode, command) + if mode == 'handle': + expected = code == 0 and 'EMPTY_FILE_READ bytes=0' in text + report['probes'][-1]['disposition'] = 'clean' if expected else 'failed' + else: + leaked = code != 0 and 'LeakSanitizer: detected memory leaks' in text + clean = code == 0 and 'LeakSanitizer' not in text + expected = leaked or clean + report['probes'][-1]['disposition'] = 'upstream_leak_reproduced' if leaked else 'clean' if clean else 'failed' + failed |= not expected + report['status'] = 'failed' if failed else 'passed' + (out / 'receipt.json').write_text(json.dumps(report, indent=2) + '\n') + print(json.dumps(report, indent=2)) + return 1 if failed else 0 +if __name__ == '__main__': sys.exit(main()) diff --git a/Scripts/preflight.py b/Scripts/preflight.py index c30c67e..751a1d2 100755 --- a/Scripts/preflight.py +++ b/Scripts/preflight.py @@ -49,6 +49,36 @@ def collect(): # A preflight pass identifies tools; actual manifest/build/test gates still require execution. return result +def collect_linux(): + """Separate Linux receipts; Apple native qualification is never substituted.""" + result = {'schema_version': 1, 'time_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(), + 'os': platform.system(), 'architecture': platform.machine(), 'status': 'blocked', + 'commands': [], 'errors': [], 'swift_language_mode_required': '6', 'manifest_compiled': False} + if platform.system() != 'Linux': + result['errors'] = ['Actual Linux execution required; a cross-build is not runtime qualification'] + return result + result['commands'] = [run(['swift', '--version']), run(['uname', '-a']), run(['ldd', '--version'])] + try: + result['distribution'] = dict(line.split('=', 1) for line in pathlib.Path('/etc/os-release').read_text().splitlines() if '=' in line) + except OSError as error: + result['distribution'] = {} + result['errors'].append(str(error)) + result['libc'] = platform.libc_ver() + result['python'] = sys.version + try: + compiler_ok = result['commands'][0]['exit_code'] == 0 and version(result['commands'][0].get('stdout', '')) >= (6,4,0) + except ValueError: + compiler_ok = False + if not compiler_ok: + result['errors'].append('Swift 6.4+ required') + if result['distribution'].get('ID', '').strip('"') != 'ubuntu' or result['distribution'].get('VERSION_ID', '').strip('"') != '24.04': + result['errors'].append('Initial native Linux qualification requires Ubuntu 24.04') + if result['architecture'] not in ('aarch64', 'x86_64') or result['libc'][0] != 'glibc': + result['errors'].append('Initial native Linux qualification requires 64-bit ARM64/x86_64 glibc') + result['swift'] = result['commands'][0].get('stdout', '') + result['status'] = 'blocked' if result['errors'] else 'passed' + return result + def main(): parser=argparse.ArgumentParser(description=__doc__) parser.add_argument('--output',default=str(ROOT/'.build/handoff/preflight.json')) diff --git a/Scripts/public_check.py b/Scripts/public_check.py index f7b2f53..7a6f2fe 100644 --- a/Scripts/public_check.py +++ b/Scripts/public_check.py @@ -9,6 +9,10 @@ def main(): errors = [] + digest_source = ROOT / 'Sources/JSON/Integrity/JSONSHA256Core.swift' + digest_mirror = ROOT / 'Tools/Qualification/Sources/Qualification/JSONSHA256Core.swift' + if digest_source.read_bytes() != digest_mirror.read_bytes(): + errors.append('Qualification digest mirror differs from the private source') selected = files() names = {p.relative_to(ROOT).as_posix() for p in selected} git = subprocess.run(['git', 'rev-parse', '--show-toplevel'], cwd=ROOT, capture_output=True, text=True) diff --git a/Scripts/qualification_watchdog.py b/Scripts/qualification_watchdog.py index b508fb8..b6fa854 100644 --- a/Scripts/qualification_watchdog.py +++ b/Scripts/qualification_watchdog.py @@ -1,14 +1,14 @@ #!/usr/bin/env python3 """Bounded external watchdog for a public-engine qualification command; no in-process timeout race.""" -import argparse,datetime,hashlib,json,pathlib,subprocess,sys,time -from preflight import collect +import argparse,datetime,hashlib,json,pathlib,platform,subprocess,sys,time +from preflight import collect,collect_linux ROOT=pathlib.Path(__file__).resolve().parents[1] def main(): p=argparse.ArgumentParser();p.add_argument('--timeout',type=int,default=900);p.add_argument('--output',default='.build/handoff/watchdog.json');p.add_argument('command',nargs=argparse.REMAINDER);a=p.parse_args() command=a.command[1:] if a.command[:1]==['--'] else a.command if not command or not 1<=a.timeout<=1800:p.error('Provide a command and 1...1800 second timeout') destination=ROOT/a.output;destination.parent.mkdir(parents=True,exist_ok=True);log=destination.with_suffix('.log') - receipt={'schema_version':1,'environment':collect(),'command':command,'timeout_seconds':a.timeout,'time_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'status':'failed'} + receipt={'schema_version':1,'environment':collect_linux() if platform.system()=='Linux' else collect(),'command':command,'timeout_seconds':a.timeout,'time_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'status':'failed'} started=time.monotonic() with log.open('w') as stream: try: diff --git a/Scripts/qualify_linux_reproduction.py b/Scripts/qualify_linux_reproduction.py new file mode 100644 index 0000000..f6e139a --- /dev/null +++ b/Scripts/qualify_linux_reproduction.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Build and exercise a manifest-verified, independently extracted Linux candidate.""" +import argparse, hashlib, json, pathlib, re, subprocess, tarfile +from archive_candidate import archive, files, verify + +ROOT = pathlib.Path(__file__).resolve().parents[1] + +def main(): + p = argparse.ArgumentParser(); p.add_argument('--output', required=True); a = p.parse_args() + out = ROOT / a.output; out.mkdir(parents=True, exist_ok=True) + first, second = out / 'candidate.tar.gz', out / 'candidate-rebuild.tar.gz' + selected = files(); archive(first, selected); archive(second, selected) + assert first.read_bytes() == second.read_bytes(), 'Candidate rebuild differs' + second.unlink(); entries = verify(first) + extracted = out / 'extracted'; extracted.mkdir(exist_ok=False) + with tarfile.open(first) as tar: + # verify() checked regular entries, exact prefix, safe paths and manifest. + tar.extractall(extracted, filter='data') + source = extracted / 'Swift-JSON' + commands = [ + ['swift', 'build', '-c', 'release'], + ['swift', 'test', '-c', 'release', '--disable-xctest', '--filter', 'sha256|builtinIntegrity|pinnedOfficialMetaSchemasLoadAsImmutableResources'], + ['swift', 'run', '-c', 'release', '--package-path', 'Examples/Consumer', 'Consumer']] + receipt = {'status': 'running', 'archive_sha256': hashlib.sha256(first.read_bytes()).hexdigest(), + 'archive_bytes': first.stat().st_size, 'archive_entries': entries, 'byte_equal_rebuild': True, + 'git_directory_present': (source / '.git').exists(), 'commands': []} + for i, command in enumerate(commands): + log = out / ('command-%02d.log' % i) + with log.open('w') as stream: + code = subprocess.run(command, cwd=source, stdout=stream, stderr=subprocess.STDOUT, timeout=900).returncode + text = log.read_text().replace(str(ROOT), '.') + log.write_text(text) + counts = [int(n) for n in re.findall(r'Test run with (\d+) tests? in[^\n]*passed', text)] + passed = code == 0 + if command[:2] == ['swift', 'test']: passed &= sorted(counts) == [2, 4] + if 'Consumer' in command: passed &= 'Consumer passed:' in text + receipt['commands'].append({'command': command, 'exit_code': code, 'passed': passed, + 'swift_testing_function_counts': counts, 'log': log.relative_to(ROOT).as_posix(), + 'log_sha256': hashlib.sha256(log.read_bytes()).hexdigest()}) + if not passed: receipt['status'] = 'failed'; break + else: receipt['status'] = 'passed' + # Keep the exact candidate and proof, avoiding duplicated extracted build caches. + (out / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') + print(json.dumps(receipt, indent=2)) + return 0 if receipt['status'] == 'passed' else 1 +if __name__ == '__main__': raise SystemExit(main()) diff --git a/Sources/JSON/Integrity/JSONHashMeter.swift b/Sources/JSON/Integrity/JSONHashMeter.swift new file mode 100644 index 0000000..4b1cf63 --- /dev/null +++ b/Sources/JSON/Integrity/JSONHashMeter.swift @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: Apache-2.0 +extension JSONWorkMeter: JSONHashAccounting { + package mutating func hashWork(_ amount: UInt64) throws { + var remaining = amount + while remaining != 0 { + let next = min(remaining, limits[.cancellationQuantum]) + // Work can arrive after earlier parser/loader charges. Check before + // each block rather than assuming the meter's prior quantum is empty. + try checkpoint(); try work(next); remaining -= next + } + } + package mutating func hashRetain(_ bytes: UInt64) throws { try retain(bytes) } + package mutating func hashRelease(_ bytes: UInt64) throws { try release(bytes) } + package mutating func hashInput(_ bytes: UInt64) throws { try require(.inputBytes, bytes) } +} diff --git a/Sources/JSON/Integrity/JSONSHA256Core.swift b/Sources/JSON/Integrity/JSONSHA256Core.swift new file mode 100644 index 0000000..a25242a --- /dev/null +++ b/Sources/JSON/Integrity/JSONSHA256Core.swift @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 +// Original implementation of FIPS 180-4 SHA-256. No cryptography API is exported. +// This file is mirrored byte-for-byte by the trusted qualification executable; +// Scripts/public_check.py checks that mirror. Production uses JSONWorkMeter. +package protocol JSONHashAccounting: ~Copyable { + mutating func hashWork(_ amount: UInt64) throws + mutating func hashRetain(_ bytes: UInt64) throws + mutating func hashRelease(_ bytes: UInt64) throws + mutating func hashInput(_ bytes: UInt64) throws +} +package enum JSONHashFailure: Error { case lengthOverflow } + +package enum JSONSHA256Core { + // Fixed immutable standard constants, shared like the pinned Unicode tables. + private static let constants: [UInt32] = [ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 + ] + private static func rotate(_ value: UInt32, by count: Int) -> UInt32 { + (value >> count) | (value << (32 - count)) + } + private static func compress( + _ words: inout [UInt32], state: inout [UInt32], accounting: inout B + ) throws { + for i in 16..<64 { + // Conservative primitive blocks include shifts, accesses and arithmetic. + // Each block is <=64 units; the production meter checks cancellation. + try accounting.hashWork(64) + let x = words[i - 15], y = words[i - 2] + let s0 = rotate(x, by: 7) ^ rotate(x, by: 18) ^ (x >> 3) + let s1 = rotate(y, by: 17) ^ rotate(y, by: 19) ^ (y >> 10) + // Wrapping UInt32 addition is required by SHA-256, never resource math. + words[i] = words[i - 16] &+ s0 &+ words[i - 7] &+ s1 + } + try accounting.hashWork(8) + var a = state[0], b = state[1], c = state[2], d = state[3] + var e = state[4], f = state[5], g = state[6], h = state[7] + for i in 0..<64 { + try accounting.hashWork(64) + let s1 = rotate(e, by: 6) ^ rotate(e, by: 11) ^ rotate(e, by: 25) + let choice = (e & f) ^ (~e & g) + let t1 = h &+ s1 &+ choice &+ constants[i] &+ words[i] + let s0 = rotate(a, by: 2) ^ rotate(a, by: 13) ^ rotate(a, by: 22) + let majority = (a & b) ^ (a & c) ^ (b & c) + let t2 = s0 &+ majority + h = g; g = f; f = e; e = d &+ t1 + d = c; c = b; b = a; a = t1 &+ t2 + } + try accounting.hashWork(32) + state[0] = state[0] &+ a; state[1] = state[1] &+ b + state[2] = state[2] &+ c; state[3] = state[3] &+ d + state[4] = state[4] &+ e; state[5] = state[5] &+ f + state[6] = state[6] &+ g; state[7] = state[7] &+ h + } + /// Returns 64 lowercase ASCII hex bytes, retaining that output in the caller's + /// ledger. Scratch is released on success, rejection or cancellation. + package static func hexadecimal( + _ bytes: C, accounting: inout B + ) throws -> String where C.Element == UInt8 { + // 8 state words + 64 schedule words + 8 compression words + a fixed frame. + let scratch: UInt64 = 336 + try accounting.hashRetain(scratch) + defer { try? accounting.hashRelease(scratch) } + try accounting.hashWork(72) + var state: [UInt32] = [0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19] + var words = Array(repeating: UInt32(0), count: 64) + var used = 0, length: UInt64 = 0 + for byte in bytes { + guard length < UInt64.max / 8 else { throw JSONHashFailure.lengthOverflow } + length += 1 + try accounting.hashInput(length); try accounting.hashWork(8) + words[used / 4] |= UInt32(byte) << (24 - (used % 4) * 8) + used += 1 + if used == 64 { + try compress(&words, state: &state, accounting: &accounting) + for i in 0..<16 { try accounting.hashWork(2); words[i] = 0 } + used = 0 + } + } + try accounting.hashWork(8) + words[used / 4] |= UInt32(0x80) << (24 - (used % 4) * 8) + if used >= 56 { + try compress(&words, state: &state, accounting: &accounting) + for i in 0..<16 { try accounting.hashWork(2); words[i] = 0 } + } + try accounting.hashWork(8) + let bits = length * 8 // Length was checked before increment, no overflow. + words[14] = UInt32(bits >> 32); words[15] = UInt32(truncatingIfNeeded: bits) + try compress(&words, state: &state, accounting: &accounting) + try accounting.hashRetain(128) // Hex buffer and result are simultaneously live. + var releaseOutput = true + defer { if releaseOutput { try? accounting.hashRelease(128) } } + var output: [UInt8] = []; output.reserveCapacity(64) + for word in state { + for shift in stride(from: 28, through: 0, by: -4) { + try accounting.hashWork(4) + let nibble = UInt8((word >> shift) & 15) + output.append(nibble < 10 ? nibble + 48 : nibble + 87) + } + } + try accounting.hashWork(64) + let result = String(decoding: output, as: UTF8.self) + try accounting.hashRelease(64); releaseOutput = false + return result + } +} diff --git a/Sources/JSONSchema/Document/JSONSchemaBuiltins.swift b/Sources/JSONSchema/Document/JSONSchemaBuiltins.swift index e35e698..cdab4b6 100644 --- a/Sources/JSONSchema/Document/JSONSchemaBuiltins.swift +++ b/Sources/JSONSchema/Document/JSONSchemaBuiltins.swift @@ -1,6 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 import Foundation -import CryptoKit import JSON extension JSONSchemaRegistry { /// Loads the integrity-checked packaged Draft 2020-12 meta-schema registry. @@ -49,31 +48,13 @@ extension JSONSchemaRegistry { data.append(chunk) try meter.release(UInt64(amount)) } + try meter.checkpoint(); try meter.retain(1) do { - try meter.checkpoint(); try meter.retain(1) guard try handle.read(upToCount: 1)?.isEmpty ?? true else { throw JSONSchemaCompilationFailure(.metaSchema) } } catch { throw JSONSchemaCompilationFailure(.metaSchema) } try meter.release(1) guard UInt64(data.count) == size else { throw JSONSchemaCompilationFailure(.metaSchema) } - var digest = SHA256(), offset = 0 - while offset < data.count { - let end = min(offset + Int(meter.limits[.cancellationQuantum]),data.count) - try meter.retain(UInt64(end-offset)) - let block = Data(data[offset.. UInt8 { nibble < 10 ? nibble+48 : nibble+87 } - for byte in digest.finalize() { - try meter.work(2); try meter.retain(2) - hexadecimal.append(hex(byte >> 4)); hexadecimal.append(hex(byte & 15)) - } - try meter.retain(64); let hash = String(decoding: hexadecimal,as: UTF8.self) - for _ in hexadecimal { try meter.work() } - try meter.release(128) - guard hash == expectedHash else { throw JSONSchemaCompilationFailure(.metaSchema) } + try verifyBuiltin(data, size: size, expectedHash: expectedHash, meter: &meter) var bytes: [UInt8] = [] for byte in data { try meter.work(); try meter.retain(1); bytes.append(byte) } let raw = try JSONParser().parse(bytes, meter: &meter) @@ -85,4 +66,13 @@ extension JSONSchemaRegistry { } return documents } + package static func verifyBuiltin(_ data: Data, size: UInt64, expectedHash: String, + meter: inout JSONWorkMeter) throws { + try meter.work() + guard UInt64(data.count) == size else { throw JSONSchemaCompilationFailure(.metaSchema) } + let hash = try JSONSHA256Core.hexadecimal(data, accounting: &meter) + defer { try? meter.release(64) } + try meter.work(64) + guard hash.utf8.elementsEqual(expectedHash.utf8) else { throw JSONSchemaCompilationFailure(.metaSchema) } + } } diff --git a/Tests/Fixtures/Integrity/Provenance.json b/Tests/Fixtures/Integrity/Provenance.json new file mode 100644 index 0000000..30564ce --- /dev/null +++ b/Tests/Fixtures/Integrity/Provenance.json @@ -0,0 +1,17 @@ +{ + "algorithm": "FIPS 180-4 SHA-256", + "standard": "https://doi.org/10.6028/NIST.FIPS.180-4", + "originalCodeLicense": "Apache-2.0", + "generator": "Python standard-library hashlib.sha256", + "fixture": "SHA256Vectors.json", + "fixture_sha256": "1b92c746b1f1af580cce162eefcb7b2d482e384fee8b23906bd07cba7899d20a", + "vectors": 24, + "formula": "byte[i]=(i*37+11)%256; lengths fixed in fixture", + "additionalKnownAnswers": [ + "empty", + "abc", + "abcdbc...nopq", + "one million ASCII a" + ], + "upstreamSourceAdapted": false +} diff --git a/Tests/Fixtures/Integrity/SHA256Vectors.json b/Tests/Fixtures/Integrity/SHA256Vectors.json new file mode 100644 index 0000000..5719494 --- /dev/null +++ b/Tests/Fixtures/Integrity/SHA256Vectors.json @@ -0,0 +1,103 @@ +{ + "schema_version": 1, + "algorithm": "SHA-256", + "generator": "Python standard-library hashlib.sha256; byte i=(i*37+11) mod 256", + "cases": [ + { + "length": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "length": 1, + "sha256": "e7cf46a078fed4fafd0b5e3aff144802b853f8ae459a4f0c14add3314b7cc3a6" + }, + { + "length": 2, + "sha256": "cdc63a6325d5fa92515578c0b418e6eeec1c6d085937a24fc43c2126ea517457" + }, + { + "length": 3, + "sha256": "b39fad1a1075f64570b3226d339ea818f9c66ecd2f1c59fd8b9c5a32b54c513f" + }, + { + "length": 7, + "sha256": "e759cdfe6d8119afd0dccf1b85394ff66d02636fd01db3189dae43e91e3c1385" + }, + { + "length": 31, + "sha256": "0cc420417cb6a768e632bf1be8d1f197c9c0e9dd1dd2021e3bd37c46cee0516f" + }, + { + "length": 55, + "sha256": "2900465fcb533e05a158fd2b3be0e5e3b03740d83060aa3580e0d98a96bf2384" + }, + { + "length": 56, + "sha256": "31454ff48ef36af2f08fd511bdc37d9d5855ac23e992e5ff5445cb6b7674a674" + }, + { + "length": 57, + "sha256": "bcc0a5d3791b985b7550e04ca660a6c63a589ba1edd2283c8e110e5b515df124" + }, + { + "length": 63, + "sha256": "5f6401b96532c36de4e65beec0409b69b1d181864c8009b7a04f43e5d56350d1" + }, + { + "length": 64, + "sha256": "94eb5de4943613fd048dc93393ab06877405faa39c11f53e9386083339833e7e" + }, + { + "length": 65, + "sha256": "fc518669b6eb4b4dd91827ecacef86689c725bd5bab888fd3b26dbb196eec954" + }, + { + "length": 95, + "sha256": "47b36f08053588e6added63bcd14bec0c3017ef2e11cbf22fcb900863d70e02a" + }, + { + "length": 119, + "sha256": "b0dc41b1a384e2f1203f0351b38fbeaafceef577ce1191d5bfc25da39f721eae" + }, + { + "length": 120, + "sha256": "5df24dd802ac26132ce608dcb5f09841eef039ee0f152acf98d26d17fe4e88e6" + }, + { + "length": 121, + "sha256": "5ed5a129bb49444fe2585d785920135a02f64350edf57c5f3ce7605b86f39039" + }, + { + "length": 127, + "sha256": "0fe729ff19257bd6fec853acc2ea355f6b34b58e6c0f684c3e188fcdfcd9baae" + }, + { + "length": 128, + "sha256": "0aedd4856f8eba0963627336ad5144a9a7dbe12498e6066f0165fc97d8ddee4c" + }, + { + "length": 129, + "sha256": "4f1757ae4bffbae86d775b831765b75af154d52f7deaa46dd378051a2d3ad57f" + }, + { + "length": 255, + "sha256": "3c835ac0bba7147eaa568a76183d465e72ac456df24b55e01d44dc87be05a971" + }, + { + "length": 256, + "sha256": "3ef33734daae0e353f132ff5f3241d8f86ba81f851c0b9685149f079c16eb45b" + }, + { + "length": 257, + "sha256": "08570fb3dfab53bcb3e8f4ca60b7331d9c603460a45d978226e18dd019468d07" + }, + { + "length": 1024, + "sha256": "ffbad8f947474cfdd5b2bb22d7e0bf5ee8ba2b7af859d0c2bb28622db6a4be47" + }, + { + "length": 4096, + "sha256": "4e441a3533bb2c10cd5649981d395744213e09a336746b5a3458fee4057205ec" + } + ] +} diff --git a/Tests/JSONSchemaTests/ResourceIntegrityTests.swift b/Tests/JSONSchemaTests/ResourceIntegrityTests.swift new file mode 100644 index 0000000..fcccf34 --- /dev/null +++ b/Tests/JSONSchemaTests/ResourceIntegrityTests.swift @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation +import Testing +import JSON +@testable import JSONSchema + +@Test func builtinIntegrityRejectsCorruptionAndPreservesOperationalFailures() throws { + let data = Data("abc".utf8) + let expected = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + var valid = try JSONWorkMeter(limits: .balanced, domain: .schemaCompilation) + try JSONSchemaRegistry.verifyBuiltin(data, size: 3, expectedHash: expected, meter: &valid) + #expect(valid.used(.retainedBytes) == 0) + for (bytes, size, hash) in [(Data("abd".utf8), UInt64(3), expected), (data, 4, expected), (data, 3, String(repeating: "0", count: 64))] { + var invocation = try JSONWorkMeter(limits: .balanced, domain: .schemaCompilation) + #expect(throws: JSONSchemaCompilationFailure.self) { try JSONSchemaRegistry.verifyBuiltin(bytes, size: size, expectedHash: hash, meter: &invocation) } + } + var limits = JSONLimits.balanced; limits[.compileWorkUnits] = 100 + var exhausted = try JSONWorkMeter(limits: limits, domain: .schemaCompilation) + #expect(throws: JSONFailure.self) { try JSONSchemaRegistry.verifyBuiltin(data, size: 3, expectedHash: expected, meter: &exhausted) } + #expect(exhausted.used(.retainedBytes) == 0) +} diff --git a/Tests/JSONTests/CodecTests.swift b/Tests/JSONTests/CodecTests.swift index aa890a0..51b221b 100644 --- a/Tests/JSONTests/CodecTests.swift +++ b/Tests/JSONTests/CodecTests.swift @@ -66,7 +66,12 @@ func everyByteSplitUsesIdenticalGrammar(text: String) throws { let name = file.lastPathComponent let duplicate = name == "y_object_duplicated_key.json" || name == "y_object_duplicated_key_and_value.json" do { - _ = try JSONParser().parse(Data(contentsOf: file)) + // Keep the empty-file vector; Swift 6.4 Linux Data(contentsOf:) + // leaks its read buffer for that input. FileHandle closes explicitly. + let handle = try FileHandle(forReadingFrom: file) + defer { try? handle.close() } + let bytes = try handle.readToEnd() ?? Data() + _ = try JSONParser().parse(bytes) #expect(!name.hasPrefix("n_") && !duplicate, "Unexpected acceptance: \(name)") accepted += 1 } catch let failure as JSONFailure { diff --git a/Tests/JSONTests/Release1_1/PayloadTests.swift b/Tests/JSONTests/Release1_1/PayloadTests.swift index 476c272..cc59e30 100644 --- a/Tests/JSONTests/Release1_1/PayloadTests.swift +++ b/Tests/JSONTests/Release1_1/PayloadTests.swift @@ -1,6 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 import Foundation -import CryptoKit import Testing @testable import JSON @@ -24,7 +23,8 @@ private struct PayloadPair: Codable { let a: JSONCodablePayload; let b: JSONCoda #expect(fixtures.count == 8) for fixture in fixtures { let bytes = Data(fixture.inner_json.utf8) - #expect(SHA256.hash(data: bytes).map { String(format: "%02x", $0) }.joined() == fixture.inner_utf8_sha256) + var meter = try JSONWorkMeter(limits: .balanced, domain: .core) + #expect(try JSONSHA256Core.hexadecimal(bytes, accounting: &meter) == fixture.inner_utf8_sha256) let root = try JSONParser().parse(bytes) let payload = try JSONCodablePayload(preserving: root) let encoded = try JSONEncoder().encode(payload) diff --git a/Tests/JSONTests/SHA256Tests.swift b/Tests/JSONTests/SHA256Tests.swift new file mode 100644 index 0000000..12ba8bd --- /dev/null +++ b/Tests/JSONTests/SHA256Tests.swift @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation +import Testing +import Synchronization +@testable import JSON + +private struct DigestVectors: Decodable { + struct Vector: Decodable { let length: Int; let sha256: String } + let cases: [Vector] +} +@Test func sha256IndependentKnownAnswersCoverPaddingAndBlockBoundaries() throws { + let root = URL(fileURLWithPath: #filePath).deletingLastPathComponent().deletingLastPathComponent() + let fixture = try JSONDecoder().decode(DigestVectors.self, from: Data(contentsOf: root.appendingPathComponent("Fixtures/Integrity/SHA256Vectors.json"))) + #expect(fixture.cases.count == 24) + for vector in fixture.cases { + let bytes = (0.. UInt64 in + var meter = try JSONWorkMeter(limits: .balanced, domain: .schemaCompilation) + meter.checkpointObserver = { work in + if work >= 1024 { withUnsafeCurrentTask { $0?.cancel() } } + } + do { + _ = try JSONSHA256Core.hexadecimal(Array(repeating: UInt8(97), count: 1024), accounting: &meter) + Issue.record("Expected cancellation inside compression") + } catch is CancellationError { + #expect(meter.used(.retainedBytes) == 0) + #expect(throws: CancellationError.self) { try meter.checkpoint() } + return meter.used(.workUnits) + } + return 0 + } + #expect(try await task.value >= 1024) +} diff --git a/Tools/MCPReadiness/Sources/MCPReadinessKit/Readiness.swift b/Tools/MCPReadiness/Sources/MCPReadinessKit/Readiness.swift index f3ecad5..3f379a8 100644 --- a/Tools/MCPReadiness/Sources/MCPReadinessKit/Readiness.swift +++ b/Tools/MCPReadiness/Sources/MCPReadinessKit/Readiness.swift @@ -28,7 +28,12 @@ public struct ReadinessReceipt: Codable, Sendable { public let cumulativeStages: [RemainingWork.Stage] } private func resource(_ name: String) throws -> [UInt8] { + #if os(Linux) + // Linux resourceURL already selects the copied directory named Resources. + let path = Bundle.module.bundleURL.appendingPathComponent("Resources").appendingPathComponent(name) + #else let path = Bundle.module.resourceURL!.appendingPathComponent("Resources").appendingPathComponent(name) + #endif return Array(try Data(contentsOf: path)) } public func runReadiness() throws -> ReadinessReceipt { diff --git a/Tools/Qualification/Sources/Qualification/AuthoredQualification.swift b/Tools/Qualification/Sources/Qualification/AuthoredQualification.swift index be8c9d2..d611239 100644 --- a/Tools/Qualification/Sources/Qualification/AuthoredQualification.swift +++ b/Tools/Qualification/Sources/Qualification/AuthoredQualification.swift @@ -41,7 +41,7 @@ private func isResource(_ operation: () throws -> Void, metric: JSONResourceMetr func qualifyAuthored(root: URL) async throws -> [AuthoredReceipt] { let url = root.appendingPathComponent("Tests/Fixtures/core-regressions.json"), data = try Data(contentsOf: url) let fixture = try JSONParser().parse(data) - var semantics = AuthoredReceipt(name: "authored_public_semantics", input_sha256: digest(data)) + var semantics = AuthoredReceipt(name: "authored_public_semantics", input_sha256: try digest(data)) for item in try fixture["codec"].array() { let text = try item["utf8_text"].text(), id = try item["id"].text(), expected = try item["expected"].text() let result = try verdict(Array(text.utf8)) diff --git a/Tools/Qualification/Sources/Qualification/JSONSHA256Core.swift b/Tools/Qualification/Sources/Qualification/JSONSHA256Core.swift new file mode 100644 index 0000000..a25242a --- /dev/null +++ b/Tools/Qualification/Sources/Qualification/JSONSHA256Core.swift @@ -0,0 +1,113 @@ +// SPDX-License-Identifier: Apache-2.0 +// Original implementation of FIPS 180-4 SHA-256. No cryptography API is exported. +// This file is mirrored byte-for-byte by the trusted qualification executable; +// Scripts/public_check.py checks that mirror. Production uses JSONWorkMeter. +package protocol JSONHashAccounting: ~Copyable { + mutating func hashWork(_ amount: UInt64) throws + mutating func hashRetain(_ bytes: UInt64) throws + mutating func hashRelease(_ bytes: UInt64) throws + mutating func hashInput(_ bytes: UInt64) throws +} +package enum JSONHashFailure: Error { case lengthOverflow } + +package enum JSONSHA256Core { + // Fixed immutable standard constants, shared like the pinned Unicode tables. + private static let constants: [UInt32] = [ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 + ] + private static func rotate(_ value: UInt32, by count: Int) -> UInt32 { + (value >> count) | (value << (32 - count)) + } + private static func compress( + _ words: inout [UInt32], state: inout [UInt32], accounting: inout B + ) throws { + for i in 16..<64 { + // Conservative primitive blocks include shifts, accesses and arithmetic. + // Each block is <=64 units; the production meter checks cancellation. + try accounting.hashWork(64) + let x = words[i - 15], y = words[i - 2] + let s0 = rotate(x, by: 7) ^ rotate(x, by: 18) ^ (x >> 3) + let s1 = rotate(y, by: 17) ^ rotate(y, by: 19) ^ (y >> 10) + // Wrapping UInt32 addition is required by SHA-256, never resource math. + words[i] = words[i - 16] &+ s0 &+ words[i - 7] &+ s1 + } + try accounting.hashWork(8) + var a = state[0], b = state[1], c = state[2], d = state[3] + var e = state[4], f = state[5], g = state[6], h = state[7] + for i in 0..<64 { + try accounting.hashWork(64) + let s1 = rotate(e, by: 6) ^ rotate(e, by: 11) ^ rotate(e, by: 25) + let choice = (e & f) ^ (~e & g) + let t1 = h &+ s1 &+ choice &+ constants[i] &+ words[i] + let s0 = rotate(a, by: 2) ^ rotate(a, by: 13) ^ rotate(a, by: 22) + let majority = (a & b) ^ (a & c) ^ (b & c) + let t2 = s0 &+ majority + h = g; g = f; f = e; e = d &+ t1 + d = c; c = b; b = a; a = t1 &+ t2 + } + try accounting.hashWork(32) + state[0] = state[0] &+ a; state[1] = state[1] &+ b + state[2] = state[2] &+ c; state[3] = state[3] &+ d + state[4] = state[4] &+ e; state[5] = state[5] &+ f + state[6] = state[6] &+ g; state[7] = state[7] &+ h + } + /// Returns 64 lowercase ASCII hex bytes, retaining that output in the caller's + /// ledger. Scratch is released on success, rejection or cancellation. + package static func hexadecimal( + _ bytes: C, accounting: inout B + ) throws -> String where C.Element == UInt8 { + // 8 state words + 64 schedule words + 8 compression words + a fixed frame. + let scratch: UInt64 = 336 + try accounting.hashRetain(scratch) + defer { try? accounting.hashRelease(scratch) } + try accounting.hashWork(72) + var state: [UInt32] = [0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19] + var words = Array(repeating: UInt32(0), count: 64) + var used = 0, length: UInt64 = 0 + for byte in bytes { + guard length < UInt64.max / 8 else { throw JSONHashFailure.lengthOverflow } + length += 1 + try accounting.hashInput(length); try accounting.hashWork(8) + words[used / 4] |= UInt32(byte) << (24 - (used % 4) * 8) + used += 1 + if used == 64 { + try compress(&words, state: &state, accounting: &accounting) + for i in 0..<16 { try accounting.hashWork(2); words[i] = 0 } + used = 0 + } + } + try accounting.hashWork(8) + words[used / 4] |= UInt32(0x80) << (24 - (used % 4) * 8) + if used >= 56 { + try compress(&words, state: &state, accounting: &accounting) + for i in 0..<16 { try accounting.hashWork(2); words[i] = 0 } + } + try accounting.hashWork(8) + let bits = length * 8 // Length was checked before increment, no overflow. + words[14] = UInt32(bits >> 32); words[15] = UInt32(truncatingIfNeeded: bits) + try compress(&words, state: &state, accounting: &accounting) + try accounting.hashRetain(128) // Hex buffer and result are simultaneously live. + var releaseOutput = true + defer { if releaseOutput { try? accounting.hashRelease(128) } } + var output: [UInt8] = []; output.reserveCapacity(64) + for word in state { + for shift in stride(from: 28, through: 0, by: -4) { + try accounting.hashWork(4) + let nibble = UInt8((word >> shift) & 15) + output.append(nibble < 10 ? nibble + 48 : nibble + 87) + } + } + try accounting.hashWork(64) + let result = String(decoding: output, as: UTF8.self) + try accounting.hashRelease(64); releaseOutput = false + return result + } +} diff --git a/Tools/Qualification/Sources/Qualification/Main.swift b/Tools/Qualification/Sources/Qualification/Main.swift index 8f9a018..5e1404e 100644 --- a/Tools/Qualification/Sources/Qualification/Main.swift +++ b/Tools/Qualification/Sources/Qualification/Main.swift @@ -1,6 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 import Foundation +#if canImport(Darwin) import Darwin +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#endif @main struct Qualification { static func main() async { @@ -18,7 +24,9 @@ import Darwin let root = URL(fileURLWithPath: FileManager.default.currentDirectoryPath, isDirectory: true) var report = QualificationReport(command: CommandLine.arguments.dropFirst().joined(separator: " ")) do { + #if os(macOS) guard ProcessInfo.processInfo.operatingSystemVersion.majorVersion >= 27 else { throw HarnessFailure("macOS 27 native baseline required") } + #endif let suites = try loadValue(root.appendingPathComponent(registry))["suites"].array() guard !suites.isEmpty else { throw HarnessFailure("No registered suites") } for suite in suites { diff --git a/Tools/Qualification/Sources/Qualification/Reporting.swift b/Tools/Qualification/Sources/Qualification/Reporting.swift index 22da6ca..3a07384 100644 --- a/Tools/Qualification/Sources/Qualification/Reporting.swift +++ b/Tools/Qualification/Sources/Qualification/Reporting.swift @@ -1,6 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 import Foundation -import CryptoKit import JSON import JSONSchema @@ -13,7 +12,17 @@ func errorCode(_ error: any Error) -> String { if let error = error as? JSONSchemaEvaluationFailure { return "schemaEvaluation:\(error.category)" } return String(describing: type(of: error)) } -func digest(_ bytes: Data) -> String { SHA256.hash(data: bytes).map { String(format: "%02x", $0) }.joined() } +// Fixture/provenance work is trusted harness work, outside owned library quotas. +private struct FixtureHashAccounting: JSONHashAccounting { + mutating func hashWork(_ amount: UInt64) throws {} + mutating func hashRetain(_ bytes: UInt64) throws {} + mutating func hashRelease(_ bytes: UInt64) throws {} + mutating func hashInput(_ bytes: UInt64) throws {} +} +func digest(_ bytes: Data) throws -> String { + var accounting = FixtureHashAccounting() + return try JSONSHA256Core.hexadecimal(bytes, accounting: &accounting) +} func loadValue(_ url: URL) throws -> JSON { try JSONParser().parse(Data(contentsOf: url)) } extension JSON { subscript(_ name: String) -> JSON { objectValue?[name] ?? .null } @@ -62,10 +71,20 @@ struct QualificationReport: Encodable, Sendable { let schema_version = 1 let command: String let time_utc = ISO8601DateFormatter().string(from: Date()) - let environment = ["os": ProcessInfo.processInfo.operatingSystemVersionString, "architecture": "arm64", "swift_language": "6", "required_tools": "6.4", "resource_profile": "balanced", "logical_accounting_version": "1"] + let environment = ["os": ProcessInfo.processInfo.operatingSystemVersionString, "architecture": qualificationArchitecture, + "swift_language": "6", "required_tools": "6.4", "resource_profile": "balanced", "logical_accounting_version": "1"] var status = "failed", suites: [SuiteReceipt] = [], authored: [AuthoredReceipt] = [], errors: [String] = [] let runtime_network_access = false } +private var qualificationArchitecture: String { + #if arch(arm64) + "arm64" + #elseif arch(x86_64) + "x86_64" + #else + "unqualified" + #endif +} struct VerifiedSuite { let root: URL, repository: String, commit: String, expected: Int, manifestHash: String let files: [String] @@ -74,13 +93,13 @@ struct VerifiedSuite { guard expected > 0, try entry["status"].text() == "imported" else { throw HarnessFailure("Suite missing or zero: " + repository) } root = try Self.path(try entry["fixture_path"].text(), relativeTo: repositoryRoot) let manifestURL = try Self.path(try entry["manifest"].text(), relativeTo: repositoryRoot) - let bytes = try Data(contentsOf: manifestURL); manifestHash = digest(bytes) + let bytes = try Data(contentsOf: manifestURL); manifestHash = try digest(bytes) let manifest = try JSONParser().parse(bytes) guard try manifest["commit"].text() == commit, try manifest["repository"].text() == repository, try manifest["expected_cases"].integer() == expected else { throw HarnessFailure("Suite manifest identity mismatch") } var paths: [String] = [], licenses = 0 for record in try manifest["files"].array() { let path = try record["path"].text(), data = try Data(contentsOf: Self.path(path, relativeTo: root)) - guard digest(data) == (try record["sha256"].text()), data.count == (try record["bytes"].integer()) else { throw HarnessFailure("Fixture hash/size mismatch: " + path) } + guard try digest(data) == record["sha256"].text(), data.count == (try record["bytes"].integer()) else { throw HarnessFailure("Fixture hash/size mismatch: " + path) } if try record["license"].bool() { licenses += 1 } paths.append(path) }