From 45c8d90ef7fd5fe954551d64360eb52e086143fc Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Wed, 14 Oct 2026 01:17:00 +0200 Subject: [PATCH 01/24] docs: audit 1.2.0 tests and plan 1.2.1 qualification --- .../Release1.2.1/AcceptanceMatrix.json | 512 ++++++ .../Qualification/Release1.2.1/Audit.md | 123 ++ .../Release1.2.1/CoverageBaseline.json | 1377 +++++++++++++++++ .../Qualification/Release1.2.1/Plan.md | 229 +++ .../Qualification/Release1.2.1/README.md | 20 + 5 files changed, 2261 insertions(+) create mode 100644 Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json create mode 100644 Documentation/Qualification/Release1.2.1/Audit.md create mode 100644 Documentation/Qualification/Release1.2.1/CoverageBaseline.json create mode 100644 Documentation/Qualification/Release1.2.1/Plan.md create mode 100644 Documentation/Qualification/Release1.2.1/README.md diff --git a/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json b/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json new file mode 100644 index 0000000..7dcd8d4 --- /dev/null +++ b/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json @@ -0,0 +1,512 @@ +{ + "schema_version": 1, + "source_version": "1.2.0", + "source_commit": "25fb04f747e583209b4c4511abf39e11ef5b4f80", + "target_version": "1.2.1", + "status": "plan_only", + "rows": [ + { + "id": "HAR-01", + "work_package": "T00", + "module": "All", + "topic": "Complete coverage export", + "status": "planned_extension", + "existing_evidence": [ + "fresh 192/173 root coverage run", + "default export omitted schema/regex/streaming mappings" + ], + "extension": "Export every test binary; fail missing module/executable/profile coverage; report generated and unmapped files explicitly.", + "oracle": "Expected source/module membership plus known omitted-object negative probe", + "acceptance": [ + "all five implementation modules present", + "all exact discoveries retained", + "branch/MC/DC marked unavailable" + ] + }, + { + "id": "HAR-02", + "work_package": "T00", + "module": "All", + "topic": "Unused code and evidence denominator", + "status": "planned_extension", + "existing_evidence": [ + "JSONOperationContext/JSONFingerprint declaration-only reference scan and 0% coverage" + ], + "extension": "Record retain/use/remove disposition before an adjusted active-code denominator. Preserve unadjusted totals.", + "oracle": "Direct source/reference inspection and code review", + "acceptance": [ + "no artificial test-only callers", + "cleanup separate from tests-only scope" + ] + }, + { + "id": "CORE-01", + "work_package": "T01", + "module": "JSON", + "topic": "Numeric and primitive type matrix", + "status": "planned_extension", + "existing_evidence": [ + "machineIntegerBoundaries", + "boundedBinaryBridgePreservesLongHalfwayAndSubnormalTails", + "qaSmallIntegerProjectionUsesMathematicalRangeOracle" + ], + "extension": "All documented native signed/unsigned widths, boundaries/equivalent decimal spellings, binary extrema/ties and Decimal precision/exponent failures.", + "oracle": "Fixed exact mathematical vectors and platform type limits", + "acceptance": [ + "exact versus rounded distinction", + "error category/code and path", + "native Int/WASI32 scope separate" + ] + }, + { + "id": "CORE-02", + "work_package": "T01", + "module": "JSON", + "topic": "Exact object/value invariants", + "status": "planned_extension", + "existing_evidence": [ + "scalarIdentityAndOrderedObjectSemantics", + "qaIndexThresholdEditsAgreeWithOrderedScalarOracle", + "controlledIndexCollisionsAndRefusedRehashPreservePriorMetadata" + ], + "extension": "Seeded equality/hash/order/copy-on-write laws and edit histories around each real index boundary, preserving baseline collisions.", + "oracle": "Independent ordered member lists and fixed decimal identity vectors", + "acceptance": [ + "scalar distinction", + "equal values hash equally within a process", + "order stable on replace/remove/reinsert", + "no persistent hash claim" + ] + }, + { + "id": "COD-01", + "work_package": "T01", + "module": "JSON", + "topic": "Explicit native container paths", + "status": "planned_extension", + "existing_evidence": [ + "nativeCodingUsesValuesAndComposedBytes", + "superCodingAndNonSendableModelsWorkSynchronously" + ], + "extension": "Nested keyed/unkeyed containers from keyed and unkeyed parents; super-for-key; single-value primitives; empty/end/nil index state.", + "oracle": "Hand-authored model decode/encode probes with literal expected values/paths", + "acceptance": [ + "exact fields and order where contracted", + "EOF/wrong-kind coding paths", + "successful roundtrip plus independent expected representation" + ] + }, + { + "id": "COD-02", + "work_package": "T01", + "module": "JSON", + "topic": "Latched nonthrowing Codable operations", + "status": "planned_extension", + "existing_evidence": [ + "codingKeyEnumerationRejectsChangedScalarSpelling", + "readAccessSharedProjectionLimitsAndTerminalAccounting", + "payloadCaughtOperationalFailureStaysTerminalAndCorruptionDoesNot" + ], + "extension": "contains/allKeys/nestedContainer/superEncoder failure/cancellation and catches; ordinary projection recovery without reset.", + "oracle": "Explicit failing-limit/caller models with named phase markers", + "acceptance": [ + "owned failure terminal", + "ordinary/caller errors correctly classified", + "work not refunded", + "no result certified after failed ownership" + ] + }, + { + "id": "COD-03", + "work_package": "T01", + "module": "JSON", + "topic": "Foreign coder fidelity and fallback", + "status": "planned_extension", + "existing_evidence": [ + "qaGeneratedExactRoundTripsAcrossNativeAndForeignPayloads", + "stringDictionaryProjectionRejectsScalarCollisions" + ], + "extension": "Controlled foreign scalar/keyed/unkeyed coders and Int64/UInt64/Decimal fallback, six kinds, unsupported exact values and propagated errors.", + "oracle": "Deliberate independent coder probes and fixed representability vectors", + "acceptance": [ + "no hidden Double fallback", + "foreign limitations explicit", + "Version1 wrapper remains exact", + "all six kinds represented" + ] + }, + { + "id": "COD-04", + "work_package": "T01", + "module": "JSON", + "topic": "Overloads/literals/conveniences", + "status": "planned_extension", + "existing_evidence": [ + "nativeCodingUsesValuesAndComposedBytes", + "payloadNativeCompositionUsesOneCumulativeMeter" + ], + "extension": "Literal/convenience value shapes, String/Data/byte parser facade parity and native value-coding convenience reports.", + "oracle": "Same small golden documents across independently expected entry points", + "acceptance": [ + "correct admitted result", + "configured limits intersection", + "no duplicate/reset work" + ] + }, + { + "id": "DIA-01", + "work_package": "T02", + "module": "JSON", + "topic": "Source and structured error fidelity", + "status": "planned_extension", + "existing_evidence": [ + "sourceSpansUseTypedPathsAndInvalidateOnEdit", + "qaMappedScalarPathsUseUTF8ByteCoordinates", + "diagnosticNeverRetainsCallerErrorDescription" + ], + "extension": "CR/LF/CRLF, multibyte/escaped keys, nested source spans, full first-failing paths and selected failure enums.", + "oracle": "Golden original byte offsets, half-open ranges and typed path fixtures", + "acceptance": [ + "category/code/metric", + "overflow attemptedUsage nil", + "source replacement invalidates mapping", + "caller description not traversed" + ] + }, + { + "id": "DIA-02", + "work_package": "T02", + "module": "JSON", + "topic": "Preview/redaction/completeness", + "status": "planned_extension", + "existing_evidence": [ + "previewRedactsExactKeysBeforeTraversingValues", + "previewCapsTotalTraversalAndOutputWithoutAdmittingWholeDocument" + ], + "extension": "Small/zero byte/depth/node/marker boundaries, nested redactions, scalar-distinct keys and UTF8 truncation.", + "oracle": "Golden allowed snippets and independently bounded byte counts", + "acceptance": [ + "redact before reading rejected payload", + "completeness explicit", + "no replacement-character corruption", + "no serializer claim" + ] + }, + { + "id": "BUD-01", + "work_package": "T02", + "module": "All", + "topic": "49 real operation boundary owners", + "status": "planned_extension", + "existing_evidence": [ + "publicOperationsReachEveryActiveResourceCap", + "everyResourcePolicyMatchesNativeProfilesAndInclusiveOverflowBoundaries" + ], + "extension": "Add real-operation N-1/N/N+1 and composed-lifetime rows for all meaningful metrics; retain zero quantum/scratch exceptions.", + "oracle": "Same complete operation calibration plus explicit owner/outcome map", + "acceptance": [ + "all49 baseline probes unchanged", + "no nested reset", + "peak versus live correct", + "synthetic overflow does not allocate huge inputs" + ] + }, + { + "id": "BUD-02", + "work_package": "T02", + "module": "All", + "topic": "Terminal work/cancellation and ownership", + "status": "planned_extension", + "existing_evidence": [ + "retiredWorkOverflowIsAtomicAndTerminal", + "cancellationReachesOwnedWork", + "sha256CompressionCancellationLatchesWithoutRetainingScratch" + ], + "extension": "Cancellation/failure at compile/evaluate/coding/stream/Patch owned checkpoints and await-return boundaries; exact refund behavior.", + "oracle": "Counted progress observers and actor gates with cleanup", + "acceptance": [ + "cancellation after real progress", + "no later owned work/handoffs after terminal error", + "consumed work retained", + "caller work excluded" + ] + }, + { + "id": "SCH-01", + "work_package": "T03", + "module": "JSONSchema", + "topic": "Applicators/evaluated locations/output", + "status": "planned_extension", + "existing_evidence": [ + "evaluatedLocationsFlowThroughSuccessfulReferencesAndComposition", + "arrayContainsAndConditionalAnnotationsRespectTheirBranches", + "outputCapacityCannotChangeVerdictsOrTruncateSilently" + ], + "extension": "Combined successful/failed branch and zero/one/multiple-match matrix with unevaluated state, output and diagnostic policies.", + "oracle": "Independent expected valid/invalid/operational decision table and official output schema", + "acceptance": [ + "annotation success/rollback rules", + "no quota-as-invalidity", + "output conversion does not reevaluate", + "incompleteness explicit" + ] + }, + { + "id": "SCH-02", + "work_package": "T03", + "module": "JSONSchema", + "topic": "References and immutable registries", + "status": "planned_extension", + "existing_evidence": [ + "productiveRecursionProgressesAndNonproductiveCyclesThrow", + "URIResolutionUsesRFC3986AndPercentFragmentsRemainEncoded", + "concurrentSchemaEvaluationsHaveExclusiveDynamicRegexAndAnnotationState" + ], + "extension": "Nested lexical bases, anchors/encoded pointers/dynamic shadowing, registry snapshots and shared-plan recovery combinations.", + "oracle": "Golden URI/resource/instance/keyword locations and separate invocation histories", + "acceptance": [ + "productive recursion allowed", + "cycles operational", + "no online resolution", + "no global mutable cache/state leak" + ] + }, + { + "id": "SCH-03", + "work_package": "T03", + "module": "JSONSchema", + "topic": "Raw/compiled inspection and compilation shapes", + "status": "planned_extension", + "existing_evidence": [ + "rawDeclarationInspectionNeverInfersOrNarrowsConstraints", + "compiledInspectionExposesStaticTargetsAndDynamicAmbiguity", + "schemaBuilderSharesConstructionAndDocumentAdmissionWork" + ], + "extension": "Public declaration getter/report paths, absent/null/unknown metadata, Boolean/raw graph nodes and malformed keyword shapes; selected builder composition additions.", + "oracle": "Exact raw fixture declarations and documented required vocabulary/dialect rules", + "acceptance": [ + "no inferred defaults/effective schema", + "dynamic target ambiguity preserved", + "selected builder phases cumulative", + "compile error versus resource/cancel distinguished" + ] + }, + { + "id": "REG-01", + "work_package": "T03", + "module": "JSONRegexRuntime", + "topic": "VM semantic decisions", + "status": "planned_extension", + "existing_evidence": [ + "deterministicIndependentRegexDifferentialCases", + "scalarBacktrackingSemantics", + "pinnedUnicodePropertiesAndIdentifierNames" + ], + "extension": "Paired capture rollback/unset, nullable repetition, nested lookaround/direction, anchors/line terminators and grammar/property negative matrices.", + "oracle": "Pinned independently generated outcomes, constrained to ECMA262-2020 and Unicode13", + "acceptance": [ + "match and no-match paired", + "unsupported/malformed compiled errors explicit", + "newer engine grammar not adopted", + "immutable program exclusive state" + ] + }, + { + "id": "REG-02", + "work_package": "T03", + "module": "JSONRegexRuntime", + "topic": "Regex budget/cleanup decisions", + "status": "planned_extension", + "existing_evidence": [ + "adversarialRegexPathsExhaustWorkInsteadOfReturningInvalidity", + "regexSearchLimitResetsButAggregateLimitNeverResets", + "sharedRegexProgramHasExclusiveConcurrentSearchState" + ], + "extension": "Instruction/state/capture/compile/search/aggregate boundaries across candidates, backreferences, sharing and interruption.", + "oracle": "Measured complete invocation and retained-state invariants", + "acceptance": [ + "all owned scans charged", + "no branch swallows resource/cancel", + "scratch released", + "new invocation recovers independently" + ] + }, + { + "id": "STR-01", + "work_package": "T04", + "module": "JSONStreaming", + "topic": "Held source/sink backpressure", + "status": "planned_extension", + "existing_evidence": [ + "earlyStopDoesNotPullAheadAndChunkAdmissionIsImmediate", + "awaitedSinkProducesIdenticalBytesAndStatistics", + "qaSinkCancellationAfterAcceptanceKeepsPrefixAndStopsHandoffs" + ], + "extension": "Gate actual suspension; release/fail/cancel with counters before/after resume and bounded test cleanup.", + "oracle": "Actor-owned pull/generation/handoff logs and expected accepted prefix", + "acceptance": [ + "one handoff/pull outstanding", + "no generation while sink held", + "accepted bytes cannot be recalled", + "source producer cancellation not invented" + ] + }, + { + "id": "STR-02", + "work_package": "T04", + "module": "JSONStreaming", + "topic": "Framing, partitions and terminal behavior", + "status": "planned_extension", + "existing_evidence": [ + "streamEverySplitAgreesWithNativeParser", + "strictNDJSONPreservesPhysicalBoundariesAndLocations", + "rootArrayAllElementsCompleteOnceAtEverySplit" + ], + "extension": "Bounded seeded multi-part splits/empty chunks, late failures, record boundaries, trailing whitespace/EOF and terminal next behavior.", + "oracle": "Golden strict documents, explicit framing decisions and original byte coordinates", + "acceptance": [ + "no early completion", + "no skipped malformed records", + "exactly once completion", + "no further pull after terminal error" + ] + }, + { + "id": "PAT-01", + "work_package": "T04", + "module": "JSONPatch", + "topic": "RFC6902 operation/state matrix", + "status": "planned_extension", + "existing_evidence": [ + "allRFC6902OperationsAndArrayMoveOrdering", + "rootRemovalRecreationAndNullRemainDistinct", + "patchFailuresAreTransactionalAndStrict" + ], + "extension": "Six operations by object/array/root state, same-path move, descendant/missing parents, fields, scalar/escaped keys and copy independence.", + "oracle": "Normative fixture results and bounded independent ordered-value model", + "acceptance": [ + "absence versus null", + "remove before target resolution", + "input unchanged on failure", + "irrelevant fields treated per RFC" + ] + }, + { + "id": "PAT-02", + "work_package": "T04", + "module": "JSONPatch", + "topic": "RFC7396/staging/budget sequences", + "status": "planned_extension", + "existing_evidence": [ + "mergePatchRFC7396ExamplesAndScalarKeys", + "mergePatchAdmitsInputsIndividuallyAndChecksIntermediateStates", + "discardedStagesReleaseOwnershipWithoutRefundingWork" + ], + "extension": "Normative appendix fixtures and small generated histories, intermediate peaks, stage failure and cumulative work.", + "oracle": "Normative expected outputs and independent member-list histories", + "acceptance": [ + "arrays replace", + "null member deletes/root null remains JSON", + "intermediate cap applies", + "work not refunded by removal" + ] + }, + { + "id": "E2E-01", + "work_package": "T05", + "module": "Public products", + "topic": "Portable public workflow", + "status": "planned_extension", + "existing_evidence": [ + "Examples/Consumer ProductExamples", + "20 rooted public consumer vectors", + "Tools/WasmAudit 13 public functions" + ], + "extension": "SwiftTesting bytes/read/schema/Patch/write/reparse/Version1 workflows with golden results and no private imports.", + "oracle": "Named independent golden inputs/results plus actual public accounting reports", + "acceptance": [ + "four public products only", + "regex through public schema", + "exact scalar/number/presence preserved", + "positive/invalid/operational separate" + ] + }, + { + "id": "E2E-02", + "work_package": "T05", + "module": "Public products", + "topic": "Transactional import and error injection", + "status": "planned_extension", + "existing_evidence": [ + "provisionalEventsNeverCertifyMalformedTail", + "failedStagePermanentlyTerminatesTheCallerAccount", + "MobileQualification disk phases" + ], + "extension": "Late tail/IO/quota/cancellation failures and a caller-staged commit plus cumulative remaining-work composition.", + "oracle": "Caller-owned stage logs, accepted prefix and complete-operation budget", + "acceptance": [ + "no test-application commit before completion", + "no fabricated failure report", + "no retry with full allowance", + "actual disk phases remain separate" + ] + }, + { + "id": "ASS-01", + "work_package": "T06", + "module": "All", + "topic": "Seeded valid/invalid replay and mutants", + "status": "planned_extension", + "existing_evidence": [ + "qaGeneratedExactRoundTripsAcrossNativeAndForeignPayloads", + "deterministicMalformedReplayPreservesStrictProgressAndRoundTrips" + ], + "extension": "Bounded additional replay and a 20-40 single-site mutant pilot across high-risk invariants.", + "oracle": "Independent seed/fixture outcomes and manually classified non-equivalent mutants", + "acceptance": [ + "minimized replay retained", + "survived/equivalent/unbuildable/timeout separate", + "no blanket nonzero tests", + "no conditional optimization selected" + ] + }, + { + "id": "QUAL-01", + "work_package": "T07", + "module": "All", + "topic": "Actual platform/safety/consumer closure", + "status": "planned_extension", + "existing_evidence": [ + "1.2.0 Native/Linux seven passed jobs", + "65 WASI commands with explicit per-host scopes" + ], + "extension": "Refresh affected actual lanes and exact discoveries; portability and source archive/public consumer checks after changes.", + "oracle": "Materialized receipts bound to exact source, environment and input pins", + "acceptance": [ + "all49 caps retained", + "policy cases separate", + "WASI sanitizers unavailable not passes", + "physical iPad deferred", + "no 1.3 features in patch" + ] + } + ], + "proposed_targets": { + "authored_line_percent_per_module": 98, + "authored_region_percent_per_module": 95, + "policy": "Proposed review targets plus full agreed contract/decision closure; justified uncovered paths require explicit disposition. No unsupported branch percentage." + }, + "new_features": false, + "root_dependency_changes": false, + "public_product_changes": false, + "work_packages": [ + "T00", + "T01", + "T02", + "T03", + "T04", + "T05", + "T06", + "T07" + ] +} diff --git a/Documentation/Qualification/Release1.2.1/Audit.md b/Documentation/Qualification/Release1.2.1/Audit.md new file mode 100644 index 0000000..952d0e0 --- /dev/null +++ b/Documentation/Qualification/Release1.2.1/Audit.md @@ -0,0 +1,123 @@ +# 1.2.0 test audit for 1.2.1 + +Audited source: tag `1.2.0`, commit +`25fb04f747e583209b4c4511abf39e11ef5b4f80`. This is an audit and proposed +test expansion; no new behavior is selected or new acceptance gate declared passed. +The scope is four public products and the private JSONRegexRuntime target. + +## Measured native baseline + +The source inventory has **192 root Swift Testing functions in 52 files**, plus +**173 parameterized invocations**. These numbers are separate from the thousands +of vectors executed inside individual functions. A fresh instrumented native run +passed the same exact discoveries using Swift 6.4, Swift language mode 6, +macOS 27.0, Xcode 27.2/SDK 27.2 and Apple LLVM 21.0.0. + +| Implementation module | Functions | Test files | Authored executable lines | Covered lines | Line coverage | Region coverage | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| JSON | 104 | 19 | 3,443 | 3,243 | 94.19% | 88.85% | +| JSONSchema | 51 | 18 | 2,410 | 2,361 | 97.97% | 93.62% | +| JSONRegexRuntime | 12 | 5 | 659 | 648 | 98.33% | 92.95% | +| JSONStreaming | 14 | 6 | 232 | 225 | 96.98% | 94.80% | +| JSONPatch | 11 | 4 | 185 | 179 | 96.76% | 90.57% | +| Total | 192 | 52 | 6,929 | 6,656 | 96.06% | 91.39% | + +These are **native root-suite execution measurements**, not assertion coverage, +branch coverage, public-consumer coverage or proof of every platform path. +Compiler-emitted executable lines differ from physical source lines. +Generated Unicode tables are reported separately. The regex table's emitted +lookup regions were covered; that does not prove every table entry was exercised. +The generated IDNA data file has no emitted executable mapping in this export. +Five declaration-only enum files also have no mapping; their observable cases +remain part of behavioral tests rather than being silently treated as covered. + +### Coverage tooling finding + +On this toolchain, `swift test --enable-code-coverage --disable-xctest` passed all +five bundles, but SwiftPM's default JSON export contained only JSON/Patch mappings. +An explicit LLVM export against **all five instrumented test executables** and the +merged profile produced the table above. Any future coverage lane must verify +module/file membership and reject a partial export before calculating percentages. +LLVM emitted zero branch and MC/DC counters for Swift here: these metrics are +**unavailable**, not 0% failure or 100% success. Regions provide a useful signal, +with semantic decision matrices and mutation checks supplying additional evidence. + +The machine-readable [baseline](CoverageBaseline.json) retains counts, exact +numerators, hashes, toolchain and export limitations. Raw profiles/exports and +command receipts remain local maintainer artifacts, outside source archives. + +## What is already strong + +- **Exact values and codec:** scalar-distinct keys, ordered objects/equality, + duplicate timing, machine boundaries, binary halfway/subnormal fixtures, + compact exponents, exact divisibility, malformed UTF-8/grammar, controls and + representative every-byte splits. Wide-object tests cross real metadata + thresholds and inject collisions, growth refusal and cancellation. +- **Coding and persistence:** native/foreign fidelity distinctions, presence, + string-key collapse rejection, guarded read scopes, caller errors, Base64 + canonicality, calendar strategies and exact Version 1 fixtures. The existing + QA wave includes 192 seeded documents, 772 small-integer spellings, 259 Base64 + fixtures and 1,152 padding-bit mutations. +- **Schema and regex:** full selected Draft 2020-12 input execution, productive + versus nonproductive references, dynamic scopes, evaluated locations, + successful-branch annotations, output policies, exact assertions, formats, + declaration inspection and immutable plan sharing. Nine schema-builder cases + already test cumulative construction/admission, retention and terminal failure. +- **Budgets and safety:** all 49 profiles/arithmetic boundaries, all 49 active + public cap outcomes, UInt64 versus Int capacity classification, aggregate work, + refunds/peaks, atomic overflow and inner cancellation after counted progress. + Native/Linux ASAN and six shared-state TSan functions passed for this source. +- **Patch and streaming:** all six RFC 6902 operations, root removal versus null, + staged rollback, intermediate caps, RFC 7396 examples, index transitions, + malformed tails, strict NDJSON, lifetime quotas, generated large streams and + accepted-prefix sink cancellation. + +The independent evidence is substantive and must be preserved: + +| Input or lane | Existing executed scope | +| --- | --- | +| JSONTestSuite | 318 observed; 287 evaluated successes; 31 policy cases | +| JSON Schema suite | 2,337 observed; 2,300 evaluated successes; 37 policy exclusions | +| Regex oracle | 31,824 cases over 442 patterns | +| Unicode normalization | 18,853 cases / 94,265 comparisons, Unicode 13 | +| Authored qualification | 82 semantic cases, nine budget/report cases, 4,396 malformed replays | +| Public schema consumer | 20 explicit roots: eight valid, twelve ordinary-invalid; two separate operational errors | +| External consumer | Four products, exact values/Codable/Pointer, schemas, Patch, streams/sinks and reports | +| Mobile qualification tool | Three Swift Testing functions, eight frozen public fixtures and real three-process disk phases | +| WASI tool | 13 public functions on both hosts; Node full roots/independent suites; narrower WasmKit scope | + +Case counts are not added together as a coverage score. Some workloads overlap. +Policy exclusions never count as evaluated successes; validation false remains +distinct from compilation, evaluation, resource and cancellation errors. +The tools are separate packages; they do not add root dependencies or products. + +## Concrete gaps and review items + +| Priority | Observation | Evidence and implication | +| --- | --- | --- | +| P0 | Codable container paths need explicit matrices | Encoding containers: 37.80% lines / 44.32% regions; decoding: 75.61% / 56.52%. Existing superclass/synthesized models do not exercise every nested/unkeyed/super-for-key and latched nonthrowing-container path. | +| P0 | Foreign bridges need controlled coder probes | JSONCodableBridge: 66.67% lines / 62.73% regions. Native passthrough is strong; explicit external decode/encode fallback and failure paths need precise assertions without treating Foundation as JSON authority. | +| P0 | Suspended I/O needs stronger causal tests | Existing successful sink uses Task.yield; failure and cancellation tests are useful. A held sink/source gate can prove no extra generation/pull/handoff occurs before acceptance, including error/cancellation on resume. | +| P0 | Some errors are asserted only by broad type | `#expect(throws: JSONFailure.self)` is often appropriate but can miss the wrong category/code/metric/location. Expand high-risk guard assertions to the contractual fields. | +| P1 | Cross-product workflows are mostly helper smoke tests | Consumers/qualification already provide integration evidence. Add portable Swift Testing workflows with late failure injection, repeated invocation isolation and public cumulative accounting. | +| P1 | Patch deserves a systematic operation/state matrix | Most RFC operations occur in one chained function; same-path move, root-state transitions, malformed operation fields and more normative vectors should have independent expectations. JSONPatch.swift region coverage is 87.42%. | +| P1 | Schema output/reference combinations need decision coverage | High execution coverage does not prove all interactions among branch annotations, dynamic scope, output mode and diagnostic exhaustion. Shape classification has 77.66% region coverage. | +| P1 | Regex needs deeper behavioral checks, not more easy patterns | Authored line coverage is already 98.33%. Focus on capture rollback, nullable repetition, directional lookarounds and budget/cancellation combinations; preserve the ECMA-262 2020/Unicode 13 dialect. | +| P1 | Source/preview and overloads need focused expansion | Preview regions 82.99%, JSONDocument 86.49%, read cursor 87.06%, convenience entry points 66.67%. Check exact byte paths, completeness/redaction, wrong-type recovery and equivalent facade behavior. | +| Review | Two package-private helpers have no call sites found | JSONOperationContext and JSONFingerprint appear only at their declarations across Sources/Tests/Tools/Examples and have 0% coverage. Classify intended use or remove separately after review; do not fabricate test-only callers just to increase coverage. | + +The 49 public cap probes currently verify active rejection at a zero cap, with the +explicit quantum/scratch exceptions. Arithmetic N/N+1 and overflow are already +tested for all profiles. The additional need is **operation-specific** N−1/N/N+1 +and composed-lifetime coverage, not a claim that the 49 metrics lack tests. + +## Boundaries of this audit + +This is the 1.2.0 implementation audit. It neither audits nor edits later feature +work. Existing Linux ARM64/x86_64, native safety and WASI receipts are reused for +unchanged source; this fresh coverage run is a native debug measurement only. +It supplies no new Linux/WASI coverage percentages, device/RSS/performance claims +or independent security certification. Physical iPad remains deferred under the +approved policy. E11/E12 and unselected optimizations stay deferred. + +Next: the [test expansion plan](Plan.md) and its [acceptance matrix](AcceptanceMatrix.json). diff --git a/Documentation/Qualification/Release1.2.1/CoverageBaseline.json b/Documentation/Qualification/Release1.2.1/CoverageBaseline.json new file mode 100644 index 0000000..d4693bb --- /dev/null +++ b/Documentation/Qualification/Release1.2.1/CoverageBaseline.json @@ -0,0 +1,1377 @@ +{ + "schema_version": 1, + "source_version": "1.2.0", + "source_commit": "25fb04f747e583209b4c4511abf39e11ef5b4f80", + "scope": "Native debug root Swift Testing execution, all five binary mappings; tools/consumers measured separately", + "run_exit": 0, + "root_functions": 192, + "parameterized_invocations": 173, + "root_test_files": 52, + "physical_source_files": 108, + "compiler": "Apple Swift 6.4 (swiftlang-6.4.0.34.1)", + "language_mode": 6, + "os": "macOS 27.0 ARM64", + "sdk": "27.2", + "llvm": "Apple LLVM 21.0.0", + "coverage_command": [ + "swift", + "test", + "--enable-code-coverage", + "--disable-xctest" + ], + "export_policy": "llvm-cov export with the merged profile and all five instrumented test executables; exact paths/binary/profile hashes retained in local maintainer evidence", + "default_export_incomplete": true, + "branch_coverage_available": false, + "mcdc_available": false, + "module_summaries": [ + { + "module": "JSON", + "all_source": { + "files": 50, + "lines": { + "count": 3443, + "covered": 3243, + "percent": 94.19111240197502 + }, + "regions": { + "count": 3427, + "covered": 3045, + "percent": 88.85322439451416 + }, + "functions": { + "count": 595, + "covered": 534, + "percent": 89.74789915966386 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "authored": { + "files": 50, + "lines": { + "count": 3443, + "covered": 3243, + "percent": 94.19111240197502 + }, + "regions": { + "count": 3427, + "covered": 3045, + "percent": 88.85322439451416 + }, + "functions": { + "count": 595, + "covered": 534, + "percent": 89.74789915966386 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "files": 0, + "lines": { + "count": 0, + "covered": 0, + "percent": null + }, + "regions": { + "count": 0, + "covered": 0, + "percent": null + }, + "functions": { + "count": 0, + "covered": 0, + "percent": null + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + { + "module": "JSONSchema", + "all_source": { + "files": 32, + "lines": { + "count": 2410, + "covered": 2361, + "percent": 97.96680497925311 + }, + "regions": { + "count": 3088, + "covered": 2891, + "percent": 93.62046632124353 + }, + "functions": { + "count": 384, + "covered": 365, + "percent": 95.05208333333333 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "authored": { + "files": 32, + "lines": { + "count": 2410, + "covered": 2361, + "percent": 97.96680497925311 + }, + "regions": { + "count": 3088, + "covered": 2891, + "percent": 93.62046632124353 + }, + "functions": { + "count": 384, + "covered": 365, + "percent": 95.05208333333333 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "files": 0, + "lines": { + "count": 0, + "covered": 0, + "percent": null + }, + "regions": { + "count": 0, + "covered": 0, + "percent": null + }, + "functions": { + "count": 0, + "covered": 0, + "percent": null + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + { + "module": "JSONRegexRuntime", + "all_source": { + "files": 6, + "lines": { + "count": 689, + "covered": 678, + "percent": 98.40348330914368 + }, + "regions": { + "count": 739, + "covered": 690, + "percent": 93.36941813261164 + }, + "functions": { + "count": 91, + "covered": 88, + "percent": 96.7032967032967 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "authored": { + "files": 5, + "lines": { + "count": 659, + "covered": 648, + "percent": 98.33080424886191 + }, + "regions": { + "count": 695, + "covered": 646, + "percent": 92.94964028776978 + }, + "functions": { + "count": 88, + "covered": 85, + "percent": 96.5909090909091 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "files": 1, + "lines": { + "count": 30, + "covered": 30, + "percent": 100.0 + }, + "regions": { + "count": 44, + "covered": 44, + "percent": 100.0 + }, + "functions": { + "count": 3, + "covered": 3, + "percent": 100.0 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + { + "module": "JSONStreaming", + "all_source": { + "files": 10, + "lines": { + "count": 232, + "covered": 225, + "percent": 96.98275862068965 + }, + "regions": { + "count": 269, + "covered": 255, + "percent": 94.79553903345725 + }, + "functions": { + "count": 60, + "covered": 57, + "percent": 95.0 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "authored": { + "files": 10, + "lines": { + "count": 232, + "covered": 225, + "percent": 96.98275862068965 + }, + "regions": { + "count": 269, + "covered": 255, + "percent": 94.79553903345725 + }, + "functions": { + "count": 60, + "covered": 57, + "percent": 95.0 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "files": 0, + "lines": { + "count": 0, + "covered": 0, + "percent": null + }, + "regions": { + "count": 0, + "covered": 0, + "percent": null + }, + "functions": { + "count": 0, + "covered": 0, + "percent": null + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + { + "module": "JSONPatch", + "all_source": { + "files": 4, + "lines": { + "count": 185, + "covered": 179, + "percent": 96.75675675675676 + }, + "regions": { + "count": 212, + "covered": 192, + "percent": 90.56603773584905 + }, + "functions": { + "count": 16, + "covered": 15, + "percent": 93.75 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "authored": { + "files": 4, + "lines": { + "count": 185, + "covered": 179, + "percent": 96.75675675675676 + }, + "regions": { + "count": 212, + "covered": 192, + "percent": 90.56603773584905 + }, + "functions": { + "count": 16, + "covered": 15, + "percent": 93.75 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "files": 0, + "lines": { + "count": 0, + "covered": 0, + "percent": null + }, + "regions": { + "count": 0, + "covered": 0, + "percent": null + }, + "functions": { + "count": 0, + "covered": 0, + "percent": null + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + } + ], + "authored": { + "files": 101, + "lines": { + "count": 6929, + "covered": 6656, + "percent": 96.06003752345215 + }, + "regions": { + "count": 7691, + "covered": 7029, + "percent": 91.3925367312443 + }, + "functions": { + "count": 1143, + "covered": 1056, + "percent": 92.38845144356955 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "files": 1, + "lines": { + "count": 30, + "covered": 30, + "percent": 100.0 + }, + "regions": { + "count": 44, + "covered": 44, + "percent": 100.0 + }, + "functions": { + "count": 3, + "covered": 3, + "percent": 100.0 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "unmapped_source_classification": [ + { + "path": "Sources/JSON/Pointer/JSONLocationComponent.swift", + "classification": "declaration_only_without_mapping", + "covered_claim": false + }, + { + "path": "Sources/JSONSchema/Formats/JSONIDNAData.swift", + "classification": "generated_data_without_mapping", + "covered_claim": false + }, + { + "path": "Sources/JSONSchema/Output/JSONValidationOutput.swift", + "classification": "declaration_only_without_mapping", + "covered_claim": false + }, + { + "path": "Sources/JSONStreaming/JSONFraming.swift", + "classification": "declaration_only_without_mapping", + "covered_claim": false + }, + { + "path": "Sources/JSONStreaming/JSONTopLevelArrayEvent.swift", + "classification": "declaration_only_without_mapping", + "covered_claim": false + }, + { + "path": "Sources/JSONPatch/JSONPatchResult.swift", + "classification": "declaration_only_without_mapping", + "covered_claim": false + } + ], + "hotspots": [ + { + "path": "Sources/JSON/Limits/JSONOperationContext.swift", + "summary": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "functions": { + "count": 1, + "covered": 0, + "percent": 0 + }, + "instantiations": { + "count": 1, + "covered": 0, + "percent": 0 + }, + "lines": { + "count": 3, + "covered": 0, + "percent": 0 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "regions": { + "count": 2, + "covered": 0, + "notcovered": 2, + "percent": 0 + } + }, + "generated": false + }, + { + "path": "Sources/JSON/Value/JSONFingerprint.swift", + "summary": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "functions": { + "count": 2, + "covered": 0, + "percent": 0 + }, + "instantiations": { + "count": 2, + "covered": 0, + "percent": 0 + }, + "lines": { + "count": 50, + "covered": 0, + "percent": 0 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "regions": { + "count": 51, + "covered": 0, + "notcovered": 51, + "percent": 0 + } + }, + "generated": false + }, + { + "path": "Sources/JSON/Coding/JSONEncodingContainers.swift", + "summary": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "functions": { + "count": 40, + "covered": 13, + "percent": 32.5 + }, + "instantiations": { + "count": 40, + "covered": 13, + "percent": 32.5 + }, + "lines": { + "count": 82, + "covered": 31, + "percent": 37.80487804878049 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "regions": { + "count": 88, + "covered": 39, + "notcovered": 49, + "percent": 44.31818181818182 + } + }, + "generated": false + }, + { + "path": "Sources/JSON/Coding/JSONDecodingContainers.swift", + "summary": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "functions": { + "count": 39, + "covered": 23, + "percent": 58.97435897435898 + }, + "instantiations": { + "count": 39, + "covered": 23, + "percent": 58.97435897435898 + }, + "lines": { + "count": 82, + "covered": 62, + "percent": 75.60975609756098 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "regions": { + "count": 92, + "covered": 52, + "notcovered": 40, + "percent": 56.52173913043478 + } + }, + "generated": false + }, + { + "path": "Sources/JSON/Construction/JSONConstruction.swift", + "summary": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "functions": { + "count": 5, + "covered": 3, + "percent": 60 + }, + "instantiations": { + "count": 5, + "covered": 3, + "percent": 60 + }, + "lines": { + "count": 5, + "covered": 3, + "percent": 60 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "regions": { + "count": 5, + "covered": 3, + "notcovered": 2, + "percent": 60 + } + }, + "generated": false + }, + { + "path": "Sources/JSON/Coding/JSONCodableBridge.swift", + "summary": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "functions": { + "count": 10, + "covered": 10, + "percent": 100 + }, + "instantiations": { + "count": 10, + "covered": 10, + "percent": 100 + }, + "lines": { + "count": 84, + "covered": 56, + "percent": 66.66666666666666 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "regions": { + "count": 110, + "covered": 69, + "notcovered": 41, + "percent": 62.727272727272734 + } + }, + "generated": false + }, + { + "path": "Sources/JSON/Coding/JSONValueConveniences.swift", + "summary": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "functions": { + "count": 3, + "covered": 2, + "percent": 66.66666666666666 + }, + "instantiations": { + "count": 3, + "covered": 2, + "percent": 66.66666666666666 + }, + "lines": { + "count": 9, + "covered": 6, + "percent": 66.66666666666666 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "regions": { + "count": 6, + "covered": 4, + "notcovered": 2, + "percent": 66.66666666666666 + } + }, + "generated": false + }, + { + "path": "Sources/JSONStreaming/JSONByteChunk.swift", + "summary": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "functions": { + "count": 3, + "covered": 2, + "percent": 66.66666666666666 + }, + "instantiations": { + "count": 3, + "covered": 2, + "percent": 66.66666666666666 + }, + "lines": { + "count": 3, + "covered": 2, + "percent": 66.66666666666666 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "regions": { + "count": 3, + "covered": 2, + "notcovered": 1, + "percent": 66.66666666666666 + } + }, + "generated": false + }, + { + "path": "Sources/JSON/Value/JSON.swift", + "summary": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "functions": { + "count": 3, + "covered": 3, + "percent": 100 + }, + "instantiations": { + "count": 3, + "covered": 3, + "percent": 100 + }, + "lines": { + "count": 48, + "covered": 44, + "percent": 91.66666666666666 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "regions": { + "count": 46, + "covered": 35, + "notcovered": 11, + "percent": 76.08695652173914 + } + }, + "generated": false + }, + { + "path": "Sources/JSONSchema/Compilation/JSONSchemaShape.swift", + "summary": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "functions": { + "count": 6, + "covered": 6, + "percent": 100 + }, + "instantiations": { + "count": 6, + "covered": 6, + "percent": 100 + }, + "lines": { + "count": 73, + "covered": 68, + "percent": 93.15068493150685 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "regions": { + "count": 94, + "covered": 73, + "notcovered": 21, + "percent": 77.6595744680851 + } + }, + "generated": false + }, + { + "path": "Sources/JSON/Diagnostics/JSONPreview.swift", + "summary": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "functions": { + "count": 7, + "covered": 7, + "percent": 100 + }, + "instantiations": { + "count": 7, + "covered": 7, + "percent": 100 + }, + "lines": { + "count": 98, + "covered": 95, + "percent": 96.93877551020408 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "regions": { + "count": 147, + "covered": 122, + "notcovered": 25, + "percent": 82.99319727891157 + } + }, + "generated": false + }, + { + "path": "Sources/JSON/Value/JSONKey.swift", + "summary": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "functions": { + "count": 9, + "covered": 8, + "percent": 88.88888888888889 + }, + "instantiations": { + "count": 9, + "covered": 8, + "percent": 88.88888888888889 + }, + "lines": { + "count": 29, + "covered": 28, + "percent": 96.55172413793103 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "regions": { + "count": 24, + "covered": 20, + "notcovered": 4, + "percent": 83.33333333333334 + } + }, + "generated": false + }, + { + "path": "Sources/JSONSchema/Document/JSONSchemaDocument.swift", + "summary": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "functions": { + "count": 4, + "covered": 3, + "percent": 75 + }, + "instantiations": { + "count": 4, + "covered": 3, + "percent": 75 + }, + "lines": { + "count": 15, + "covered": 12, + "percent": 80 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "regions": { + "count": 12, + "covered": 10, + "notcovered": 2, + "percent": 83.33333333333334 + } + }, + "generated": false + }, + { + "path": "Sources/JSONSchema/Compilation/JSONSchemaPlan.swift", + "summary": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "functions": { + "count": 7, + "covered": 6, + "percent": 85.71428571428571 + }, + "instantiations": { + "count": 7, + "covered": 6, + "percent": 85.71428571428571 + }, + "lines": { + "count": 10, + "covered": 9, + "percent": 90 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "regions": { + "count": 13, + "covered": 11, + "notcovered": 2, + "percent": 84.61538461538461 + } + }, + "generated": false + }, + { + "path": "Sources/JSON/Document/JSONDocument.swift", + "summary": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "functions": { + "count": 4, + "covered": 4, + "percent": 100 + }, + "instantiations": { + "count": 4, + "covered": 4, + "percent": 100 + }, + "lines": { + "count": 28, + "covered": 26, + "percent": 92.85714285714286 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "regions": { + "count": 37, + "covered": 32, + "notcovered": 5, + "percent": 86.48648648648648 + } + }, + "generated": false + }, + { + "path": "Sources/JSONSchema/Document/JSONSchemaBuiltins.swift", + "summary": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "functions": { + "count": 5, + "covered": 4, + "percent": 80 + }, + "instantiations": { + "count": 5, + "covered": 4, + "percent": 80 + }, + "lines": { + "count": 67, + "covered": 66, + "percent": 98.50746268656717 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "regions": { + "count": 68, + "covered": 59, + "notcovered": 9, + "percent": 86.76470588235294 + } + }, + "generated": false + }, + { + "path": "Sources/JSON/Reading/JSONReadCursor.swift", + "summary": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "functions": { + "count": 63, + "covered": 58, + "percent": 92.06349206349206 + }, + "instantiations": { + "count": 63, + "covered": 58, + "percent": 92.06349206349206 + }, + "lines": { + "count": 273, + "covered": 262, + "percent": 95.97069597069597 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "regions": { + "count": 170, + "covered": 148, + "notcovered": 22, + "percent": 87.05882352941177 + } + }, + "generated": false + }, + { + "path": "Sources/JSONPatch/JSONPatch.swift", + "summary": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "functions": { + "count": 10, + "covered": 9, + "percent": 90 + }, + "instantiations": { + "count": 10, + "covered": 9, + "percent": 90 + }, + "lines": { + "count": 116, + "covered": 111, + "percent": 95.6896551724138 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "regions": { + "count": 151, + "covered": 132, + "notcovered": 19, + "percent": 87.41721854304636 + } + }, + "generated": false + }, + { + "path": "Sources/JSONRegexRuntime/RegexParser.swift", + "summary": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "functions": { + "count": 51, + "covered": 48, + "percent": 94.11764705882352 + }, + "instantiations": { + "count": 51, + "covered": 48, + "percent": 94.11764705882352 + }, + "lines": { + "count": 327, + "covered": 316, + "percent": 96.63608562691131 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "regions": { + "count": 380, + "covered": 336, + "notcovered": 44, + "percent": 88.42105263157895 + } + }, + "generated": false + }, + { + "path": "Sources/JSONSchema/Formats/JSONIDNAMapping.swift", + "summary": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "functions": { + "count": 3, + "covered": 3, + "percent": 100 + }, + "instantiations": { + "count": 3, + "covered": 3, + "percent": 100 + }, + "lines": { + "count": 48, + "covered": 48, + "percent": 100 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "regions": { + "count": 62, + "covered": 55, + "notcovered": 7, + "percent": 88.70967741935483 + } + }, + "generated": false + } + ], + "unused_helpers": [ + { + "path": "Sources/JSON/Limits/JSONOperationContext.swift", + "source_and_test_reference_scan": "Declaration only", + "policy": "Requires disposition; do not write test-only calls to increase coverage" + }, + { + "path": "Sources/JSON/Value/JSONFingerprint.swift", + "source_and_test_reference_scan": "Declaration only", + "policy": "Requires disposition; do not write test-only calls to increase coverage" + } + ], + "raw_export_sha256": "481ddfe3f7d96bd3dc07738c447f124af0ae3327935672ac71067531ecff6581", + "original_default_export_sha256": "a94b4e6f87415c72c740d62bb3f8f22369548faba34d73547ca7f6e6c3ffc5bf", + "profile_sha256": "38d1f678d32b0351057b9ceb1e55739f2add92b5d84a23bdc601a459891ebe66", + "binary_sha256": [ + { + "module": "JSON", + "sha256": "52b7ee27cdfec6a83edb1bacdb7e803c8a58bda34ca82b5d34b541a071e5758f" + }, + { + "module": "JSONSchema", + "sha256": "5ec38245dfe5d51f66c893b42891823ef2da1db161cd1a5b67fc3ef2a95ef4e0" + }, + { + "module": "JSONRegexRuntime", + "sha256": "deec10278f74f7268b6af80c26be29e9020a69e7cf9cb22684efbbcd3497c0e4" + }, + { + "module": "JSONStreaming", + "sha256": "b1487a066d1e6c467a831df9bae2ff65a6b3dc5dfc6b3df3d63db7b006e40b68" + }, + { + "module": "JSONPatch", + "sha256": "92e3fc6c7e23c76ce5000a32ab019624a0d10dcf0880e34db22537dbcb61142b" + } + ], + "measured_utc": "2026-10-13T23:00:21.272850+00:00", + "limitations": [ + "Executed lines/regions do not establish assertion quality or all decisions", + "Compiler-synthesized/data-only mappings are not silently marked covered", + "No fresh portable/device/RSS or throughput claim", + "Existing unchanged-source qualification remains separate" + ] +} diff --git a/Documentation/Qualification/Release1.2.1/Plan.md b/Documentation/Qualification/Release1.2.1/Plan.md new file mode 100644 index 0000000..1f63e88 --- /dev/null +++ b/Documentation/Qualification/Release1.2.1/Plan.md @@ -0,0 +1,229 @@ +# 1.2.1 test expansion plan + +Build on the audited `1.2.0` snapshot without changing its public API, products, +finite profiles, dialect or supported-platform scope. The release focus is tests, +qualification harnesses and accurate evidence. This document proposes work; the +new suites are not implemented or qualified yet. See [audit](Audit.md), +[measured baseline](CoverageBaseline.json) and [acceptance matrix](AcceptanceMatrix.json). + +## Definition of completion + +The aim is near-complete evidence for observable contracts and high-risk decisions, +not a promise to execute every possible JSON input. Every matrix row needs positive, +negative and applicable resource/cancellation cases with an independent expected +result. Every meaningful new defect gets a minimized regression. Existing tests, +fixture pins, policy counts and release history remain intact. + +Use these proposed numerical review targets, alongside semantic acceptance: + +- At least 98% authored executable-line and 95% region coverage **per module** on + the supported native root suite, with no regression in covered obligations. +- Every zero/unmapped or remaining uncovered source path has a recorded disposition: + missing test, declaration/data-only mapping, platform-specific path, verified + unreachable guard or genuinely unused implementation. Exemptions need reasoning + and evidence; do not broadly exclude error handling or generated lookup algorithms. +- All documented high-risk contract/decision rows and all 49 metric-owner rows + have meaningful assertions. Branch/MC/DC percentages cannot gate this compiler + because it emits no counters for them. +- Coverage is a review trigger. A narrowly justified path may remain uncovered; + no weaker assertion, source removal or test-only call is allowed to make a badge green. + +The two unreferenced helpers require disposition before deciding an active-code +denominator. Keep the unadjusted baseline visible even if a later reviewed cleanup +removes them. Generated tables remain a separate stratum with integrity/vector +evidence. Documented public-symbol coverage is not test coverage. + +## Work packages and sequence + +### T00 — trustworthy measurement and contract ledger + +Implement a coverage lane/export helper with exact module and executable membership. +Collect all five binaries and the merged profile on this Xcode backend; support the +actual observed Linux layout separately. Normalize public paths and bind exports +to source hashes, compiler/SDK, command exits and exact test discoveries. Verify +that an intentionally omitted binary is rejected by the harness. Retain raw output +and classify unavailable branch counters explicitly. + +Maintain a machine-readable expected discovery map. Update native, safety, QA and +WASI consumers together when adding tests; preserve required check names, all 49 +cap probes and exact parameter/vector cardinality. Do not replace existing checks +with nonzero/exit-zero assertions. Record the two unused-helper decisions separately. + +Deliverable: measured baseline/diff, complete contract rows and reliable public +coverage artifacts. No dependency on new product behavior. + +### T01 — native Codable, foreign bridges and exact value entry points + +First implementation wave, because it addresses the largest measured active gap: + +- Explicit keyed/unkeyed/single-value models; nested keyed/unkeyed containers from + both parents; superEncoder/superDecoder with and without a key; empty containers, + decodeNil index movement, end-of-container and wrong-kind paths. +- Nonthrowing container operations that latch an owned failure: catch attempts + must not certify a result or reset the invocation ledger. Distinguish recoverable + ordinary type/caller errors from terminal owned failure/cancellation. +- Verify CodingKey identity, contains/allKeys and complete coding paths, including + scalar-distinct and numeric keys, unknown keys and absent/null/model mismatch. +- Controlled foreign Decoder/Encoder probes: Int64→UInt64→Decimal fallback, exact + rejection when none can represent a number, six JSON kinds and nested paths. + Preserve external-coder ownership/fidelity boundaries; no hidden Double fallback. +- Exercise literal/convenience and overload paths through observable values and + bounded admission, not only direct constructor calls. Extend integer-width, + finite rounding/extreme/tie and Decimal precision matrices with fixed expected + vectors. Compile any additional generic integer type before claiming support. + +Favor parameterized Swift Testing matrices. Suggested scope: 25–40 focused +functions covering many named vectors; this is sizing, not a final cardinality gate. + +### T02 — diagnostics, source maps and operation-specific budgets + +Check category/code/metric, attempted usage or overflow nil, full typed path and +UTF-8 byte coordinates for contractual errors. Cover CR/LF/CRLF, multibyte scalars, +escaped keys, nested arrays and source invalidation. Test preview redaction before +traversal, byte/depth/node exhaustion, completeness and marker-fit boundaries. + +Extend the existing 49-owner matrix with small real-operation N−1/N/N+1 vectors +where meaningful. Preserve exceptions: zero scratch succeeds; invalid quantum is +configuration failure. Test admission, active peaks, scratch refund, nonrefunded +work, subsystem/aggregate composition and terminal refusal after caught errors. +Use synthetic ledgers for UInt64/Int overflow; never allocate Int.max-sized fixtures. +Calibrate success and failure with the **same complete operation**, avoiding the +earlier exact-versus-rounded calibration mistake. + +### T03 — schema and regex decision combinations + +Add a bounded decision matrix for applicators, references and annotations: +successful/failed branches, anyOf multiple successes, oneOf zero/one/multiple, +not/if/then/else, contains/min/max, dependencies and unevaluated locations. +Combine with flag/basic output, collected annotations and complete/best-effort +diagnostics. Ordinary invalidity remains a completed result; an exhausted branch +must never become invalidity or a passing alternative. + +Expand lexical base/resource transitions, Pointer/anchor and percent-decoding, +dynamic shadowing, productive recursion, immutable registry replacement and plan +reuse after success/invalidity/operational failure. Test raw/compiled inspection +without inference/default insertion, including absent/null metadata and budgets. +Existing nine cumulative schema-builder regressions remain; add only new composed +interactions rather than duplicate them. + +Regex expansion targets private VM decisions: nullable/count repetition, nested +lookarounds, capture reset/unset/rollback, forward/backreferences, directional +lookbehind, anchors/line terminators, exact Unicode identifiers/properties and +candidate/start-position quotas. Expand paired positive/no-match and malformed +cases using independently produced, pinned oracle fixtures. A newer Node parser +does not redefine the advertised ECMA-262 2020 grammar. Preserve Unicode 13 and +the unavailable WASI sanitizer disposition. No engine, flag or fallback expansion. + +### T04 — strict streaming and transactional Patch + +Build reusable finite, checked-Sendable actor gates for a held source/sink. Assert +pull/generation/handoff counts while suspended, then release, throw or cancel. +Always unblock gates in cleanup; use an external process watchdog for hangs. +No arbitrary sleeps and no shared mutable test singleton. Include repeated next +after terminal errors where defined, no later pulls/writes, no completion and +the already accepted prefix. Do not claim to cancel an arbitrary caller producer. + +Extend every-single-split cases with bounded seeded multi-part partitions and +empty chunks. Cover UTF-8/escapes/exponent/duplicate boundaries, trailing whitespace, +CRLF/final NDJSON line, malformed later records, lifetime quotas and exactly-once +root-array completion. Generated large streams retain at most admitted chunks and +current elements; no preloaded large fixture is memory evidence. + +Patch matrices cover each operation against objects, arrays, existing/removed +roots, missing parents/targets, same-path move, descendant moves, index ordering, +escaped/scalar keys and wrong/missing operation fields. Expand normative RFC 6902 +and all selected RFC 7396 Appendix vectors with provenance before adaptation. +Add bounded model/oracle sequences, copy independence, intermediate failure +rollback and repeated-stage work/retention checks. Root absence never becomes null. + +### T05 — portable end-to-end Swift Testing suite + +Prefer a new **test target only**, dependent on the four public products, without +@testable/private-meter imports. JSONRegexRuntime participates through public +schema pattern evaluation; its direct private unit suite stays separate. An +additional test target creates no library product or runtime dependency. + +Implement at least these independent scenario families: + +1. Strict bytes→exact value→scoped model/presence→offline schema→Patch/Merge Patch + →awaited writing→strict reparse, with golden final value/bytes and reports. +2. Exact Version 1 transport around the workflow, including scalar-distinct keys, + number lexemes, large integers and absent/null state. +3. Immutable shared plan and independent worker invocations, mixing valid, + ordinary-invalid and operational-error instances, then recovery on a new call. +4. Root-array/NDJSON ingestion into a **caller-owned staged** sink; a malformed + tail, quota failure or cancellation prevents the test application's commit. + Accepted chunks remain accepted; this adds no persistence/transport semantics + to the library. +5. Public remaining-work composition with success at N and failure at N−1, + separate subsystem ceilings and accounting version. Stop a failed composition + because failed public work supplies no successful report; never reset its budget. +6. Fresh packaged resource/distribution consumer, including structured missing or + corrupted resources and exact installed bundle layouts. + +Use named deterministic fixture IDs and fault points before/after each owned +boundary or await. Aim for 12–20 focused functions; derive actual final counts +from discovery, not from this estimate. Existing examples/20 frozen vectors remain +independent consumer gates and do not establish downstream MCP conformance. + +### T06 — adversarial assurance and assertion quality + +Add bounded, fixed-seed valid-model generators and invalid-byte mutations with +minimized replay files. Use explicit oracles: mathematical fixed vectors, ordered +member lists, normative results and independent regex fixtures. Round trips through +the same implementation alone cannot rule out matching encoder/decoder bugs. + +Run a pilot of 20–40 deliberate single-site mutants in disposable worktrees: +inclusive bound, dropped charge/checkpoint, swallowed operational error, erroneous +annotation merge, canonical key collapse, early completion and staged publication. +Record selected/killed/survived/equivalent/unbuildable/timed-out separately. +An unbuildable mutant is not a behavioral kill. Minimize and add real assertions +for surviving non-equivalent mutants. Expand the pilot only if it finds useful gaps; +do not make an arbitrary mutation percentage the release's sole evidence. + +### T07 — platform qualification and closeout + +Run affected native/debug/release, ASAN, strict memory safety, broadened targeted +TSan, docs/consumer and independent lanes with exact new counts. Execute actual +Linux ARM64/x86_64 and the pinned WASI hosts with their established distinct scopes. +Apple SDK/simulator and disk tests follow the existing policy; physical iPad remains +deferred. New native percentages do not imply Linux/WASI/device coverage. Source +changes discovered during tests need their own regression/fix/requalification. + +Produce the revised coverage/contract ledger, case/policy counts, raw commands, +toolchains, source/fixture identities, mutant dispositions and test runtime/flake +observations. Preserve required Native check names and the existing 10% performance +review policy. Tests-only changes need no invented production benchmark rerun; +any production fix receives affected validation/calibration. Release tags and +publication remain separate maintainer actions. + +## Execution and review structure + +Recommended local/PR decomposition: T00 harness; T01+T02 core; T03 schema/regex; +T04 Patch/streams; T05 public integration; T06+T07 acceptance/evidence. Start the +first executable increment with explicit Codable containers and precise failures, +then held I/O gates. Keep test additions cohesive and reviewable; do not author all +new tests in one unreviewable batch or add functions merely to meet a count. + +Use fast deterministic PR matrices, full independent/native/Linux lanes for +acceptance, and bounded larger adversarial runs separately. Preserve every +baseline case/pin. Explicitly record scheduling/quota budgets for tests themselves. +No new library dependency is needed for a small seeded generator or test gates. + +The patch work stays on the 1.2.0 base. If main advances to a new feature release, +prepare 1.2.1 from a maintenance branch; do not incorporate or revert later +features to make a patch. Port applicable test commits forward only after reviewing +new semantics, counts and CI conflicts. The exact target branch/merge/release +actions are resolved at the release step; this plan does not authorize them. + +## Scope preserved + +Swift tools/compiler 6.4+, Swift language mode 6, Apple SDK/platform 27+, qualified +Linux/WASI hosts, four products, private regex, empty root dependency graph, +Apache-2.0/notices, exact number/scalar identity and finite cumulative budgets. +No JSONHTML/1.3 features, MCP/HTTP/UI implementation, new platform/backend, +online schema resolver, E11/E12 or optimization selection belongs in 1.2.1. + +The exhaustive-input problem is unbounded. Completion means every agreed contract +row and critical decision is evidenced, every uncovered path is accounted for, +and independent/operational/platform gates pass with honest limitations. diff --git a/Documentation/Qualification/Release1.2.1/README.md b/Documentation/Qualification/Release1.2.1/README.md new file mode 100644 index 0000000..dd0b099 --- /dev/null +++ b/Documentation/Qualification/Release1.2.1/README.md @@ -0,0 +1,20 @@ +# 1.2.1 test quality work + +This proposed patch-release work extends tests and qualification on the published +1.2.0 implementation. It changes no product API or behavior at the planning stage. + +- [Audit of existing unit, integration and platform evidence](Audit.md) +- [Native five-module coverage baseline](CoverageBaseline.json) +- [Eight-work-package test expansion plan](Plan.md) +- [Machine-readable acceptance groups](AcceptanceMatrix.json) + +Source baseline: `25fb04f747e583209b4c4511abf39e11ef5b4f80` (`1.2.0`). +Native baseline: 192 root functions, 173 parameterized invocations; 96.06% authored +executable lines and 91.39% regions. These execution measures do not establish +assertion completeness or unsupported branch/device coverage. + +Recommended first wave: complete coverage export/cardinality checks, explicit +Codable containers and precise failures, then held source/sink backpressure. +Subsequent waves expand combined schema/regex decisions, Patch, public end-to-end +flows and bounded adversarial/mutation evidence. Existing independent suites, +all 49 caps and platform-specific limitations remain authoritative. From 3c0c03b718c677ce2cb0a3f6c66488b5111d1098 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Wed, 14 Oct 2026 05:20:21 +0200 Subject: [PATCH 02/24] test: qualify Codable containers and export complete module coverage --- Documentation/CI/README.md | 21 +- Documentation/Qualification/README.md | 7 + .../Qualification/Release1.2.1/Plan.md | 4 +- .../Qualification/Release1.2.1/README.md | 1 + .../Qualification/Release1.2.1/Wave1.md | 57 +++ .../Qualification/TestDiscovery.json | 30 ++ Scripts/ci.py | 28 +- Scripts/coverage.py | 186 ++++++++ Scripts/test_discovery.py | 15 + Scripts/test_qualification_harness.py | 100 +++++ Scripts/wasm_support.py | 6 +- Tests/JSONTests/CodableContainerTests.swift | 413 ++++++++++++++++++ 12 files changed, 858 insertions(+), 10 deletions(-) create mode 100644 Documentation/Qualification/Release1.2.1/Wave1.md create mode 100644 Documentation/Qualification/TestDiscovery.json create mode 100644 Scripts/coverage.py create mode 100644 Scripts/test_discovery.py create mode 100644 Scripts/test_qualification_harness.py create mode 100644 Tests/JSONTests/CodableContainerTests.swift diff --git a/Documentation/CI/README.md b/Documentation/CI/README.md index 567f988..df239e0 100644 --- a/Documentation/CI/README.md +++ b/Documentation/CI/README.md @@ -44,6 +44,25 @@ streaming regression is documented in ../Performance/README.md. Run locally with `python3 Scripts/ci.py --lane native`, `--lane safety` or `--lane docs`. The first hosted result must be inspected before marking CI qualified. +## Coverage and test discovery + +`python3 Scripts/ci.py --lane coverage` runs the full instrumented native Swift +Testing suite and exports all five test executables against its merged profile. +SwiftPM's default export can contain only a subset of executable modules on the +supported Xcode backend. The lane rejects missing/duplicate source mappings, +stale unmapped classifications, impossible counters and changed discovery. An +actual LLVM export omitting the schema executable must be rejected. Branch/MC/DC +counters are reported as unavailable when the compiler emits none; generated +Unicode data and executable lookup regions remain separate. + +[TestDiscovery.json](../Qualification/TestDiscovery.json) is the exact root, +parameterized, QA and shared-state discovery contract. Native, safety and WASI +read that same contract. Native and coverage also run ten Python harness behavior +tests; these check qualification tooling and do not count as Swift library tests. +The existing required Native check names/settings and all 49 cap probes remain. +The coverage lane is available locally; this increment does not activate a new +hosted check or claim a Linux/WASI coverage run. + ## Linux development lanes The additive Linux workflow selects the pinned official Swift 6.4.0 Ubuntu 24.04 @@ -63,7 +82,7 @@ distribution, architecture or libc claims. `python3 Scripts/ci.py --lane wasm` requires the matching Swift.org 6.4.0 compiler, SDK and WasmKit plus Node. It records cross-build and runtime outcomes separately. -It preserves all 192/173 root discoveries, independent corpus policies and 49 cap +It preserves the exact root discoveries in TestDiscovery.json, independent corpus policies and 49 cap outcomes; both hosts also execute the frozen public consumer. The [WASI contract](../Architecture/WebAssembly.md) defines host/safety scopes. This local lane neither activates hosted CI nor substitutes for Apple/Linux gates. diff --git a/Documentation/Qualification/README.md b/Documentation/Qualification/README.md index b7ee680..1c7f4a1 100644 --- a/Documentation/Qualification/README.md +++ b/Documentation/Qualification/README.md @@ -89,3 +89,10 @@ The WASI increment adds three boundary/accounting functions: 192 root functions invocations and all 49 public metric outcomes. Native 64-bit overflow/rendezvous cases remain; wasm32 uses UInt64 ledgers and cooperative checkpoint cancellation. Run the [WASI lane](../Architecture/WebAssembly.md) for its explicit per-host scope. + +The first [1.2.1 test wave](Release1.2.1/Wave1.md) adds 23 public Codable behavior +functions: exactly 215 root functions (JSON 127, schema 51, regex 12, streaming 14, +Patch 11), retaining all 173 parameterized invocations and all 49 cap outcomes. +The focused QA lane now expects 34/3/3/2 functions in JSON/schema/streaming/Patch. +Historical counts above stay tied to their original receipts. The exact current +[discovery contract](TestDiscovery.json) is shared by native/safety/WASI harnesses. diff --git a/Documentation/Qualification/Release1.2.1/Plan.md b/Documentation/Qualification/Release1.2.1/Plan.md index 1f63e88..86fe20d 100644 --- a/Documentation/Qualification/Release1.2.1/Plan.md +++ b/Documentation/Qualification/Release1.2.1/Plan.md @@ -2,8 +2,8 @@ Build on the audited `1.2.0` snapshot without changing its public API, products, finite profiles, dialect or supported-platform scope. The release focus is tests, -qualification harnesses and accurate evidence. This document proposes work; the -new suites are not implemented or qualified yet. See [audit](Audit.md), +qualification harnesses and accurate evidence. The [first wave](Wave1.md) implements +T00 and a focused T01 subset; the remaining work below is still proposed. See [audit](Audit.md), [measured baseline](CoverageBaseline.json) and [acceptance matrix](AcceptanceMatrix.json). ## Definition of completion diff --git a/Documentation/Qualification/Release1.2.1/README.md b/Documentation/Qualification/Release1.2.1/README.md index dd0b099..816598e 100644 --- a/Documentation/Qualification/Release1.2.1/README.md +++ b/Documentation/Qualification/Release1.2.1/README.md @@ -7,6 +7,7 @@ This proposed patch-release work extends tests and qualification on the publishe - [Native five-module coverage baseline](CoverageBaseline.json) - [Eight-work-package test expansion plan](Plan.md) - [Machine-readable acceptance groups](AcceptanceMatrix.json) +- [Implemented first wave and remaining scope](Wave1.md) Source baseline: `25fb04f747e583209b4c4511abf39e11ef5b4f80` (`1.2.0`). Native baseline: 192 root functions, 173 parameterized invocations; 96.06% authored diff --git a/Documentation/Qualification/Release1.2.1/Wave1.md b/Documentation/Qualification/Release1.2.1/Wave1.md new file mode 100644 index 0000000..f24150e --- /dev/null +++ b/Documentation/Qualification/Release1.2.1/Wave1.md @@ -0,0 +1,57 @@ +# 1.2.1 first test wave + +Base: published 1.2.0, `25fb04f747e583209b4c4511abf39e11ef5b4f80`. +This isolated increment implements T00 measurement/discovery and a focused T01 +Codable-container subset. Production Sources, APIs, products, platform declarations, +runtime dependencies, finite profiles, fixture pins and required check names remain +unchanged. Later 1.3 feature work is outside this branch. + +## Added behavior + +[CodableContainerTests.swift](../../../Tests/JSONTests/CodableContainerTests.swift) +adds 23 nonparameterized Swift Testing functions using public JSON interfaces: + +- Keyed/unkeyed nested containers, named/default/unkeyed super containers, explicit + empty values and repeated keyed requests/replacement preserving insertion order. +- All 14 standard Bool/String/Float/Double/signed/unsigned primitive overloads, + nil and generic exact JSON values against independently specified golden values. +- Unkeyed decodeNil/index/end behavior and recovery after ordinary type errors; + unknown fields, key presence and explicit null. +- Complete typed coding paths for missing keys, wrong kinds and inexact integer + projection; underlying structured causes and floating range errors. +- All nonthrowing keyed/unkeyed child-request resource failures, contains/allKeys + failure latching, conflicting containers, unset roots and caught cancellation. +- Identical complete nested operations succeeding at N and failing at N−1 under + one cumulative work ledger, with exact domain/metric/limit/attempted/overflow + assertions. Caller errors remain distinct; fresh invocations remain independent. + +The root expectation changes from 192 to **215** functions. Parameterized discovery +stays **173**; these numbers are distinct from vectors looped within a function. +The existing QA filter includes the new cases, with exact **34/3/3/2** module counts. +No passing placeholder, softened original expectation or test-only private caller +is added. The two unused helpers retain their original uncovered disposition. + +## Measurement and qualification harness + +Run `python3 Scripts/ci.py --lane coverage` on the supported native toolchain. +Its exporter collects each instrumented executable and merged profile, binds source +and binary identities, checks the full source denominator and classifies generated +and unmapped files. It must reject an actual export with the schema binary omitted. +Branch/MC/DC unavailability is explicit. Ten negative/positive Python harness tests +exercise tooling boundaries; they are separate from Swift behavior counts. + +[TestDiscovery.json](../TestDiscovery.json) replaces duplicated current root-count +constants in native and WASI tooling. All 49 metric probes and independent policy +counts remain exact. The coverage lane is an additional local native lane; Linux +layout fixture tests do not certify execution on Linux. The existing native/safety +and focused QA lanes are required for this increment, with command receipts and +final source identities retained outside the public source distribution. + +## Remaining plan scope + +The broader [plan](Plan.md) remains active. Foreign coder probe matrices, wider +numeric strategy combinations, precise source/preview diagnostics, held I/O gates, +schema/regex/Patch decision expansion, public end-to-end scenarios and adversarial +assurance are later waves. The proposed 98% line/95% region targets have not been +declared achieved by adding these cases. New Linux/WASI and device results are not +implied by a native run. No release or remote action is part of this increment. diff --git a/Documentation/Qualification/TestDiscovery.json b/Documentation/Qualification/TestDiscovery.json new file mode 100644 index 0000000..c2dd4bf --- /dev/null +++ b/Documentation/Qualification/TestDiscovery.json @@ -0,0 +1,30 @@ +{ + "schema_version": 1, + "root_functions": { + "JSONTests": 127, + "JSONSchemaTests": 51, + "JSONRegexRuntimeTests": 12, + "JSONStreamingTests": 14, + "JSONPatchTests": 11 + }, + "parameterized_invocations": 173, + "qa_functions": { + "JSONTests": 34, + "JSONSchemaTests": 3, + "JSONStreamingTests": 3, + "JSONPatchTests": 2 + }, + "shared_state_functions": {"JSONTests": 4, "JSONSchemaTests": 1, "JSONRegexRuntimeTests": 1}, + "coverage_unmapped": { + "Sources/JSON/Pointer/JSONLocationComponent.swift": "declaration-only enum", + "Sources/JSONSchema/Output/JSONValidationOutput.swift": "declaration-only enum", + "Sources/JSONSchema/Formats/JSONIDNAData.swift": "generated data, no executable mapping", + "Sources/JSONStreaming/JSONFraming.swift": "declaration-only enum", + "Sources/JSONStreaming/JSONTopLevelArrayEvent.swift": "declaration-only enum", + "Sources/JSONPatch/JSONPatchResult.swift": "declaration-only enum" + }, + "coverage_generated": [ + "Sources/JSONSchema/Formats/JSONIDNAData.swift", + "Sources/JSONRegexRuntime/RegexUnicodeTables.swift" + ] +} diff --git a/Scripts/ci.py b/Scripts/ci.py index cf1d6db..ece29c5 100644 --- a/Scripts/ci.py +++ b/Scripts/ci.py @@ -4,6 +4,7 @@ import argparse, datetime, hashlib, json, os, pathlib, re, subprocess, sys from archive_candidate import files from preflight import collect, collect_linux +from test_discovery import DISCOVERY, ROOT_COUNTS, PARAMETER_INVOCATIONS ROOT = pathlib.Path(__file__).resolve().parents[1] SHARED_TESTS = 'independentCodecFacadesAndExactValuesCanBeSharedAcrossTasks|concurrentSchemaEvaluationsHaveExclusiveDynamicRegexAndAnnotationState|sharedRegexProgramHasExclusiveConcurrentSearchState|immutableIndexedObjectCanBeSharedAcrossTasks|qaSharedPayloadAndRootHaveIndependentConcurrentReadScopes|sha256InputCapacityAndConcurrentInvocationsRemainIndependent' @@ -20,6 +21,11 @@ def commands(lane, output): ['python3', '-B', 'Scripts/public_check.py'], ['python3', '-B', 'Scripts/wasm_support.py', '--output', str(output / 'probes')], ] + if lane == 'coverage': + return [ + ['python3', '-B', 'Scripts/test_qualification_harness.py'], + ['python3', '-B', 'Scripts/coverage.py', '--output', str(output / 'export')], + ] if lane == 'linux': # Root behavior/corpora remain required. SwiftData and Apple SDK/lifetime # probes are separate Apple acceptance; they never become Linux passes. @@ -41,6 +47,7 @@ def commands(lane, output): if lane == 'native': return [ ['python3', '-B', 'Scripts/public_check.py'], + ['python3', '-B', 'Scripts/test_qualification_harness.py'], ['swift', 'package', 'dump-package'], ['swift', 'package', 'show-dependencies', '--format', 'json'], ['swift', 'build', '-c', 'release'], @@ -94,10 +101,10 @@ def check_result(command, code, log): counts = [int(n) for n in re.findall(r'Test run with (\d+) tests? in[^\n]*passed', log)] passed = code == 0 if command[:2] == ['swift', 'test']: - expected = [13] if 'Tools/WasmAudit' in command else [2, 3, 3, 11] if 'qa' in command else [7] if R0_TESTS in command else [3] if 'Tools/MCPReadiness' in command or 'Tools/MobileQualification' in command else [1] if 'Tools/Qualification' in command else [1, 1, 4] if 'thread' in command else [11, 12, 14, 51, 104] + expected = [13] if 'Tools/WasmAudit' in command else sorted(DISCOVERY['qa_functions'].values()) if 'qa' in command else [7] if R0_TESTS in command else [3] if 'Tools/MCPReadiness' in command or 'Tools/MobileQualification' in command else [1] if 'Tools/Qualification' in command else sorted(DISCOVERY['shared_state_functions'].values()) if 'thread' in command else sorted(ROOT_COUNTS.values()) passed = passed and sorted(counts) == expected if '--filter' not in command and not any(x.startswith('Tools/') for x in command): - passed = passed and sum(int(n) for n in re.findall(r'with (\d+) test cases passed', log)) == 173 + passed = passed and sum(int(n) for n in re.findall(r'with (\d+) test cases passed', log)) == PARAMETER_INVOCATIONS if 'Tools/Qualification' in command: probes = re.findall(r'^CAP_PROBE (\w+) (\S+)$', log, re.MULTILINE) matrix = json.loads((ROOT / 'Documentation/Qualification/LimitsMatrix.json').read_text()) @@ -110,11 +117,13 @@ def check_result(command, code, log): passed = json.loads(log)['dependencies'] == [] if command[:2] == ['swift', 'run'] and 'Tools/MobileQualification' in command: passed = passed and all(line in log.splitlines() for line in ['PUBLIC_BEHAVIOR passed fixtures=8', 'STREAM_CANCELLATION passed', 'DISK_PERSISTENCE passed containers=3 rows=8', 'MOBILE_QUALIFICATION passed cases=3']) + if 'Scripts/test_qualification_harness.py' in command: + passed = passed and 'HARNESS_TESTS passed cases=10' in log.splitlines() return passed, counts def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--lane', choices=['native', 'safety', 'docs', 'r0', 'qa', 'linux', 'linux-safety', 'wasm'], required=True) + parser.add_argument('--lane', choices=['native', 'safety', 'docs', 'r0', 'qa', 'coverage', 'linux', 'linux-safety', 'wasm'], required=True) parser.add_argument('--output') args = parser.parse_args() output = ROOT / (args.output or '.build/ci/' + args.lane) @@ -165,11 +174,22 @@ def save(): if receipt['status'] == 'passed' and args.lane == 'wasm': try: wasi = json.loads((output / 'probes/wasi.json').read_text()) - if wasi['status'] != 'passed' or wasi['root_counts'] != {'JSONTests':104,'JSONSchemaTests':51,'JSONRegexRuntimeTests':12,'JSONStreamingTests':14,'JSONPatchTests':11}: + if wasi['status'] != 'passed' or wasi['root_counts'] != ROOT_COUNTS: raise ValueError('Missing WASI qualification boundary') receipt['materialized_wasi'] = {'path':'probes/wasi.json','sha256':hashlib.sha256((output / 'probes/wasi.json').read_bytes()).hexdigest()} except (OSError,KeyError,ValueError,TypeError) as error: receipt['status'] = 'failed'; receipt['evidence_error'] = str(error) + if receipt['status'] == 'passed' and args.lane == 'coverage': + try: + coverage = json.loads((output / 'export/coverage.json').read_text()) + if (coverage['status'] != 'passed' or coverage['root_counts'] != ROOT_COUNTS + or coverage['parameterized_invocations'] != PARAMETER_INVOCATIONS + or coverage['omitted_binary_probe']['status'] != 'rejected' + or set(coverage['modules']) != {name.removesuffix('Tests') for name in ROOT_COUNTS}): + raise ValueError('Missing complete native coverage boundary') + receipt['materialized_coverage'] = {'path':'export/coverage.json', 'sha256':hashlib.sha256((output / 'export/coverage.json').read_bytes()).hexdigest()} + except (OSError,KeyError,ValueError,TypeError) as error: + receipt['status'] = 'failed'; receipt['evidence_error'] = str(error) receipt['source_inputs_unchanged'] = snapshot() == inputs if not receipt['source_inputs_unchanged']: receipt['status'] = 'failed' diff --git a/Scripts/coverage.py b/Scripts/coverage.py new file mode 100644 index 0000000..6497f7e --- /dev/null +++ b/Scripts/coverage.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Export every instrumented root bundle; reject partial or stale Swift coverage.""" +import argparse +import datetime +import hashlib +import json +import pathlib +import platform +import shutil +import subprocess +import sys +import time +from archive_candidate import files +from test_discovery import DISCOVERY, ROOT, ROOT_COUNTS, PARAMETER_INVOCATIONS, check_root + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + +def executable_objects(bin_directory): + result = {} + for name in ROOT_COUNTS: + candidates = [bin_directory / (name + '.xctest') / 'Contents/MacOS' / name, + bin_directory / (name + '.xctest'), + bin_directory / (name + '-test-runner')] + found = [p for p in candidates if p.is_file()] + if len(found) != 1: + raise ValueError('Missing or ambiguous instrumented executable: ' + name) + result[name] = found[0] + return result + +def validate_export(export, root=ROOT, policy=DISCOVERY): + """Validate the executable source denominator before calculating percentages.""" + expected = {p.relative_to(root).as_posix() for p in (root / 'Sources').rglob('*.swift')} + unmapped = set(policy['coverage_unmapped']) + generated = set(policy['coverage_generated']) + if not unmapped <= expected or not generated <= expected: + raise ValueError('Coverage classification references absent source') + mapped = {} + for data in export['data']: + for record in data['files']: + file = pathlib.Path(record['filename']).resolve() + try: + name = file.relative_to(root.resolve()).as_posix() + except ValueError: + continue # SDK/macro code is outside the library denominator. + if not name.startswith('Sources/'): + continue + if name not in expected or name in mapped: + raise ValueError('Unexpected or duplicate library mapping: ' + name) + for kind in ['lines', 'regions', 'functions', 'branches', 'mcdc']: + count = record['summary'][kind]['count'] + covered = record['summary'][kind]['covered'] + if type(count) is not int or type(covered) is not int or not 0 <= covered <= count: + raise ValueError('Invalid coverage counter: ' + name + '/' + kind) + mapped[name] = record['summary'] + missing = expected - unmapped - set(mapped) + if missing: + raise ValueError('Incomplete library export: ' + ', '.join(sorted(missing))) + if unmapped & set(mapped): + raise ValueError('Unmapped classification changed; review compiler/source mapping') + + def totals(names): + result = {} + for kind in ['lines', 'regions', 'functions', 'branches', 'mcdc']: + count = sum(mapped[name][kind]['count'] for name in names) + covered = sum(mapped[name][kind]['covered'] for name in names) + result[kind] = dict(count=count, covered=covered, + percent=100*covered/count if count else None) + return result + + modules = {} + for test_target in policy['root_functions']: + module = test_target.removesuffix('Tests') + names = {name for name in mapped if name.startswith('Sources/' + module + '/')} + if not names or not totals(names)['regions']['count']: + raise ValueError('Missing executable module coverage: ' + module) + modules[module] = dict(mapped_files=len(names), authored=totals(names - generated), + generated=totals(names & generated)) + return dict(modules=modules, authored=totals(set(mapped)-generated), + generated=totals(set(mapped)&generated), + files={name:mapped[name] for name in sorted(mapped)}, + unmapped=policy['coverage_unmapped'], + branch_counters_available=bool(totals(mapped)['branches']['count']), + mcdc_counters_available=bool(totals(mapped)['mcdc']['count'])) + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output', required=True) + parser.add_argument('--verify-export', help='Check an existing export without compiling/running tests') + args = parser.parse_args() + output = ROOT / args.output + output.mkdir(parents=True, exist_ok=True) + receipt = dict(schema_version=1, status='running', commands=[], started_utc=datetime.datetime.now(datetime.timezone.utc).isoformat()) + def save(): + (output / 'coverage.json').write_text(json.dumps(receipt, indent=2)+'\n') + def run(command, name, binary=False): + start = time.monotonic() + log = output / name + with log.open('wb') as stream: + process = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT, timeout=1800) + receipt['commands'].append(dict(command=[str(c).replace(str(ROOT), '.') for c in command], + exit_code=process.returncode, seconds=time.monotonic()-start, log=name, sha256=digest(log))) + save() + if process.returncode: + raise ValueError('Command failed: ' + ' '.join(map(str, command))) + return log.read_bytes() if binary else log.read_text() + try: + if args.verify_export: + export = json.loads((ROOT / args.verify_export).read_text()) + receipt.update(validate_export(export), scope='existing export verification only', status='passed') + else: + inputs = [{'path':p.relative_to(ROOT).as_posix(), 'sha256':digest(p)} for p in files()] + receipt['source_inputs'] = inputs + receipt['source_manifest_sha256'] = hashlib.sha256(json.dumps(inputs, sort_keys=True).encode()).hexdigest() + receipt['toolchain'] = run(['swift', '--version'], 'compiler.log').strip() + if platform.system() == 'Darwin': + llvm = ['xcrun', 'llvm-cov'] + else: + tool = pathlib.Path(shutil.which('swift')).resolve().parent / 'llvm-cov' + if not tool.is_file(): + raise ValueError('Swift toolchain llvm-cov required') + llvm = [str(tool)] + receipt['llvm'] = run(llvm + ['--version'], 'llvm.log').strip() + tests = run(['swift', 'test', '--enable-code-coverage', '--disable-xctest'], 'tests.log') + if not check_root(tests): + raise ValueError('Exact root discovery/parameterized cardinality changed') + receipt.update(root_counts=ROOT_COUNTS, parameterized_invocations=PARAMETER_INVOCATIONS) + bin_directory = pathlib.Path(run(['swift', 'build', '--show-bin-path'], 'bin-path.log').strip()) + objects = executable_objects(bin_directory) + coverage_path = pathlib.Path(run(['swift', 'test', '--show-codecov-path'], 'profile-path.log').strip()) + profile = coverage_path.parent / 'default.profdata' + if not profile.is_file(): + raise ValueError('Merged coverage profile missing') + shutil.copy2(profile, output / 'default.profdata') + receipt['profile_sha256'] = digest(profile) + receipt['binaries'] = {name:dict(path=p.relative_to(ROOT).as_posix(), sha256=digest(p)) for name,p in objects.items()} + first, *others = objects.values() + command = llvm + ['export', str(first), '-instr-profile=' + str(profile)] + for other in others: + command.extend(['-object', str(other)]) + raw = run(command, 'all-modules.raw.json', binary=True) + export = json.loads(raw) + receipt.update(validate_export(export)) + try: + validate_export(json.loads(coverage_path.read_text())) + receipt['default_export'] = dict(status='complete') + except ValueError as error: + receipt['default_export'] = dict(status='incomplete', error=str(error)) + # This actual exporter probe must omit the schema-only executable; + # regex code alone is also linked in schema tests and is not a useful omission. + omitted = [p for name,p in objects.items() if name != 'JSONSchemaTests'] + negative_command = llvm + ['export', str(omitted[0]), '-instr-profile=' + str(profile)] + for other in omitted[1:]: + negative_command.extend(['-object', str(other)]) + negative = run(negative_command, 'omitted-schema.raw.json', binary=True) + try: + validate_export(json.loads(negative)) + except ValueError as error: + if 'Incomplete library export:' not in str(error): + raise + receipt['omitted_binary_probe'] = dict(status='rejected', omitted='JSONSchemaTests', reason=str(error)) + else: + raise ValueError('Harness accepted export missing the schema executable') + # Public summaries and normalized export contain no local source path. + for data in export['data']: + for record in data['files']: + record['filename'] = record['filename'].replace(str(ROOT), '.') + for function in data.get('functions', []): + function['filenames'] = [p.replace(str(ROOT), '.') for p in function['filenames']] + (output / 'all-modules.json').write_text(json.dumps(export)+'\n') + receipt['raw_export_sha256'] = hashlib.sha256(raw).hexdigest() + receipt['normalized_export_sha256'] = digest(output / 'all-modules.json') + if inputs != [{'path':p.relative_to(ROOT).as_posix(), 'sha256':digest(p)} for p in files()]: + raise ValueError('Source changed during instrumented run') + receipt.update(status='passed', source_inputs_unchanged=True, + interpretation='Execution lines/regions, not assertion completeness; unavailable branch counters are null, generated tables are separate.') + except (OSError, ValueError, KeyError, TypeError, subprocess.TimeoutExpired) as error: + receipt.update(status='failed', error=str(error).replace(str(ROOT), '.')) + receipt['finished_utc'] = datetime.datetime.now(datetime.timezone.utc).isoformat() + save() + print(json.dumps({k:receipt[k] for k in ['status', 'error', 'root_counts', 'parameterized_invocations', 'authored'] if k in receipt}, indent=2)) + return 0 if receipt['status'] == 'passed' else 1 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/Scripts/test_discovery.py b/Scripts/test_discovery.py new file mode 100644 index 0000000..c60805d --- /dev/null +++ b/Scripts/test_discovery.py @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: Apache-2.0 +"""One exact discovery contract for native, safety and WASI qualification.""" +import json +import pathlib +import re + +ROOT = pathlib.Path(__file__).resolve().parents[1] +DISCOVERY = json.loads((ROOT / 'Documentation/Qualification/TestDiscovery.json').read_text()) +ROOT_COUNTS = DISCOVERY['root_functions'] +PARAMETER_INVOCATIONS = DISCOVERY['parameterized_invocations'] + +def check_root(log): + counts = [int(n) for n in re.findall(r'Test run with (\d+) tests? in[^\n]*passed', log)] + invocations = sum(int(n) for n in re.findall(r'with (\d+) test cases passed', log)) + return sorted(counts) == sorted(ROOT_COUNTS.values()) and invocations == PARAMETER_INVOCATIONS diff --git a/Scripts/test_qualification_harness.py b/Scripts/test_qualification_harness.py new file mode 100644 index 0000000..d24e2a5 --- /dev/null +++ b/Scripts/test_qualification_harness.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Negative coverage/discovery harness tests; never substitutes for Swift behavior.""" +import copy +import json +import pathlib +import tempfile +import unittest +from coverage import executable_objects, validate_export +from test_discovery import ROOT_COUNTS, PARAMETER_INVOCATIONS, check_root +from ci import check_result + +class HarnessTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = pathlib.Path(self.temporary.name) + self.names = ['Sources/JSON/A.swift', 'Sources/JSONPatch/B.swift'] + self.policy = dict(root_functions={'JSONTests':1, 'JSONPatchTests':1}, + coverage_unmapped={'Sources/JSON/Declaration.swift':'declaration'}, coverage_generated=[]) + for name in [*self.names, *self.policy['coverage_unmapped']]: + source = self.root / name; source.parent.mkdir(parents=True, exist_ok=True) + source.write_text('// isolated harness fixture\n') + summary = {kind:dict(count=2, covered=1) for kind in ['lines','regions','functions']} + summary.update({kind:dict(count=0, covered=0) for kind in ['branches','mcdc']}) + self.export = dict(data=[dict(files=[dict(filename=str(self.root/name), summary=copy.deepcopy(summary)) for name in self.names])]) + + def test_complete_denominator_and_unavailable_counters(self): + result = validate_export(self.export, self.root, self.policy) + self.assertEqual(result['authored']['lines'], dict(count=4, covered=2, percent=50)) + self.assertIsNone(result['authored']['branches']['percent']) + self.assertFalse(result['branch_counters_available']) + self.assertEqual(set(result['modules']), {'JSON','JSONPatch'}) + + def test_omitted_module_is_not_a_higher_percentage(self): + self.export['data'][0]['files'].pop() + with self.assertRaisesRegex(ValueError, 'Incomplete library export'): + validate_export(self.export, self.root, self.policy) + + def test_duplicate_mapping_is_rejected(self): + self.export['data'][0]['files'].append(self.export['data'][0]['files'][0]) + with self.assertRaisesRegex(ValueError, 'duplicate library mapping'): + validate_export(self.export, self.root, self.policy) + + def test_unknown_library_mapping_is_rejected(self): + self.export['data'][0]['files'][0]['filename'] = str(self.root/'Sources/JSON/Old.swift') + with self.assertRaisesRegex(ValueError, 'Unexpected'): + validate_export(self.export, self.root, self.policy) + + def test_new_mapping_requires_review_of_unmapped_disposition(self): + entry = copy.deepcopy(self.export['data'][0]['files'][0]) + entry['filename'] = str(self.root/'Sources/JSON/Declaration.swift') + self.export['data'][0]['files'].append(entry) + with self.assertRaisesRegex(ValueError, 'classification changed'): + validate_export(self.export, self.root, self.policy) + + def test_impossible_counters_are_rejected(self): + for count,covered in [(1,2),(-1,0),(True,0),(2,0.5)]: + broken = copy.deepcopy(self.export) + broken['data'][0]['files'][0]['summary']['lines'].update(count=count,covered=covered) + with self.assertRaisesRegex(ValueError, 'Invalid coverage counter'): + validate_export(broken, self.root, self.policy) + + def test_stale_classification_cannot_exclude_a_missing_file(self): + (self.root/'Sources/JSON/Declaration.swift').unlink() + with self.assertRaisesRegex(ValueError, 'absent source'): + validate_export(self.export, self.root, self.policy) + + def test_exact_discovery_rejects_missing_duplicate_and_changed_parameters(self): + summaries = ['Test run with '+str(n)+' tests in 0 suites passed' for n in ROOT_COUNTS.values()] + cases = 'with '+str(PARAMETER_INVOCATIONS)+' test cases passed' + valid = '\n'.join([*summaries,cases]) + self.assertTrue(check_root(valid)) + self.assertTrue(check_result(['swift','test'],0,valid)[0]) + for broken in ['\n'.join([*summaries[:-1],cases]), valid+'\n'+summaries[0], valid.replace(cases,'with 1 test cases passed'), '']: + self.assertFalse(check_root(broken)) + self.assertFalse(check_result(['swift','test'],0,broken)[0]) + self.assertFalse(check_result(['swift','test'],1,valid)[0]) + + def test_darwin_executable_membership_is_exact(self): + for name in ROOT_COUNTS: + p = self.root / (name+'.xctest') / 'Contents/MacOS' / name + p.parent.mkdir(parents=True); p.touch() + self.assertEqual(set(executable_objects(self.root)),set(ROOT_COUNTS)) + p.unlink() + with self.assertRaisesRegex(ValueError,'Missing or ambiguous'): + executable_objects(self.root) + + def test_linux_runner_and_ambiguous_layout(self): + for name in ROOT_COUNTS: + (self.root/(name+'-test-runner')).touch() + self.assertEqual(set(executable_objects(self.root)),set(ROOT_COUNTS)) + (self.root/(name+'.xctest')).touch() + with self.assertRaisesRegex(ValueError,'Missing or ambiguous'): + executable_objects(self.root) + +if __name__ == '__main__': + result = unittest.TextTestRunner(verbosity=2).run(unittest.defaultTestLoader.loadTestsFromTestCase(HarnessTests)) + print('HARNESS_TESTS '+('passed' if result.wasSuccessful() else 'failed')+' cases='+str(result.testsRun)) + raise SystemExit(0 if result.wasSuccessful() else 1) diff --git a/Scripts/wasm_support.py b/Scripts/wasm_support.py index dde4ac8..339abce 100644 --- a/Scripts/wasm_support.py +++ b/Scripts/wasm_support.py @@ -3,11 +3,11 @@ """Qualify the full-language wasm32 WASI command target with materialized outcomes.""" import argparse, hashlib, json, os, pathlib, re, shutil, signal, subprocess, time from archive_candidate import files +from test_discovery import ROOT_COUNTS, PARAMETER_INVOCATIONS ROOT = pathlib.Path(__file__).resolve().parents[1] SDK = 'swift-6.4.0-RELEASE_wasm' -COUNTS = {'JSONTests': 104, 'JSONSchemaTests': 51, 'JSONRegexRuntimeTests': 12, - 'JSONStreamingTests': 14, 'JSONPatchTests': 11} +COUNTS = ROOT_COUNTS GROUPS = {'core': 6, 'resources': 1, 'schema': 2, 'streaming': 3} def main(): @@ -71,7 +71,7 @@ def invoke(engine, path, mode='command', arguments=(), repo=True, resources=True for name, count in COUNTS.items(): text = run('node-' + config + '-' + name, invoke('node', roots[name], 'tests'), tests(count)) invocations += sum(int(n) for n in re.findall(r'with (\d+) test cases passed', text)) - if invocations != 173: raise ValueError('Changed root parameterized discovery: ' + str(invocations)) + if invocations != PARAMETER_INVOCATIONS: raise ValueError('Changed root parameterized discovery: ' + str(invocations)) for package, name, scratch in [('Tools/WasmAudit', 'WasmAudit', '.build/wasm-audit-consumer'), ('Tools/Qualification', 'Qualification', '.build/wasm-qualification'), ('Tools/MCPReadiness', 'MCPReadiness', '.build/wasm-readiness')]: diff --git a/Tests/JSONTests/CodableContainerTests.swift b/Tests/JSONTests/CodableContainerTests.swift new file mode 100644 index 0000000..bedd41b --- /dev/null +++ b/Tests/JSONTests/CodableContainerTests.swift @@ -0,0 +1,413 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation +import Testing +import JSON + +private enum Field: String, CodingKey { case object, array, value, named, superValue = "super", missing, a, b } +private enum ProbeError: Error, Equatable { case expectedFailure, caller } + +private func capture(_ body: () throws -> Void) -> any Error { + do { try body(); Issue.record("Expected operation to fail"); return ProbeError.expectedFailure } + catch { return error } +} +private func path(_ context: DecodingError.Context, _ names: [String], _ indices: [Int?]) { + #expect(context.codingPath.map(\.stringValue) == names) + #expect(context.codingPath.map(\.intValue) == indices) +} +private func resource(_ error: any Error, metric: JSONResourceMetric, limit: UInt64, + attempted: UInt64, domain: JSONOperationDomain = .coding) throws { + let failure = try #require(error as? JSONFailure) + #expect(failure.category == .resource && failure.code == .limitExceeded) + #expect(failure.location == nil) + let violation = try #require(failure.resource) + #expect(violation.domain == domain && violation.metric == metric) + #expect(violation.limit == limit && violation.attemptedUsage == attempted && !violation.overflow) +} +private func wrongKind(_ error: any Error, names: [String], indices: [Int?]) { + guard case DecodingError.typeMismatch(_, let context) = error else { + Issue.record("Expected typeMismatch, received \(type(of: error))"); return + } + path(context, names, indices) + #expect(context.underlyingError as? JSONFailure == JSONFailure(.typeMismatch, code: .wrongKind)) +} + +private struct KeyedTree: Codable, Equatable { + let number: Int + let text: String + init() { number = 7; text = "e\u{301}" } + func encode(to encoder: any Encoder) throws { + var root = encoder.container(keyedBy: Field.self) + var object = root.nestedContainer(keyedBy: Field.self, forKey: .object) + #expect(object.codingPath.map(\.stringValue) == ["object"]) + try object.encode(number, forKey: .value) + var array = root.nestedUnkeyedContainer(forKey: .array) + #expect(array.codingPath.map(\.stringValue) == ["array"]) + try array.encodeNil(); try array.encode(text) + let named = root.superEncoder(forKey: .named) + #expect(named.codingPath.map(\.stringValue) == ["named"]) + var namedValue = named.singleValueContainer(); try namedValue.encode(true) + let inherited = root.superEncoder() + #expect(inherited.codingPath.map(\.stringValue) == ["super"]) + var inheritedValue = inherited.singleValueContainer(); try inheritedValue.encode(2) + } + init(from decoder: any Decoder) throws { + let root = try decoder.container(keyedBy: Field.self) + let object = try root.nestedContainer(keyedBy: Field.self, forKey: .object) + #expect(object.codingPath.map(\.stringValue) == ["object"]) + number = try object.decode(Int.self, forKey: .value) + var array = try root.nestedUnkeyedContainer(forKey: .array) + #expect(array.codingPath.map(\.stringValue) == ["array"]) + #expect(try array.decodeNil()); text = try array.decode(String.self) + #expect(array.isAtEnd && array.currentIndex == 2 && array.count == 2) + let named = try root.superDecoder(forKey: .named) + #expect(named.codingPath.map(\.stringValue) == ["named"]) + #expect(try named.singleValueContainer().decode(Bool.self)) + let inherited = try root.superDecoder() + #expect(inherited.codingPath.map(\.stringValue) == ["super"]) + #expect(try inherited.singleValueContainer().decode(Int.self) == 2) + } +} + +@Test func qaCodableKeyedNestedAndNamedSuperContainersMatchGolden() throws { + let golden = try JSONParser().parse(#"{"object":{"value":7},"array":[null,"é"],"named":true,"super":2}"#) + let encoded = try JSONValueEncoder().encode(KeyedTree()) + #expect(encoded == golden) + #expect(encoded.objectValue?.map { $0.key.string } == ["object", "array", "named", "super"]) + #expect(try JSONValueDecoder().decode(KeyedTree.self, from: golden) == KeyedTree()) +} + +private struct UnkeyedTree: Codable { + init() {} + func encode(to encoder: any Encoder) throws { + var root = encoder.unkeyedContainer() + #expect(root.count == 0) + try root.encodeNil() + var object = root.nestedContainer(keyedBy: Field.self) + #expect(object.codingPath.map(\.stringValue) == ["1"]) + #expect(object.codingPath.map(\.intValue) == [1]) + try object.encode(7, forKey: .value) + var array = root.nestedUnkeyedContainer(); try array.encode(true) + let inherited = root.superEncoder() + #expect(inherited.codingPath.map(\.intValue) == [3]) + var value = inherited.singleValueContainer(); try value.encode("base") + #expect(root.count == 4) + } + init(from decoder: any Decoder) throws { + var root = try decoder.unkeyedContainer() + #expect(root.count == 4 && root.currentIndex == 0 && !root.isAtEnd) + #expect(try root.decodeNil()) + let object = try root.nestedContainer(keyedBy: Field.self) + #expect(object.codingPath.map(\.intValue) == [1]) + #expect(try object.decode(Int.self, forKey: .value) == 7) + var array = try root.nestedUnkeyedContainer() + #expect(array.codingPath.map(\.intValue) == [2]) + #expect(try array.decode(Bool.self) && array.isAtEnd) + let inherited = try root.superDecoder() + #expect(inherited.codingPath.map(\.intValue) == [3]) + #expect(try inherited.singleValueContainer().decode(String.self) == "base") + #expect(root.isAtEnd && root.currentIndex == 4) + } +} +@Test func qaCodableUnkeyedNestedAndSuperContainersMatchGolden() throws { + let golden = try JSONParser().parse(#"[null,{"value":7},[true],"base"]"#) + #expect(try JSONValueEncoder().encode(UnkeyedTree()) == golden) + _ = try JSONValueDecoder().decode(UnkeyedTree.self, from: golden) +} + +private struct EmptyContainers: Encodable { + let mode: Int + func encode(to encoder: any Encoder) throws { + switch mode { + case 0: _ = encoder.container(keyedBy: Field.self) + case 1: _ = encoder.unkeyedContainer() + case 2: + var c = encoder.container(keyedBy: Field.self) + _ = c.nestedContainer(keyedBy: Field.self, forKey: .object) + _ = c.nestedUnkeyedContainer(forKey: .array) + default: + var c = encoder.unkeyedContainer() + _ = c.nestedContainer(keyedBy: Field.self); _ = c.nestedUnkeyedContainer() + } + } +} +@Test func qaCodableEmptyContainersAreValuesWithoutPlaceholderNulls() throws { + for (mode, text) in ["{}", "[]", #"{"object":{},"array":[]}"#, "[{},[]]"].enumerated() { + #expect(try JSONValueEncoder().encode(EmptyContainers(mode: mode)) == JSONParser().parse(text)) + } +} + +private struct ReusedKeyed: Encodable { + func encode(to encoder: any Encoder) throws { + var first = encoder.container(keyedBy: Field.self) + try first.encode(1, forKey: .a); try first.encode(2, forKey: .b) + var again = encoder.container(keyedBy: Field.self) + try again.encode(3, forKey: .a) + #expect(first.codingPath.isEmpty && again.codingPath.isEmpty) + } +} +@Test func qaCodableReusedKeyedContainerReplacesWithoutReordering() throws { + let value = try JSONValueEncoder().encode(ReusedKeyed()) + #expect(try JSONWriter().writeString(value) == #"{"a":3,"b":2}"#) +} + +private struct SingleBox: Codable { + let value: T + init(_ value: T) { self.value = value } + func encode(to encoder: any Encoder) throws { var c = encoder.singleValueContainer(); try c.encode(value) } + init(from decoder: any Decoder) throws { value = try decoder.singleValueContainer().decode(T.self) } +} +private func primitive(_ value: T, _ golden: String) throws { + let expected = try JSONParser().parse(golden) + #expect(try JSONValueEncoder().encode(SingleBox(value)) == expected) + #expect(try JSONValueDecoder().decode(SingleBox.self, from: expected).value == value) +} +@Test func qaCodableSingleValuePrimitiveOverloadsHaveIndependentExpectedValues() throws { + try primitive(true, "true"); try primitive("e\u{301}", #""é""#) + try primitive(Double(1.5), "1.5"); try primitive(Float(0.5), "0.5") + // These vectors also compile and execute on wasm32: no host-width Int.max assumption. + try primitive(Int(-123), "-123"); try primitive(Int8.min, "-128") + try primitive(Int16.min, "-32768"); try primitive(Int32.min, "-2147483648") + try primitive(Int64.min, "-9223372036854775808") + try primitive(UInt(123), "123"); try primitive(UInt8.max, "255") + try primitive(UInt16.max, "65535"); try primitive(UInt32.max, "4294967295") + try primitive(UInt64.max, "18446744073709551615") +} +private struct NilSingle: Codable { + init() {} + func encode(to encoder: any Encoder) throws { var c = encoder.singleValueContainer(); try c.encodeNil() } + init(from decoder: any Decoder) throws { #expect(try decoder.singleValueContainer().decodeNil()) } +} +@Test func qaCodableSingleNilAndExactGenericPayloadPreserveKinds() throws { + #expect(try JSONValueEncoder().encode(NilSingle()) == .null) + _ = try JSONValueDecoder().decode(NilSingle.self, from: .null) + for text in ["null", "false", #""x""#, "9007199254740993", "[1]", #"{"é":1,"é":2}"#] { + let value = try JSONParser().parse(text) + #expect(try JSONValueEncoder().encode(SingleBox(value)) == value) + #expect(try JSONValueDecoder().decode(SingleBox.self, from: value).value == value) + } +} + +private struct CursorRecovery: Decodable { + init(from decoder: any Decoder) throws { + var c = try decoder.unkeyedContainer() + #expect(try c.decodeNil() && c.currentIndex == 1) + #expect(try !c.decodeNil() && c.currentIndex == 1) + wrongKind(capture { _ = try c.decode(Int.self) }, names: ["1"], indices: [1]) + #expect(c.currentIndex == 1) + #expect(try c.decode(String.self) == "text" && c.currentIndex == 2) + #expect(try c.decode(Int.self) == 7 && c.currentIndex == 3 && c.isAtEnd) + let error = capture { _ = try c.decodeNil() } + guard case DecodingError.valueNotFound(_, let context) = error else { Issue.record("Expected exhausted container"); return } + path(context, [], []); #expect(context.underlyingError == nil) + #expect(c.currentIndex == 3) + } +} +@Test func qaCodableUnkeyedNilIndexAndOrdinaryErrorsRemainRecoverable() throws { + _ = try JSONValueDecoder().decode(CursorRecovery.self, from: JSONParser().parse(#"[null,"text",7]"#)) +} + +private struct NestedInteger: Decodable { + init(from decoder: any Decoder) throws { + let c = try decoder.container(keyedBy: Field.self) + let object = try c.nestedContainer(keyedBy: Field.self, forKey: .object) + var array = try object.nestedUnkeyedContainer(forKey: .array) + let item = try array.nestedContainer(keyedBy: Field.self) + _ = try item.decode(Int8.self, forKey: .value) + } +} +@Test func qaCodableNestedWrongKindHasCompleteTypedCodingPath() throws { + let value = try JSONParser().parse(#"{"object":{"array":[{"value":"bad"}]}}"#) + wrongKind(capture { _ = try JSONValueDecoder().decode(NestedInteger.self, from: value) }, + names: ["object", "array", "0", "value"], indices: [nil, nil, 0, nil]) +} +@Test func qaCodableNestedMissingKeyHasParentPathAndExactMissingKey() throws { + let value = try JSONParser().parse(#"{"object":{"array":[{}]}}"#) + let error = capture { _ = try JSONValueDecoder().decode(NestedInteger.self, from: value) } + guard case DecodingError.keyNotFound(let key, let context) = error else { Issue.record("Expected keyNotFound"); return } + #expect(key.stringValue == "value" && key.intValue == nil) + path(context, ["object", "array", "0"], [nil, nil, 0]); #expect(context.underlyingError == nil) +} +@Test func qaCodableNestedInexactIntegerHasConversionCauseAndFullPath() throws { + for token in ["128", "-129", "1.5"] { + let value = try JSONParser().parse("{\"object\":{\"array\":[{\"value\":\(token)}]}}") + let error = capture { _ = try JSONValueDecoder().decode(NestedInteger.self, from: value) } + guard case DecodingError.dataCorrupted(let context) = error else { Issue.record("Expected dataCorrupted"); continue } + path(context, ["object", "array", "0", "value"], [nil, nil, 0, nil]) + #expect(context.underlyingError as? JSONFailure == JSONFailure(.conversion, code: .inexactConversion)) + } +} +@Test func qaCodableNestedContainerWrongKindIdentifiesItsBoundary() throws { + for (text, names, indices): (String, [String], [Int?]) in [ + (#"{"object":[]}"#, ["object"], [nil]), + (#"{"object":{"array":{}}}"#, ["object", "array"], [nil, nil]), + (#"{"object":{"array":[false]}}"#, ["object", "array", "0"], [nil, nil, 0]) + ] { + let value = try JSONParser().parse(text) + wrongKind(capture { _ = try JSONValueDecoder().decode(NestedInteger.self, from: value) }, names: names, indices: indices) + } +} + +private struct KeyEnumeration: Decodable { + init(from decoder: any Decoder) throws { + let c = try decoder.container(keyedBy: Field.self) + #expect(c.allKeys.map(\.stringValue) == ["value"]) + #expect(c.contains(.value) && !c.contains(.missing)) + #expect(try c.decodeNil(forKey: .value)) + let error = capture { _ = try c.decodeNil(forKey: .missing) } + guard case DecodingError.keyNotFound(let key, let context) = error else { Issue.record("Expected missing key"); return } + #expect(key.stringValue == "missing"); path(context, [], []) + } +} +@Test func qaCodableKeyEnumerationUnknownKeysAndNullPresenceAreDistinct() throws { + _ = try JSONValueDecoder().decode(KeyEnumeration.self, from: JSONParser().parse(#"{"ignored":1,"value":null}"#)) +} + +private struct FailingKeyedChild: Encodable { + let mode: Int + func encode(to encoder: any Encoder) throws { + var c = encoder.container(keyedBy: Field.self); try c.encode(1, forKey: .a) + switch mode { + case 0: _ = c.nestedContainer(keyedBy: Field.self, forKey: .b) + case 1: _ = c.nestedUnkeyedContainer(forKey: .b) + case 2: _ = c.superEncoder(forKey: .b) + default: _ = c.superEncoder() + } + try resource(capture { try c.encode(3, forKey: .a) }, metric: .objectMembers, limit: 1, attempted: 2) + // Caller deliberately catches the refusal. The invocation must still fail. + } +} +@Test func qaCodableAllNonthrowingKeyedChildRequestsLatchTheFirstFailure() throws { + var limits = JSONLimits.balanced; limits[.objectMembers] = 1 + for mode in 0..<4 { + try resource(capture { _ = try JSONValueEncoder(options: .init(limits: limits)).encode(FailingKeyedChild(mode: mode)) }, + metric: .objectMembers, limit: 1, attempted: 2) + } +} +private struct FailingUnkeyedChild: Encodable { + let mode: Int + func encode(to encoder: any Encoder) throws { + var c = encoder.unkeyedContainer(); try c.encode(1) + switch mode { + case 0: _ = c.nestedContainer(keyedBy: Field.self) + case 1: _ = c.nestedUnkeyedContainer() + default: _ = c.superEncoder() + } + try resource(capture { try c.encode(2) }, metric: .arrayItems, limit: 1, attempted: 2) + } +} +@Test func qaCodableAllNonthrowingUnkeyedChildRequestsLatchTheFirstFailure() throws { + var limits = JSONLimits.balanced; limits[.arrayItems] = 1 + for mode in 0..<3 { + try resource(capture { _ = try JSONValueEncoder(options: .init(limits: limits)).encode(FailingUnkeyedChild(mode: mode)) }, + metric: .arrayItems, limit: 1, attempted: 2) + } +} + +private struct IgnoreValue: Decodable { init(from decoder: any Decoder) throws {} } +private struct ContainsFailure: Decodable { + init(from decoder: any Decoder) throws { + let c = try decoder.container(keyedBy: Field.self) + #expect(!c.contains(.value)) + let first = try #require(capture { _ = try c.decode(Int.self, forKey: .value) } as? JSONFailure) + #expect(first.resource?.metric == .workUnits) + #expect(!c.contains(.missing)) + #expect(capture { _ = try c.decodeNil(forKey: .value) } as? JSONFailure == first) + } +} +private struct AllKeysFailure: Decodable { + init(from decoder: any Decoder) throws { + let c = try decoder.container(keyedBy: Field.self) + #expect(c.allKeys.isEmpty) + let first = try #require(capture { _ = try c.decode(Int.self, forKey: .value) } as? JSONFailure) + #expect(first.resource?.metric == .workUnits) + #expect(c.allKeys.isEmpty) + #expect(capture { _ = try c.superDecoder() } as? JSONFailure == first) + } +} +@Test func qaCodableContainsCannotTurnExhaustionIntoAbsenceOrSuccess() throws { + let value = try JSONParser().parse(#"{"value":7}"#) + let admitted = try JSONValueDecoder().decodeWithReport(IgnoreValue.self, from: value).statistics[.workUnits] + var limits = JSONLimits.balanced; limits[.workUnits] = admitted + try resource(capture { _ = try JSONValueDecoder(options: .init(limits: limits)).decode(ContainsFailure.self, from: value) }, + metric: .workUnits, limit: admitted, attempted: admitted + 1) +} +@Test func qaCodableAllKeysCannotTurnExhaustionIntoAnEmptySuccessfulModel() throws { + let value = try JSONParser().parse(#"{"value":7}"#) + let admitted = try JSONValueDecoder().decodeWithReport(IgnoreValue.self, from: value).statistics[.workUnits] + var limits = JSONLimits.balanced; limits[.workUnits] = admitted + try resource(capture { _ = try JSONValueDecoder(options: .init(limits: limits)).decode(AllKeysFailure.self, from: value) }, + metric: .workUnits, limit: admitted, attempted: admitted + 1) +} + +private struct ConflictContainers: Encodable { + let startsKeyed: Bool + func encode(to encoder: any Encoder) throws { + if startsKeyed { _ = encoder.container(keyedBy: Field.self); _ = encoder.unkeyedContainer() } + else { _ = encoder.unkeyedContainer(); _ = encoder.container(keyedBy: Field.self) } + } +} +@Test func qaCodableConflictingNonthrowingRootContainersDoNotReturnAValue() throws { + for keyed in [true, false] { + let error = capture { _ = try JSONValueEncoder().encode(ConflictContainers(startsKeyed: keyed)) } + guard case EncodingError.invalidValue(_, let context) = error else { Issue.record("Expected conflicting-container invalidValue"); continue } + #expect(context.codingPath.isEmpty && context.underlyingError == nil) + } +} +private struct NoEncodedValue: Encodable { func encode(to encoder: any Encoder) throws {} } +@Test func qaCodableUnsetRootReportsAnEncodingFailure() throws { + let error = capture { _ = try JSONValueEncoder().encode(NoEncodedValue()) } + guard case EncodingError.invalidValue(let value, let context) = error else { Issue.record("Expected unset invalidValue"); return } + #expect(value as? String == "unset" && context.codingPath.isEmpty) +} + +@Test func qaCodableNestedEncodingSharesOneInclusiveWorkLedger() throws { + let baseline = try JSONValueEncoder().encodeWithReport(KeyedTree()) + let used = baseline.statistics[.workUnits] + #expect(used > 0 && baseline.statistics.domain == .coding && baseline.statistics.accountingVersion == 1) + var limits = JSONLimits.balanced; limits[.workUnits] = used + let bounded = try JSONValueEncoder(options: .init(limits: limits)).encodeWithReport(KeyedTree()) + #expect(bounded.result == baseline.result && bounded.statistics[.workUnits] == used) + limits[.workUnits] = used - 1 + try resource(capture { _ = try JSONValueEncoder(options: .init(limits: limits)).encode(KeyedTree()) }, + metric: .workUnits, limit: used - 1, attempted: used) +} +@Test func qaCodableNestedDecodingSharesOneInclusiveWorkLedger() throws { + let golden = try JSONParser().parse(#"{"object":{"value":7},"array":[null,"é"],"named":true,"super":2}"#) + let baseline = try JSONValueDecoder().decodeWithReport(KeyedTree.self, from: golden) + let used = baseline.statistics[.workUnits] + #expect(used > 0 && baseline.statistics.domain == .coding && baseline.statistics.accountingVersion == 1) + var limits = JSONLimits.balanced; limits[.workUnits] = used + let bounded = try JSONValueDecoder(options: .init(limits: limits)).decodeWithReport(KeyedTree.self, from: golden) + #expect(bounded.result == KeyedTree() && bounded.statistics[.workUnits] == used) + limits[.workUnits] = used - 1 + try resource(capture { _ = try JSONValueDecoder(options: .init(limits: limits)).decode(KeyedTree.self, from: golden) }, + metric: .workUnits, limit: used - 1, attempted: used) +} + +private struct CancelDuringContainer: Encodable { + func encode(to encoder: any Encoder) throws { + var c = encoder.container(keyedBy: Field.self) + withUnsafeCurrentTask { $0?.cancel() } + _ = c.nestedUnkeyedContainer(forKey: .array) + #expect(capture { try c.encode(1, forKey: .value) } is CancellationError) + } +} +@Test func qaCodableCaughtContainerCancellationRemainsTerminal() async { + let task = Task { capture { _ = try JSONValueEncoder().encode(CancelDuringContainer()) } is CancellationError } + #expect(await task.value) +} + +private struct ThrowCaller: Encodable { func encode(to encoder: any Encoder) throws { throw ProbeError.caller } } +@Test func qaCodableCallerErrorRemainsDistinctFromOwnedFailures() throws { + let error = capture { _ = try JSONValueEncoder().encode(ThrowCaller()) } + #expect(error as? ProbeError == .caller) + // A fresh independent invocation remains usable after caller or owned failure. + #expect(try JSONValueEncoder().encode(SingleBox(7)) == 7) +} + +@Test func qaCodableFloatRangeFailureCarriesNestedPath() throws { + let value = try JSONParser().parse("[1e1000]") + let error = capture { _ = try JSONValueDecoder().decode([Float].self, from: value) } + guard case DecodingError.dataCorrupted(let context) = error else { Issue.record("Expected finite-range dataCorrupted"); return } + path(context, ["0"], [0]); #expect(context.underlyingError == nil) +} From 21d5428793c4a972f3cfbe34471739784d21c2f0 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Wed, 14 Oct 2026 05:54:40 +0200 Subject: [PATCH 03/24] docs: record first-wave native evidence and remaining test scope --- .../Release1.2.1/AcceptanceMatrix.json | 45 +- .../Qualification/Release1.2.1/README.md | 4 + .../Qualification/Release1.2.1/Wave1.md | 34 +- .../Release1.2.1/Wave1Outcomes.json | 564 ++++++++++++++++++ 4 files changed, 627 insertions(+), 20 deletions(-) create mode 100644 Documentation/Qualification/Release1.2.1/Wave1Outcomes.json diff --git a/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json b/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json index 7dcd8d4..0317d8c 100644 --- a/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json +++ b/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json @@ -3,14 +3,14 @@ "source_version": "1.2.0", "source_commit": "25fb04f747e583209b4c4511abf39e11ef5b4f80", "target_version": "1.2.1", - "status": "plan_only", + "status": "first_wave_native_qualified_remaining_plan", "rows": [ { "id": "HAR-01", "work_package": "T00", "module": "All", "topic": "Complete coverage export", - "status": "planned_extension", + "status": "native_qualified_linux_execution_pending", "existing_evidence": [ "fresh 192/173 root coverage run", "default export omitted schema/regex/streaming mappings" @@ -21,14 +21,15 @@ "all five implementation modules present", "all exact discoveries retained", "branch/MC/DC marked unavailable" - ] + ], + "first_wave_evidence": "Wave1Outcomes.json" }, { "id": "HAR-02", "work_package": "T00", "module": "All", "topic": "Unused code and evidence denominator", - "status": "planned_extension", + "status": "retained_pending_separate_use_or_removal_review", "existing_evidence": [ "JSONOperationContext/JSONFingerprint declaration-only reference scan and 0% coverage" ], @@ -37,14 +38,15 @@ "acceptance": [ "no artificial test-only callers", "cleanup separate from tests-only scope" - ] + ], + "first_wave_evidence": "Wave1Outcomes.json" }, { "id": "CORE-01", "work_package": "T01", "module": "JSON", "topic": "Numeric and primitive type matrix", - "status": "planned_extension", + "status": "partially_native_qualified_explicit_primitive_subset", "existing_evidence": [ "machineIntegerBoundaries", "boundedBinaryBridgePreservesLongHalfwayAndSubnormalTails", @@ -56,7 +58,8 @@ "exact versus rounded distinction", "error category/code and path", "native Int/WASI32 scope separate" - ] + ], + "first_wave_evidence": "Wave1Outcomes.json" }, { "id": "CORE-02", @@ -83,7 +86,7 @@ "work_package": "T01", "module": "JSON", "topic": "Explicit native container paths", - "status": "planned_extension", + "status": "native_qualified_explicit_container_subset", "existing_evidence": [ "nativeCodingUsesValuesAndComposedBytes", "superCodingAndNonSendableModelsWorkSynchronously" @@ -94,14 +97,15 @@ "exact fields and order where contracted", "EOF/wrong-kind coding paths", "successful roundtrip plus independent expected representation" - ] + ], + "first_wave_evidence": "Wave1Outcomes.json" }, { "id": "COD-02", "work_package": "T01", "module": "JSON", "topic": "Latched nonthrowing Codable operations", - "status": "planned_extension", + "status": "native_qualified_first_wave", "existing_evidence": [ "codingKeyEnumerationRejectsChangedScalarSpelling", "readAccessSharedProjectionLimitsAndTerminalAccounting", @@ -114,7 +118,8 @@ "ordinary/caller errors correctly classified", "work not refunded", "no result certified after failed ownership" - ] + ], + "first_wave_evidence": "Wave1Outcomes.json" }, { "id": "COD-03", @@ -158,7 +163,7 @@ "work_package": "T02", "module": "JSON", "topic": "Source and structured error fidelity", - "status": "planned_extension", + "status": "partially_native_qualified_codable_paths_and_causes", "existing_evidence": [ "sourceSpansUseTypedPathsAndInvalidateOnEdit", "qaMappedScalarPathsUseUTF8ByteCoordinates", @@ -171,7 +176,8 @@ "overflow attemptedUsage nil", "source replacement invalidates mapping", "caller description not traversed" - ] + ], + "first_wave_evidence": "Wave1Outcomes.json" }, { "id": "DIA-02", @@ -216,7 +222,7 @@ "work_package": "T02", "module": "All", "topic": "Terminal work/cancellation and ownership", - "status": "planned_extension", + "status": "partially_native_qualified_codable_terminal_behavior", "existing_evidence": [ "retiredWorkOverflowIsAtomicAndTerminal", "cancellationReachesOwnedWork", @@ -229,7 +235,8 @@ "no later owned work/handoffs after terminal error", "consumed work retained", "caller work excluded" - ] + ], + "first_wave_evidence": "Wave1Outcomes.json" }, { "id": "SCH-01", @@ -475,7 +482,7 @@ "work_package": "T07", "module": "All", "topic": "Actual platform/safety/consumer closure", - "status": "planned_extension", + "status": "native_first_wave_qualified_other_platform_execution_pending", "existing_evidence": [ "1.2.0 Native/Linux seven passed jobs", "65 WASI commands with explicit per-host scopes" @@ -488,7 +495,8 @@ "WASI sanitizers unavailable not passes", "physical iPad deferred", "no 1.3 features in patch" - ] + ], + "first_wave_evidence": "Wave1Outcomes.json" } ], "proposed_targets": { @@ -508,5 +516,6 @@ "T05", "T06", "T07" - ] + ], + "first_wave_evidence": "Wave1Outcomes.json" } diff --git a/Documentation/Qualification/Release1.2.1/README.md b/Documentation/Qualification/Release1.2.1/README.md index 816598e..5f082a7 100644 --- a/Documentation/Qualification/Release1.2.1/README.md +++ b/Documentation/Qualification/Release1.2.1/README.md @@ -19,3 +19,7 @@ Codable containers and precise failures, then held source/sink backpressure. Subsequent waves expand combined schema/regex decisions, Patch, public end-to-end flows and bounded adversarial/mutation evidence. Existing independent suites, all 49 caps and platform-specific limitations remain authoritative. + +First wave implemented and natively qualified: 23 new Swift Testing functions, +215/173 root discoveries, complete coverage export and negative omission probe. +See [actual outcomes](Wave1Outcomes.json) for receipts and remaining scope. diff --git a/Documentation/Qualification/Release1.2.1/Wave1.md b/Documentation/Qualification/Release1.2.1/Wave1.md index f24150e..09a082a 100644 --- a/Documentation/Qualification/Release1.2.1/Wave1.md +++ b/Documentation/Qualification/Release1.2.1/Wave1.md @@ -13,8 +13,9 @@ adds 23 nonparameterized Swift Testing functions using public JSON interfaces: - Keyed/unkeyed nested containers, named/default/unkeyed super containers, explicit empty values and repeated keyed requests/replacement preserving insertion order. -- All 14 standard Bool/String/Float/Double/signed/unsigned primitive overloads, - nil and generic exact JSON values against independently specified golden values. +- The 14 explicit Bool/String/Float/Double/signed/unsigned primitive overloads + through 64-bit integers, plus nil and generic exact JSON values against + independently specified golden values. - Unkeyed decodeNil/index/end behavior and recovery after ordinary type errors; unknown fields, key presence and explicit null. - Complete typed coding paths for missing keys, wrong kinds and inexact integer @@ -55,3 +56,32 @@ schema/regex/Patch decision expansion, public end-to-end scenarios and adversari assurance are later waves. The proposed 98% line/95% region targets have not been declared achieved by adding these cases. New Linux/WASI and device results are not implied by a native run. No release or remote action is part of this increment. + +## Actual native outcome + +Tested code commit: `3c0c03b718c677ce2cb0a3f6c66488b5111d1098`. +[Wave1Outcomes.json](Wave1Outcomes.json) records actual source/receipt identities, +counts, toolchain, measured coverage, review findings and limitations. Closing +documentation changes are separate from that tested code identity. + +Coverage, native acceptance, focused QA, safety and documentation passed on +macOS 27.0 ARM64 with Apple Swift 6.4, Xcode 27.2/SDK 27.2 and language mode 6. +Root debug/release and full ASAN checked 215/173; QA debug/release/ASAN checked +34/3/3/2 functions; TSan checked six shared-state functions. Strict memory safety, +49 cap probes, public consumers, independent policies and 582/582 authored DocC +symbols passed. All approved command outcomes exited 0. + +The original native CLI exited 1 when opening command 12's log failed with ENOSPC. +Its 12 completed commands were retained. After deleting only completed caches +created by this task, the approved four-command tail passed using the same CI +checks; unchanged source, prior log hashes and prior outcomes were revalidated. +The original interruption remains in the local evidence. This is a verified +resumption, not a claim that the interrupted CLI exited 0. + +Encoding/decoding container line coverage is 98.78% each; their region coverage +is 95.45%/96.74%. Overall authored lines/regions are 97.06%/92.54%. One unchanged +streaming completion-alternative region observed in the earlier baseline was not +observed in this run; retain it for deterministic T04 coverage. No cause is +inferred and its denominator remains. The SDK also declares Int128/UInt128 +container defaults; this wave qualifies the 14 explicit overloads through 64-bit +integers and leaves that additional probe in T01. diff --git a/Documentation/Qualification/Release1.2.1/Wave1Outcomes.json b/Documentation/Qualification/Release1.2.1/Wave1Outcomes.json new file mode 100644 index 0000000..5958e60 --- /dev/null +++ b/Documentation/Qualification/Release1.2.1/Wave1Outcomes.json @@ -0,0 +1,564 @@ +{ + "schema_version": 1, + "status": "first_wave_native_qualified_remaining_plan", + "target_version": "1.2.1", + "source_base": "25fb04f747e583209b4c4511abf39e11ef5b4f80", + "tested_code_commit": "3c0c03b718c677ce2cb0a3f6c66488b5111d1098", + "tested_source_manifest_sha256": "79d1e970c804afdb1b0de4ba063ae6e2b893b7ea6368ea979f7ddcaa8d8235f8", + "production_sources_changed": false, + "public_api_changed": false, + "dependencies_added": 0, + "public_products": [ + "JSON", + "JSONSchema", + "JSONStreaming", + "JSONPatch" + ], + "private_target": "JSONRegexRuntime", + "compiler": "swift-driver version: 1.168.6 Apple Swift version 6.4 (swiftlang-6.4.0.34.1 clang-2100.3.34.1)\nTarget: arm64-apple-macosx27.0.0", + "sdk": "27.2", + "os": "macOS 27.0", + "architecture": "arm64", + "language_mode": 6, + "root_functions": { + "JSONTests": 127, + "JSONSchemaTests": 51, + "JSONRegexRuntimeTests": 12, + "JSONStreamingTests": 14, + "JSONPatchTests": 11 + }, + "root_function_total": 215, + "parameterized_invocations": 173, + "new_swift_functions": 23, + "qa_functions": { + "JSONTests": 34, + "JSONSchemaTests": 3, + "JSONStreamingTests": 3, + "JSONPatchTests": 2 + }, + "qa_function_total": 42, + "harness_tests": 10, + "finite_metrics": 49, + "profiles": 3, + "authored_coverage": { + "lines": { + "count": 6929, + "covered": 6725, + "percent": 97.0558522153269 + }, + "regions": { + "count": 7691, + "covered": 7117, + "percent": 92.53673124431154 + }, + "functions": { + "count": 1143, + "covered": 1097, + "percent": 95.97550306211724 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "module_coverage": { + "JSON": { + "mapped_files": 50, + "authored": { + "lines": { + "count": 3443, + "covered": 3312, + "percent": 96.19517862329364 + }, + "regions": { + "count": 3427, + "covered": 3134, + "percent": 91.45024803034724 + }, + "functions": { + "count": 595, + "covered": 575, + "percent": 96.63865546218487 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 0, + "covered": 0, + "percent": null + }, + "regions": { + "count": 0, + "covered": 0, + "percent": null + }, + "functions": { + "count": 0, + "covered": 0, + "percent": null + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + "JSONSchema": { + "mapped_files": 32, + "authored": { + "lines": { + "count": 2410, + "covered": 2361, + "percent": 97.96680497925311 + }, + "regions": { + "count": 3088, + "covered": 2891, + "percent": 93.62046632124353 + }, + "functions": { + "count": 384, + "covered": 365, + "percent": 95.05208333333333 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 0, + "covered": 0, + "percent": null + }, + "regions": { + "count": 0, + "covered": 0, + "percent": null + }, + "functions": { + "count": 0, + "covered": 0, + "percent": null + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + "JSONRegexRuntime": { + "mapped_files": 6, + "authored": { + "lines": { + "count": 659, + "covered": 648, + "percent": 98.33080424886191 + }, + "regions": { + "count": 695, + "covered": 646, + "percent": 92.94964028776978 + }, + "functions": { + "count": 88, + "covered": 85, + "percent": 96.5909090909091 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 30, + "covered": 30, + "percent": 100.0 + }, + "regions": { + "count": 44, + "covered": 44, + "percent": 100.0 + }, + "functions": { + "count": 3, + "covered": 3, + "percent": 100.0 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + "JSONStreaming": { + "mapped_files": 10, + "authored": { + "lines": { + "count": 232, + "covered": 225, + "percent": 96.98275862068965 + }, + "regions": { + "count": 269, + "covered": 254, + "percent": 94.42379182156134 + }, + "functions": { + "count": 60, + "covered": 57, + "percent": 95.0 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 0, + "covered": 0, + "percent": null + }, + "regions": { + "count": 0, + "covered": 0, + "percent": null + }, + "functions": { + "count": 0, + "covered": 0, + "percent": null + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + "JSONPatch": { + "mapped_files": 4, + "authored": { + "lines": { + "count": 185, + "covered": 179, + "percent": 96.75675675675676 + }, + "regions": { + "count": 212, + "covered": 192, + "percent": 90.56603773584905 + }, + "functions": { + "count": 16, + "covered": 15, + "percent": 93.75 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 0, + "covered": 0, + "percent": null + }, + "regions": { + "count": 0, + "covered": 0, + "percent": null + }, + "functions": { + "count": 0, + "covered": 0, + "percent": null + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + } + }, + "codable_containers": { + "encoding": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "functions": { + "count": 40, + "covered": 39, + "percent": 97.5 + }, + "instantiations": { + "count": 40, + "covered": 39, + "percent": 97.5 + }, + "lines": { + "count": 82, + "covered": 81, + "percent": 98.78048780487805 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "regions": { + "count": 88, + "covered": 84, + "notcovered": 4, + "percent": 95.45454545454545 + } + }, + "decoding": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "functions": { + "count": 39, + "covered": 38, + "percent": 97.43589743589743 + }, + "instantiations": { + "count": 39, + "covered": 38, + "percent": 97.43589743589743 + }, + "lines": { + "count": 82, + "covered": 81, + "percent": 98.78048780487805 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "regions": { + "count": 92, + "covered": 89, + "notcovered": 3, + "percent": 96.73913043478261 + } + } + }, + "default_export_complete": false, + "omitted_schema_binary_probe": "rejected", + "branch_counters_available": false, + "mcdc_counters_available": false, + "lane_receipts": { + "coverage": { + "status": "passed", + "sha256": "039454f16608ffd36a9d9b94b6b0de476a8c628d6899ce1ee0e9a113f7ebe258", + "command_count": 2, + "command_exits": [ + 0, + 0 + ] + }, + "native": { + "status": "passed", + "sha256": "70d37828e8d080ba6fb4f7bdc5ae73c54ca361dc642d33c0d6dd7880a6376562", + "command_count": 16, + "command_exits": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + "qa": { + "status": "passed", + "sha256": "d7ceb1683b2c8fc067ce7094ef51720ef8f547d3ff4b636a62a932f7c9a33116", + "command_count": 3, + "command_exits": [ + 0, + 0, + 0 + ] + }, + "safety": { + "status": "passed", + "sha256": "ef0d4ae9b90e499e39707a7f30a764547813f6e46026f1e3281e85396b17707a", + "command_count": 3, + "command_exits": [ + 0, + 0, + 0 + ] + }, + "docs": { + "status": "passed", + "sha256": "11b48ce3afe5cb03c774f053e047d55f9b0521e471d78d01e667ad326015bf9f", + "command_count": 1, + "command_exits": [ + 0 + ] + } + }, + "native_resumption": { + "original_cli_exit": 1, + "reason": "ENOSPC before command 12", + "completed_prefix_commands": 12, + "resumed_tail_commands": [ + 12, + 13, + 14, + 15 + ], + "source_and_log_hashes_verified": true, + "all_16_command_outcomes_passed": true, + "continuation_exit": 0 + }, + "docc_authored_symbols": 582, + "docc_documented_symbols": 582, + "policy_cases": { + "syntax": { + "observed": 318, + "evaluated_successes": 287, + "policy": 31 + }, + "schema": { + "observed": 2337, + "evaluated_successes": 2300, + "policy": 37 + } + }, + "review_findings": [ + { + "id": "R01", + "status": "recorded_remaining_coverage", + "path": "Sources/JSONStreaming/JSONEventSequence.swift", + "line": 82, + "columns": [ + 97, + 124 + ], + "region": ".recordCompleted(summary())", + "previous_count": 1, + "current_count": 0, + "note": "Unchanged source; all streaming behavior tests passed. One completion-alternative region was not observed. Do not infer a cause or hide the denominator; cover deterministically in T04." + }, + { + "id": "R02", + "status": "scope_clarified", + "note": "This wave tests 14 explicit primitive overloads through 64-bit integers. SDK protocol also has Int128/UInt128 defaults; their probe/qualification remains T01 work." + }, + { + "id": "R03", + "status": "helpers_retained", + "note": "JSONOperationContext/JSONFingerprint remain unreferenced and uncovered. Retained pending separate use/removal review; no test-only callers or denominator adjustment." + }, + { + "id": "R04", + "status": "interruption_resolved", + "note": "Native CLI log creation hit ENOSPC. Approved tail resumed using unchanged CI checks after source, prefix outcomes and log hashes were verified. Original exit retained." + } + ], + "limits": [ + "Execution coverage does not prove assertion completeness", + "No new Linux/WASI/device execution claimed; their new-test execution remains later qualification", + "Physical iPad remains deferred under existing policy", + "98% line/95% region targets not yet achieved across all modules", + "No performance/RSS/throughput claim from concurrent-host test timings" + ], + "remote_actions": [], + "release_actions": [], + "sibling_edits": 0 +} From 090fffd19b40b6c9812a0add199e4ec880d24cc9 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Wed, 14 Oct 2026 10:54:09 +0200 Subject: [PATCH 04/24] test: probe foreign coding and control streaming completion --- Documentation/Architecture/API.md | 7 + Documentation/Qualification/README.md | 6 + .../Qualification/Release1.2.1/README.md | 3 + .../Qualification/Release1.2.1/Wave2.md | 77 +++++ .../Qualification/TestDiscovery.json | 10 +- Scripts/ci.py | 2 +- .../ControlledStreamingTests.swift | 286 +++++++++++++++++ Tests/JSONTests/ForeignCodableTests.swift | 287 ++++++++++++++++++ Tests/JSONTests/Integer128ProbeTests.swift | 68 +++++ 9 files changed, 740 insertions(+), 6 deletions(-) create mode 100644 Documentation/Qualification/Release1.2.1/Wave2.md create mode 100644 Tests/JSONStreamingTests/ControlledStreamingTests.swift create mode 100644 Tests/JSONTests/ForeignCodableTests.swift create mode 100644 Tests/JSONTests/Integer128ProbeTests.swift diff --git a/Documentation/Architecture/API.md b/Documentation/Architecture/API.md index 9df8e3c..19826df 100644 --- a/Documentation/Architecture/API.md +++ b/Documentation/Architecture/API.md @@ -101,6 +101,13 @@ metered Base64; Date defaults to its Codable representation. Explicit ISO-8601 u UTC fractional seconds, finite years 0000...9999 and at most 128 input UTF-8 bytes. `JSONCodingOptions.userInfo` stores checked Sendable values. +The qualified 128-bit probe distinguishes exact value projection from typed +Codable. `JSONNumber.integer(as: Int128.self)`/`UInt128.self` support exact bounded +projection. Native typed Int128/UInt128 Codable currently uses the SDK container +defaults and throws unsupported-type EncodingError/DecodingError, including for +small values. Use exact JSONNumber/JSON values or JSONCodablePayload for lossless +transport; this is no claim that typed 128-bit Codable is implemented. + Foreign Codable bridges inherit the foreign coder's source fidelity. External numeric decoding tries Int64, UInt64 and Decimal, with no Double fallback. External encoding requires an exact Int64, UInt64 or Decimal representation. Neither bridge diff --git a/Documentation/Qualification/README.md b/Documentation/Qualification/README.md index 1c7f4a1..d02aa64 100644 --- a/Documentation/Qualification/README.md +++ b/Documentation/Qualification/README.md @@ -96,3 +96,9 @@ Patch 11), retaining all 173 parameterized invocations and all 49 cap outcomes. The focused QA lane now expects 34/3/3/2 functions in JSON/schema/streaming/Patch. Historical counts above stay tied to their original receipts. The exact current [discovery contract](TestDiscovery.json) is shared by native/safety/WASI harnesses. + +The [second 1.2.1 wave](Release1.2.1/Wave2.md) adds 29 behavior functions: +244 root functions (JSON 144, schema 51, regex 12, streaming 26, Patch 11), with +173 parameterized invocations retained. QA expects 51/3/15/2 functions. Safety's +targeted TSan subset adds the 12 controlled streaming functions to the existing +six cases, with exact 4/1/1/12 module counts. Historical receipts remain unchanged. diff --git a/Documentation/Qualification/Release1.2.1/README.md b/Documentation/Qualification/Release1.2.1/README.md index 5f082a7..94a8b8b 100644 --- a/Documentation/Qualification/Release1.2.1/README.md +++ b/Documentation/Qualification/Release1.2.1/README.md @@ -23,3 +23,6 @@ all 49 caps and platform-specific limitations remain authoritative. First wave implemented and natively qualified: 23 new Swift Testing functions, 215/173 root discoveries, complete coverage export and negative omission probe. See [actual outcomes](Wave1Outcomes.json) for receipts and remaining scope. + +The [second wave](Wave2.md) adds foreign-coder and 128-bit probes plus causal +streaming gates. Each wave retains its own source identity and native receipts. diff --git a/Documentation/Qualification/Release1.2.1/Wave2.md b/Documentation/Qualification/Release1.2.1/Wave2.md new file mode 100644 index 0000000..c627dd5 --- /dev/null +++ b/Documentation/Qualification/Release1.2.1/Wave2.md @@ -0,0 +1,77 @@ +# 1.2.1 second test wave + +Parent: first-wave head `21d5428793c4a972f3cfbe34471739784d21c2f0`. +This increment adds tests and qualification metadata on the isolated patch branch. +Production implementation, products, dependencies, finite profiles, platform +declarations and pinned fixtures remain unchanged. Sibling 1.3 work is outside +this branch. Later source identities and actual receipts close this record. + +## Foreign Codable + +[ForeignCodableTests.swift](../../../Tests/JSONTests/ForeignCodableTests.swift) +adds twelve public-API Swift Testing functions. Synchronous, invocation-owned +coder wrappers hide the native coder identity and recursively wrap children, +so nested cases execute the foreign JSON bridge. Controlled primitive attempts +have recorded ordering and injection points. Fixed goldens and exact errors cover: + +- Int64, UInt64 and exact Decimal selection, explicitly excluding Double/Float + fallback; deliberate integer-candidate refusal and NaN Decimal rejection. +- All six JSON kinds, nested objects/arrays, scalar-distinct keys, object order + and JSONObject's nonobject refusal. +- Exact Decimal range/precision refusal, unrepresentable numbers, direct + integer sink-error propagation and wrapped Decimal error/path/cause. +- Container acquisition errors and inability to recover values already changed + by a foreign coder. Caller coder work remains outside owned instruction quotas. + +These are controlled coder behavior tests, not a claim that every external +serializer preserves JSON fidelity. Foundation is not the JSON authority. + +## 128-bit SDK/compiler probe + +[Integer128ProbeTests.swift](../../../Tests/JSONTests/Integer128ProbeTests.swift) +adds five tests. A separately compiled public consumer observed Swift 6.4's +Int128/UInt128 definitions and actual package behavior before expectations froze. +Exact bounded JSONNumber construction/projection supports signed and unsigned +128-bit boundaries, including values beyond 64 bits; overflow/fractional values +fail exact conversion. Native typed Codable uses SDK defaults and refuses both +types even for 1, with EncodingError.invalidValue/DecodingError.typeMismatch. +Exact JSON/JSONNumber passthrough and Version 1 payload transport remain valid. + +This is a qualified limitation. No new overload/support is silently selected by +the probe and no supported baseline is lowered. A typed 128-bit implementation +would require its own reviewed behavior, budgets and platform qualification. + +## Deterministic streaming + +[ControlledStreamingTests.swift](../../../Tests/JSONStreamingTests/ControlledStreamingTests.swift) +adds twelve checked-Sendable actor-gate tests. Each finite fixture admits one +held pull or handoff. Waiters wake on the expected held boundary **or** operation +termination, so an early operational failure is observable rather than leaving +the controller waiting for a callback that never occurs. Tests release/close +held continuations and join their worker tasks. Swift Testing time limits and +an external process watchdog bound focused execution; sleeps never determine +ordering or acceptance. + +Cases cover root-array closing syntax, trailing whitespace and held EOF, +provisional elements, malformed tails, source errors, cancellation and terminal +next calls; single-document values, final NDJSON records and separated syntax +events at EOF; early element return without another source pull; held sink +acceptance, throwing sinks, cancellation after acceptance and exact output-cap +refusal. Accepted bytes remain accepted; no completion/report is fabricated. +These are public-observable pull/handoff/completion checks, not a measurement of +arbitrary caller work, invisible internal instructions or process RSS. + +The first malformed-tail probe mistakenly expected trailingData for lexical +garbage. Shared grammar reports invalidSyntax for garbage and trailingData for +a valid second JSON root. The final test retains both diagnoses with exact byte +offset 4, terminal nil and no further pulls. No library change or weaker assertion +was used to hide that initial expectation error. + +## Discovery and qualification + +The shared [discovery map](../TestDiscovery.json) expects 244/173 roots, +QA 51/3/15/2, and targeted TSan 4/1/1/12. All original cases, policies and 49 cap +outcomes remain. Native coverage/acceptance/QA/safety/docs are the affected lanes. +New Linux/WASI/device runs are separate pending evidence; no runtime or package +support expansion is implied. Broader T02/T03/T05/T06 work and global 98%/95% +coverage targets remain open. Historical first-wave outcomes are preserved. diff --git a/Documentation/Qualification/TestDiscovery.json b/Documentation/Qualification/TestDiscovery.json index c2dd4bf..597d1c3 100644 --- a/Documentation/Qualification/TestDiscovery.json +++ b/Documentation/Qualification/TestDiscovery.json @@ -1,20 +1,20 @@ { "schema_version": 1, "root_functions": { - "JSONTests": 127, + "JSONTests": 144, "JSONSchemaTests": 51, "JSONRegexRuntimeTests": 12, - "JSONStreamingTests": 14, + "JSONStreamingTests": 26, "JSONPatchTests": 11 }, "parameterized_invocations": 173, "qa_functions": { - "JSONTests": 34, + "JSONTests": 51, "JSONSchemaTests": 3, - "JSONStreamingTests": 3, + "JSONStreamingTests": 15, "JSONPatchTests": 2 }, - "shared_state_functions": {"JSONTests": 4, "JSONSchemaTests": 1, "JSONRegexRuntimeTests": 1}, + "shared_state_functions": {"JSONTests": 4, "JSONSchemaTests": 1, "JSONRegexRuntimeTests": 1, "JSONStreamingTests": 12}, "coverage_unmapped": { "Sources/JSON/Pointer/JSONLocationComponent.swift": "declaration-only enum", "Sources/JSONSchema/Output/JSONValidationOutput.swift": "declaration-only enum", diff --git a/Scripts/ci.py b/Scripts/ci.py index ece29c5..158206f 100644 --- a/Scripts/ci.py +++ b/Scripts/ci.py @@ -7,7 +7,7 @@ from test_discovery import DISCOVERY, ROOT_COUNTS, PARAMETER_INVOCATIONS ROOT = pathlib.Path(__file__).resolve().parents[1] -SHARED_TESTS = 'independentCodecFacadesAndExactValuesCanBeSharedAcrossTasks|concurrentSchemaEvaluationsHaveExclusiveDynamicRegexAndAnnotationState|sharedRegexProgramHasExclusiveConcurrentSearchState|immutableIndexedObjectCanBeSharedAcrossTasks|qaSharedPayloadAndRootHaveIndependentConcurrentReadScopes|sha256InputCapacityAndConcurrentInvocationsRemainIndependent' +SHARED_TESTS = 'independentCodecFacadesAndExactValuesCanBeSharedAcrossTasks|concurrentSchemaEvaluationsHaveExclusiveDynamicRegexAndAnnotationState|sharedRegexProgramHasExclusiveConcurrentSearchState|immutableIndexedObjectCanBeSharedAcrossTasks|qaSharedPayloadAndRootHaveIndependentConcurrentReadScopes|sha256InputCapacityAndConcurrentInvocationsRemainIndependent|qaStreaming' R0_TESTS = 'readerProbe|indexProbes|duplicateRecognitionPrecedesValueAndClosingSyntax' def snapshot(): diff --git a/Tests/JSONStreamingTests/ControlledStreamingTests.swift b/Tests/JSONStreamingTests/ControlledStreamingTests.swift new file mode 100644 index 0000000..ce9fc4d --- /dev/null +++ b/Tests/JSONStreamingTests/ControlledStreamingTests.swift @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: Apache-2.0 +import Testing +import JSON +import JSONStreaming + +private enum ControlledError: Error, Equatable { case source, sink, overlappingPull } +private enum SourceReply: Sendable { case bytes(JSONByteChunk), eof, failure } +private struct SourceSnapshot: Sendable { + let pulls: Int + let held: Bool + let finished: Bool + let values: [JSON] + let documentCompletions: Int + let recordCompletions: Int + let summary: JSONStreamSummary? +} +private actor SourceGate { + let initial: [JSONByteChunk] + var pulls = 0, finished = false, closed = false + var pending: CheckedContinuation? + var observer: CheckedContinuation? + var values: [JSON] = [] + var documents = 0, records = 0 + var summary: JSONStreamSummary? + init(_ texts: [String]) { initial = texts.map { JSONByteChunk(Array($0.utf8)) } } + func notify() { if pending != nil || finished { observer?.resume(); observer = nil } } + func next() async throws -> JSONByteChunk? { + pulls += 1 + guard pending == nil else { throw ControlledError.overlappingPull } + if closed { return nil } + if pulls <= initial.count { return initial[pulls-1] } + if pulls == initial.count+1 { + guard pending == nil else { throw ControlledError.overlappingPull } + return try await withCheckedThrowingContinuation { pending = $0; notify() } + } + return nil + } + func waitForHeldOrFinished() async { + if pending != nil || finished { return } + await withCheckedContinuation { observer = $0 } + } + func reply(_ reply: SourceReply) { + let continuation = pending; pending = nil + switch reply { + case .bytes(let bytes): continuation?.resume(returning:bytes) + case .eof: continuation?.resume(returning:nil) + case .failure: continuation?.resume(throwing:ControlledError.source) + } + } + func close() { closed = true; reply(.eof) } + func done() { finished = true; notify() } + func value(_ value: JSON) { values.append(value) } + func completed(_ summary: JSONStreamSummary, document: Bool) { + if document { documents += 1 } else { records += 1 }; self.summary = summary + } + func snapshot() -> SourceSnapshot { + .init(pulls:pulls,held:pending != nil,finished:finished,values:values, + documentCompletions:documents,recordCompletions:records,summary:summary) + } +} +private struct ControlledSource: AsyncSequence, Sendable { + typealias Element = JSONByteChunk + let gate: SourceGate + func makeAsyncIterator() -> Iterator { .init(gate:gate) } + struct Iterator: AsyncIteratorProtocol { + let gate: SourceGate + mutating func next() async throws -> JSONByteChunk? { try await gate.next() } + } +} +private struct ReadOutcome: Sendable { let error: (any Error)?; let terminalNil: Bool } +private func arrayTask(_ gate: SourceGate) -> Task { + Task { + var iterator = JSONTopLevelArraySequence(ControlledSource(gate:gate)).makeAsyncIterator() + var failure: (any Error)? + do { + while let event = try await iterator.next() { + switch event { + case .element(let index,let value): + #expect(index == UInt64(await gate.snapshot().values.count)); await gate.value(value) + case .documentCompleted(let summary): await gate.completed(summary,document:true) + } + } + } catch { failure = error } + let terminal: Bool + do { terminal = try await iterator.next() == nil } catch { terminal = false } + await gate.done() + return .init(error:failure,terminalNil:terminal) + } +} + +@Test(.timeLimit(.minutes(1))) func qaStreamingRootCompletionWaitsForEOF() async throws { + let gate = SourceGate(["[0] \r\n"]), task = arrayTask(gate) + await gate.waitForHeldOrFinished() + let before = await gate.snapshot() + #expect(before.held && !before.finished && before.pulls == 2) + #expect(before.values == [0] && before.documentCompletions == 0) + await gate.reply(.eof) + let result = await task.value; await gate.close() + let after = await gate.snapshot() + #expect(result.error == nil && result.terminalNil) + #expect(after.documentCompletions == 1 && after.pulls == 2) + #expect(after.summary?.records == 1 && after.summary?.statistics[.totalScannedNodes] == 2) +} +@Test(.timeLimit(.minutes(1))) func qaStreamingClosingSyntaxAndWhitespacePrecedeCompletion() async { + let gate = SourceGate(["[0"]), task = arrayTask(gate) + await gate.waitForHeldOrFinished() + let before = await gate.snapshot() + #expect(before.held && before.values.isEmpty && before.documentCompletions == 0) + await gate.reply(.bytes(.init(Array("] \t\r\n".utf8)))) + let result = await task.value; await gate.close() + let after = await gate.snapshot() + #expect(result.error == nil && result.terminalNil && after.values == [0]) + #expect(after.documentCompletions == 1 && after.pulls == 3) +} +@Test(.timeLimit(.minutes(1))) func qaStreamingMalformedTailNeverCompletesAndStopsPulling() async throws { + // An invalid token fails lexing; a valid second JSON root fails framing. + // Keep both exact diagnoses rather than accepting any JSONFailure. + for (tail, code) in [(" garbage", JSONFailure.Code.invalidSyntax), (" false", .trailingData)] { + let gate = SourceGate(["[0]"]), task = arrayTask(gate) + await gate.waitForHeldOrFinished() + #expect(await gate.snapshot().values == [0]) + await gate.reply(.bytes(.init(Array(tail.utf8)))) + let result = await task.value; await gate.close() + let failure = try #require(result.error as? JSONFailure) + #expect(failure.category == .syntax && failure.code == code && failure.resource == nil) + #expect(failure.location?.byteOffset == 4) + let final = await gate.snapshot() + #expect(result.terminalNil && final.documentCompletions == 0 && final.pulls == 2) + } +} +@Test(.timeLimit(.minutes(1))) func qaStreamingSourceErrorAfterElementRemainsOperationalAndTerminal() async { + let gate = SourceGate(["[0]"]), task = arrayTask(gate) + await gate.waitForHeldOrFinished(); await gate.reply(.failure) + let result = await task.value; await gate.close() + #expect(result.error as? ControlledError == .source && result.terminalNil) + let final = await gate.snapshot() + #expect(final.values == [0] && final.documentCompletions == 0 && final.pulls == 2) +} +@Test(.timeLimit(.minutes(1))) func qaStreamingCancellationWhileEOFHeldSuppressesCompletion() async { + let gate = SourceGate(["[0]"]), task = arrayTask(gate) + await gate.waitForHeldOrFinished(); task.cancel(); await gate.reply(.eof) + let result = await task.value; await gate.close() + #expect(result.error is CancellationError && result.terminalNil) + let final = await gate.snapshot() + #expect(final.values == [0] && final.documentCompletions == 0 && final.pulls == 2) +} +@Test(.timeLimit(.minutes(1))) func qaStreamingFirstElementReturnDoesNotAwaitAnotherSourceChunk() async { + let gate = SourceGate(["[0,"]) + let task = Task { + var iterator = JSONTopLevelArraySequence(ControlledSource(gate:gate)).makeAsyncIterator() + var yielded = false + do { if case .element(let index,let value) = try await iterator.next() { yielded = index == 0 && value == 0 } } + catch { } + await gate.done(); return yielded + } + await gate.waitForHeldOrFinished() + let state = await gate.snapshot() + #expect(state.finished && !state.held && state.pulls == 1) + await gate.close(); #expect(await task.value) +} +@Test(.timeLimit(.minutes(1))) func qaStreamingSeparatedEventsFinishAtEOFExactlyOnce() async { + for text in ["false", "17", "{}"] { + let gate = SourceGate([text]) + let task = Task { + var iterator = JSONEventSequence(ControlledSource(gate:gate),options:.init(framing:.whitespaceSeparatedValues)).makeAsyncIterator() + var succeeded = true + do { + while let event = try await iterator.next() { + switch event { + case .scalar(let value): await gate.value(value) + case .recordCompleted(let s): await gate.completed(s,document:false) + case .documentCompleted(let s): await gate.completed(s,document:true) + default: break + } + } + #expect(try await iterator.next() == nil) + } catch { succeeded = false } + await gate.done(); return succeeded + } + await gate.waitForHeldOrFinished() + let before = await gate.snapshot() + #expect(before.held && !before.finished && before.recordCompletions == 0 && before.documentCompletions == 0) + await gate.reply(.eof); #expect(await task.value); await gate.close() + let final = await gate.snapshot() + #expect(final.recordCompletions == 1 && final.documentCompletions == 0 && final.pulls == 2) + #expect(final.summary?.records == 1) + } +} +@Test(.timeLimit(.minutes(1))) func qaStreamingSingleValueAndFinalNDJSONRecordWaitForEOF() async { + for ndjson in [false,true] { + let gate = SourceGate([ndjson ? "1\n2" : "2"]) + let task = Task { + var succeeded = true + do { + if ndjson { + for try await value in NDJSONValueSequence(ControlledSource(gate:gate)) { await gate.value(value) } + } else { + for try await value in JSONValueSequence(ControlledSource(gate:gate)) { await gate.value(value) } + } + } catch { succeeded = false } + await gate.done(); return succeeded + } + await gate.waitForHeldOrFinished() + let before = await gate.snapshot() + #expect(before.held && !before.finished && before.values == (ndjson ? [1] : [])) + await gate.reply(.eof); #expect(await task.value); await gate.close() + #expect(await gate.snapshot().values == (ndjson ? [1,2] : [2])) + } +} + +private enum SinkReply: Sendable { case accept, reject } +private struct SinkSnapshot: Sendable { let calls:Int; let bytes:[UInt8]; let held:Bool; let finished:Bool } +private actor SinkGate { + let holdAt:Int + var calls=0, bytes:[UInt8]=[], finished=false + var pending:CheckedContinuation? + var observer:CheckedContinuation? + init(holdAt:Int=1) { self.holdAt=holdAt } + func notify() { if pending != nil || finished { observer?.resume(); observer=nil } } + func accept(_ chunk:JSONByteChunk) async throws { + calls += 1 + if calls == holdAt { try await withCheckedThrowingContinuation { pending=$0; notify() } } + bytes.append(contentsOf:chunk.bytes) + } + func waitForHeldOrFinished() async { + if pending != nil || finished { return }; await withCheckedContinuation { observer=$0 } + } + func reply(_ value:SinkReply) { + let p=pending;pending=nil + switch value { case .accept:p?.resume();case .reject:p?.resume(throwing:ControlledError.sink) } + } + func done() { finished=true;notify() } + func snapshot() -> SinkSnapshot { .init(calls:calls,bytes:bytes,held:pending != nil,finished:finished) } +} +private struct WriteOutcome: Sendable { let error:(any Error)?; let report:JSONOperationStatistics? } +private func writerTask(_ gate:SinkGate,limits:JSONLimits = .balanced) -> Task { + Task { + let outcome:WriteOutcome + do { + let report=try await JSONAsyncWriter(options:.init(limits:limits),chunkSize:2).writeWithReport(.array([1,2,3])) { try await gate.accept($0) } + outcome = .init(error:nil,report:report) + } catch { outcome = .init(error:error,report:nil) } + await gate.done();return outcome + } +} +@Test(.timeLimit(.minutes(1))) func qaStreamingHeldSinkHasOneOutstandingHandoffUntilAcceptance() async { + let gate=SinkGate(),task=writerTask(gate) + await gate.waitForHeldOrFinished() + let held=await gate.snapshot() + #expect(held.held && !held.finished && held.calls == 1 && held.bytes.isEmpty) + await gate.reply(.accept);let result=await task.value + let final=await gate.snapshot() + #expect(result.error == nil && final.bytes == Array("[1,2,3]".utf8) && final.calls == 4) + #expect(result.report?[.outputBytes] == 7 && result.report?.domain == .writing) +} +@Test(.timeLimit(.minutes(1))) func qaStreamingHeldSinkFailureKeepsAcceptedPrefixAndStopsHandoffs() async { + let gate=SinkGate(holdAt:2),task=writerTask(gate) + await gate.waitForHeldOrFinished() + let held=await gate.snapshot() + #expect(held.calls == 2 && held.bytes == Array("[1".utf8) && !held.finished) + await gate.reply(.reject);let result=await task.value + let final=await gate.snapshot() + #expect(result.error as? ControlledError == .sink && result.report == nil) + #expect(final.calls == 2 && final.bytes == Array("[1".utf8)) +} +@Test(.timeLimit(.minutes(1))) func qaStreamingCancellationAfterHeldAcceptanceKeepsOnlyAcceptedBytes() async { + let gate=SinkGate(),task=writerTask(gate) + await gate.waitForHeldOrFinished();task.cancel();await gate.reply(.accept) + let result=await task.value;let final=await gate.snapshot() + #expect(result.error is CancellationError && result.report == nil) + #expect(final.calls == 1 && final.bytes == Array("[1".utf8)) +} +@Test(.timeLimit(.minutes(1))) func qaStreamingOutputQuotaRefusalFollowsHeldChunkAcceptance() async throws { + var limits=JSONLimits.balanced;limits[.outputBytes]=2 + let gate=SinkGate(),task=writerTask(gate,limits:limits) + await gate.waitForHeldOrFinished() + let held=await gate.snapshot() + #expect(held.calls == 1 && held.held && !held.finished && held.bytes.isEmpty) + await gate.reply(.accept);let result=await task.value + let failure=try #require(result.error as? JSONFailure),resource=try #require(failure.resource) + #expect(failure.category == .resource && failure.code == .limitExceeded) + #expect(resource.domain == .writing && resource.metric == .outputBytes && resource.limit == 2) + #expect(resource.attemptedUsage == 3 && !resource.overflow && result.report == nil) + let final=await gate.snapshot() + #expect(final.calls == 1 && final.bytes == Array("[1".utf8)) +} diff --git a/Tests/JSONTests/ForeignCodableTests.swift b/Tests/JSONTests/ForeignCodableTests.swift new file mode 100644 index 0000000..588e292 --- /dev/null +++ b/Tests/JSONTests/ForeignCodableTests.swift @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation +import Testing +import JSON + +private enum ForeignError: Error, Equatable { case injected } +private final class ForeignTrace { + var calls: [String] = [] + let reject: Set + let representedInteger: Int64? + let nanDecimal: Bool + init(reject: Set = [], representedInteger: Int64? = nil, nanDecimal: Bool = false) { + self.reject = reject; self.representedInteger = representedInteger; self.nanDecimal = nanDecimal + } + func visit(_ name: String) throws { calls.append(name); if reject.contains(name) { throw ForeignError.injected } } +} +// These synchronous, invocation-owned wrappers deliberately hide the native coder +// type. They exercise the *foreign* bridge, retaining typed paths and allowing +// independently controlled primitive rejection. No Foundation JSON serialization. +private struct ForeignDecoder: Decoder { + let base: any Decoder + let trace: ForeignTrace + var codingPath: [any CodingKey] { base.codingPath } + var userInfo: [CodingUserInfoKey: Any] { base.userInfo } + func singleValueContainer() throws -> any SingleValueDecodingContainer { + try trace.visit("decode:single") + return ForeignSingleDecoder(base: base, c: try base.singleValueContainer(), trace: trace) + } + func container(keyedBy type: K.Type) throws -> KeyedDecodingContainer { + try trace.visit("decode:object") + return KeyedDecodingContainer(ForeignKeyedDecoder(c: try base.container(keyedBy: type), trace: trace)) + } + func unkeyedContainer() throws -> any UnkeyedDecodingContainer { + try trace.visit("decode:array") + return ForeignUnkeyedDecoder(c: try base.unkeyedContainer(), trace: trace) + } +} +private struct ForeignSingleDecoder: SingleValueDecodingContainer { + let base: any Decoder + let c: any SingleValueDecodingContainer + let trace: ForeignTrace + var codingPath: [any CodingKey] { base.codingPath } + func decodeNil() -> Bool { c.decodeNil() } + func decode(_ type: T.Type) throws -> T { + if type == Decimal.self { + try trace.visit("decode:Decimal") + if trace.nanDecimal { return Decimal.nan as! T } + return try c.decode(JSONNumber.self).decimalValue() as! T + } + return try T(from: ForeignDecoder(base: base, trace: trace)) + } + private func primitive(_ type: T.Type, _ body: () throws -> T) throws -> T { + try trace.visit("decode:\(type)"); return try body() + } + func decode(_ type: Bool.Type) throws -> Bool { try primitive(type) { try c.decode(type) } } + func decode(_ type: String.Type) throws -> String { try primitive(type) { try c.decode(type) } } + func decode(_ type: Double.Type) throws -> Double { try primitive(type) { try c.decode(type) } } + func decode(_ type: Float.Type) throws -> Float { try primitive(type) { try c.decode(type) } } + func decode(_ type: Int.Type) throws -> Int { try primitive(type) { try c.decode(type) } } + func decode(_ type: Int8.Type) throws -> Int8 { try primitive(type) { try c.decode(type) } } + func decode(_ type: Int16.Type) throws -> Int16 { try primitive(type) { try c.decode(type) } } + func decode(_ type: Int32.Type) throws -> Int32 { try primitive(type) { try c.decode(type) } } + func decode(_ type: Int64.Type) throws -> Int64 { try primitive(type) { if let value = trace.representedInteger { return value }; return try c.decode(type) } } + func decode(_ type: UInt.Type) throws -> UInt { try primitive(type) { try c.decode(type) } } + func decode(_ type: UInt8.Type) throws -> UInt8 { try primitive(type) { try c.decode(type) } } + func decode(_ type: UInt16.Type) throws -> UInt16 { try primitive(type) { try c.decode(type) } } + func decode(_ type: UInt32.Type) throws -> UInt32 { try primitive(type) { try c.decode(type) } } + func decode(_ type: UInt64.Type) throws -> UInt64 { try primitive(type) { try c.decode(type) } } +} +private struct ForeignKeyedDecoder: KeyedDecodingContainerProtocol { + let c: KeyedDecodingContainer + let trace: ForeignTrace + var codingPath: [any CodingKey] { c.codingPath } + var allKeys: [K] { c.allKeys } + func contains(_ key: K) -> Bool { c.contains(key) } + func decodeNil(forKey key: K) throws -> Bool { try c.decodeNil(forKey: key) } + func decode(_ type: T.Type, forKey key: K) throws -> T { try T(from: superDecoder(forKey: key)) } + func superDecoder(forKey key: K) throws -> any Decoder { ForeignDecoder(base: try c.superDecoder(forKey: key), trace: trace) } + func superDecoder() throws -> any Decoder { ForeignDecoder(base: try c.superDecoder(), trace: trace) } + func nestedContainer(keyedBy type: N.Type, forKey key: K) throws -> KeyedDecodingContainer { try superDecoder(forKey: key).container(keyedBy: type) } + func nestedUnkeyedContainer(forKey key: K) throws -> any UnkeyedDecodingContainer { try superDecoder(forKey: key).unkeyedContainer() } +} +private struct ForeignUnkeyedDecoder: UnkeyedDecodingContainer { + var c: any UnkeyedDecodingContainer + let trace: ForeignTrace + var codingPath: [any CodingKey] { c.codingPath } + var count: Int? { c.count } + var currentIndex: Int { c.currentIndex } + var isAtEnd: Bool { c.isAtEnd } + mutating func decodeNil() throws -> Bool { try c.decodeNil() } + mutating func decode(_ type: T.Type) throws -> T { try T(from: superDecoder()) } + mutating func superDecoder() throws -> any Decoder { ForeignDecoder(base: try c.superDecoder(), trace: trace) } + mutating func nestedContainer(keyedBy type: N.Type) throws -> KeyedDecodingContainer { try superDecoder().container(keyedBy: type) } + mutating func nestedUnkeyedContainer() throws -> any UnkeyedDecodingContainer { try superDecoder().unkeyedContainer() } +} +private struct ForeignEncoder: Encoder { + let base: any Encoder + let trace: ForeignTrace + var codingPath: [any CodingKey] { base.codingPath } + var userInfo: [CodingUserInfoKey: Any] { base.userInfo } + func singleValueContainer() -> any SingleValueEncodingContainer { ForeignSingleEncoder(base: base, c: base.singleValueContainer(), trace: trace) } + func container(keyedBy type: K.Type) -> KeyedEncodingContainer { KeyedEncodingContainer(ForeignKeyedEncoder(c: base.container(keyedBy: type), trace: trace)) } + func unkeyedContainer() -> any UnkeyedEncodingContainer { ForeignUnkeyedEncoder(c: base.unkeyedContainer(), trace: trace) } +} +private struct ForeignSingleEncoder: SingleValueEncodingContainer { + let base: any Encoder + var c: any SingleValueEncodingContainer + let trace: ForeignTrace + var codingPath: [any CodingKey] { base.codingPath } + mutating func encodeNil() throws { try trace.visit("encode:nil"); try c.encodeNil() } + mutating func encode(_ value: T) throws { + if let decimal = value as? Decimal { + try trace.visit("encode:Decimal") + try c.encode(JSONNumber(parsing: decimal.description)) + } else { try value.encode(to: ForeignEncoder(base: base, trace: trace)) } + } + mutating func encode(_ v: Bool) throws { try trace.visit("encode:Bool"); try c.encode(v) } + mutating func encode(_ v: String) throws { try trace.visit("encode:String"); try c.encode(v) } + mutating func encode(_ v: Double) throws { try trace.visit("encode:Double"); try c.encode(v) } + mutating func encode(_ v: Float) throws { try trace.visit("encode:Float"); try c.encode(v) } + mutating func encode(_ v: Int) throws { try trace.visit("encode:Int"); try c.encode(v) } + mutating func encode(_ v: Int8) throws { try trace.visit("encode:Int8"); try c.encode(v) } + mutating func encode(_ v: Int16) throws { try trace.visit("encode:Int16"); try c.encode(v) } + mutating func encode(_ v: Int32) throws { try trace.visit("encode:Int32"); try c.encode(v) } + mutating func encode(_ v: Int64) throws { try trace.visit("encode:Int64"); try c.encode(v) } + mutating func encode(_ v: UInt) throws { try trace.visit("encode:UInt"); try c.encode(v) } + mutating func encode(_ v: UInt8) throws { try trace.visit("encode:UInt8"); try c.encode(v) } + mutating func encode(_ v: UInt16) throws { try trace.visit("encode:UInt16"); try c.encode(v) } + mutating func encode(_ v: UInt32) throws { try trace.visit("encode:UInt32"); try c.encode(v) } + mutating func encode(_ v: UInt64) throws { try trace.visit("encode:UInt64"); try c.encode(v) } +} +private struct ForeignKeyedEncoder: KeyedEncodingContainerProtocol { + var c: KeyedEncodingContainer + let trace: ForeignTrace + var codingPath: [any CodingKey] { c.codingPath } + mutating func encodeNil(forKey key: K) throws { try c.encodeNil(forKey: key) } + mutating func encode(_ value: T, forKey key: K) throws { try value.encode(to: superEncoder(forKey: key)) } + mutating func superEncoder(forKey key: K) -> any Encoder { ForeignEncoder(base: c.superEncoder(forKey: key), trace: trace) } + mutating func superEncoder() -> any Encoder { ForeignEncoder(base: c.superEncoder(), trace: trace) } + mutating func nestedContainer(keyedBy type: N.Type, forKey key: K) -> KeyedEncodingContainer { superEncoder(forKey: key).container(keyedBy: type) } + mutating func nestedUnkeyedContainer(forKey key: K) -> any UnkeyedEncodingContainer { superEncoder(forKey: key).unkeyedContainer() } +} +private struct ForeignUnkeyedEncoder: UnkeyedEncodingContainer { + var c: any UnkeyedEncodingContainer + let trace: ForeignTrace + var codingPath: [any CodingKey] { c.codingPath } + var count: Int { c.count } + mutating func encodeNil() throws { try c.encodeNil() } + mutating func encode(_ value: T) throws { try value.encode(to: superEncoder()) } + mutating func superEncoder() -> any Encoder { ForeignEncoder(base: c.superEncoder(), trace: trace) } + mutating func nestedContainer(keyedBy type: N.Type) -> KeyedEncodingContainer { superEncoder().container(keyedBy: type) } + mutating func nestedUnkeyedContainer() -> any UnkeyedEncodingContainer { superEncoder().unkeyedContainer() } +} +private struct ForeignRead: Decodable { + let value: T + let calls: [String] + init(from decoder: any Decoder) throws { + let trace = ForeignTrace() + value = try T(from: ForeignDecoder(base: decoder, trace: trace)); calls = trace.calls + } +} +private struct ForeignWrite: Encodable { + let value: T + let trace: ForeignTrace + func encode(to encoder: any Encoder) throws { try value.encode(to: ForeignEncoder(base: encoder, trace: trace)) } +} +private func captured(_ body: () throws -> Void) -> any Error { + do { try body(); Issue.record("Expected foreign coder failure"); return ForeignError.injected } + catch { return error } +} + +@Test func qaForeignNumberDecodingSelectsInt64UInt64ThenDecimalWithoutDouble() throws { + for (text, candidates) in [ + ("-9223372036854775808", ["decode:Int64"]), + ("18446744073709551615", ["decode:Int64", "decode:UInt64"]), + ("0.125", ["decode:Int64", "decode:UInt64", "decode:Decimal"]) + ] { + let number = try JSONNumber(parsing: text) + let decoded = try JSONValueDecoder().decode(ForeignRead.self, from: .number(number)) + #expect(decoded.value == number) + #expect(decoded.calls.filter { ["decode:Int64", "decode:UInt64", "decode:Decimal"].contains($0) } == candidates) + #expect(!decoded.calls.contains("decode:Double") && !decoded.calls.contains("decode:Float")) + } +} +@Test func qaForeignNumberEncodingUsesExactRepresentationsWithoutBinaryFallback() throws { + for (text, expected) in [("-9223372036854775808", "encode:Int64"), ("18446744073709551615", "encode:UInt64"), ("0.125", "encode:Decimal")] { + let number = try JSONNumber(parsing: text), trace = ForeignTrace() + #expect(try JSONValueEncoder().encode(ForeignWrite(value: number, trace: trace)) == .number(number)) + #expect(trace.calls == [expected]) + } +} +@Test func qaForeignSixKindsAndNestedScalarKeysMatchIndependentGolden() throws { + for text in ["null", "true", #""é""#, "9007199254740993", "[null,false,0.125]", #"{"é":[1],"é":{"value":0.125}}"#] { + let value = try JSONParser().parse(text), trace = ForeignTrace() + let decoded = try JSONValueDecoder().decode(ForeignRead.self, from: value) + #expect(decoded.value == value) + #expect(try JSONValueEncoder().encode(ForeignWrite(value: value, trace: trace)) == value) + #expect(!decoded.calls.contains("decode:Double") && !trace.calls.contains("encode:Double")) + } +} +@Test func qaForeignJSONObjectBridgePreservesOrderAndRejectsNonobjects() throws { + let value = try JSONParser().parse(#"{"z":null,"é":1,"é":2}"#) + let decoded = try JSONValueDecoder().decode(ForeignRead.self, from: value) + #expect(decoded.value.map { Array($0.key.string.utf8) } == [Array("z".utf8), Array("é".utf8), Array("e\u{301}".utf8)]) + #expect(try JSONValueEncoder().encode(ForeignWrite(value: decoded.value, trace: ForeignTrace())) == value) + let error = captured { _ = try JSONValueDecoder().decode(ForeignRead.self, from: .array([])) } + guard case DecodingError.typeMismatch(let type, let context) = error else { Issue.record("Expected object type mismatch"); return } + #expect(ObjectIdentifier(type) == ObjectIdentifier(JSONObject.self) && context.codingPath.isEmpty) +} +@Test func qaForeignUnrepresentableNumbersRejectInsteadOfRounding() throws { + let number = try JSONNumber(parsing: "1e1000") + let error = captured { _ = try JSONValueDecoder().decode(ForeignRead.self, from: .number(number)) } + guard case DecodingError.typeMismatch(let type, let context) = error else { Issue.record("Expected exact-number type mismatch"); return } + #expect(ObjectIdentifier(type) == ObjectIdentifier(JSONNumber.self) && context.codingPath.isEmpty) + let encodedError = captured { _ = try JSONValueEncoder().encode(ForeignWrite(value: number, trace: ForeignTrace())) } + guard case EncodingError.invalidValue(_, let encodedContext) = encodedError else { Issue.record("Expected exact-number encoding failure"); return } + #expect(encodedContext.codingPath.isEmpty) + let cause = try #require(encodedContext.underlyingError as? JSONFailure) + #expect(cause.category == .conversion && cause.code == .inexactConversion) +} +@Test func qaForeignIntegerSinkErrorPropagatesWithoutTryingAnotherRepresentation() throws { + let trace = ForeignTrace(reject: ["encode:Int64"]) + let error = captured { _ = try JSONValueEncoder().encode(ForeignWrite(value: JSONNumber(integer: 7), trace: trace)) } + #expect(error as? ForeignError == .injected && trace.calls == ["encode:Int64"]) +} +@Test func qaForeignDecimalSinkErrorIsWrappedWithItsNestedPathAndCause() throws { + let trace = ForeignTrace(reject: ["encode:Decimal"]) + let error = captured { _ = try JSONValueEncoder().encode(ForeignWrite(value: [JSONNumber(parsing: "0.125")], trace: trace)) } + guard case EncodingError.invalidValue(_, let context) = error else { Issue.record("Expected decimal encoding failure"); return } + #expect(context.codingPath.map(\.stringValue) == ["0"] && context.codingPath.map(\.intValue) == [0]) + #expect(context.underlyingError as? ForeignError == .injected && trace.calls == ["encode:Decimal"]) +} +private struct RejectSingleContainer: Decodable { + init(from decoder: any Decoder) throws { + _ = try JSONNumber(from: ForeignDecoder(base: decoder, trace: ForeignTrace(reject: ["decode:single"]))) + } +} +@Test func qaForeignSingleContainerAcquisitionErrorPropagatesUnchanged() throws { + #expect(captured { _ = try JSONValueDecoder().decode(RejectSingleContainer.self, from: 7) } as? ForeignError == .injected) +} + +private struct ForcedNumericFallback: Decodable { + let number: JSONNumber + let calls: [String] + init(from decoder: any Decoder) throws { + let trace = ForeignTrace(reject: ["decode:Int64", "decode:UInt64"]) + number = try JSONNumber(from:ForeignDecoder(base:decoder,trace:trace)); calls = trace.calls + } +} +@Test func qaForeignRejectedIntegerCandidatesUseOnlyTheDecimalFallback() throws { + let decoded = try JSONValueDecoder().decode(ForcedNumericFallback.self,from:7) + #expect(decoded.number == JSONNumber(integer:7)) + #expect(decoded.calls == ["decode:single", "decode:Int64", "decode:UInt64", "decode:Decimal"]) +} +private struct NaNForeignNumber: Decodable { + init(from decoder: any Decoder) throws { + let trace = ForeignTrace(reject:["decode:Int64", "decode:UInt64"],nanDecimal:true) + let error = captured { _ = try JSONNumber(from:ForeignDecoder(base:decoder,trace:trace)) } + guard case DecodingError.typeMismatch(let type,let context) = error else { Issue.record("Expected rejected NaN Decimal"); return } + #expect(ObjectIdentifier(type) == ObjectIdentifier(JSONNumber.self) && context.codingPath.isEmpty) + #expect(trace.calls == ["decode:single", "decode:Int64", "decode:UInt64", "decode:Decimal"]) + } +} +@Test func qaForeignNaNDecimalNeverBecomesJSONOrABinaryFallback() throws { + _ = try JSONValueDecoder().decode(NaNForeignNumber.self,from:7) +} +private struct AlreadyChangedForeignNumber: Decodable { + let number: JSONNumber + init(from decoder: any Decoder) throws { + number = try JSONNumber(from:ForeignDecoder(base:decoder,trace:ForeignTrace(representedInteger:9_007_199_254_740_992))) + } +} +@Test func qaForeignCoderFidelityCannotRecoverPreviouslyChangedNumbers() throws { + let source = try JSONParser().parse("9007199254740993") + let decoded = try JSONValueDecoder().decode(AlreadyChangedForeignNumber.self,from:source) + #expect(decoded.number == JSONNumber(integer:9_007_199_254_740_992)) + #expect(JSON.number(decoded.number) != source) + #expect(try JSONValueDecoder().decode(JSON.self,from:source) == source) +} +@Test func qaForeignExactDecimalLimitsRejectFractionalUnderflowAndLongPrecision() throws { + for text in ["1e-1000", "0.123456789012345678901234567890123456789012345678901"] { + let value = try JSONParser().parse(text) + let error = captured { _ = try JSONValueDecoder().decode(ForeignRead.self,from:value) } + guard case DecodingError.typeMismatch(let type,let context) = error else { Issue.record("Expected finite exact Decimal rejection"); continue } + #expect(ObjectIdentifier(type) == ObjectIdentifier(JSONNumber.self) && context.codingPath.isEmpty) + } +} diff --git a/Tests/JSONTests/Integer128ProbeTests.swift b/Tests/JSONTests/Integer128ProbeTests.swift new file mode 100644 index 0000000..fc76b0e --- /dev/null +++ b/Tests/JSONTests/Integer128ProbeTests.swift @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation +import Testing +import JSON + +private func refusal(_ body: () throws -> Void) -> any Error { + do { try body(); Issue.record("Expected unsupported typed Codable operation"); return CancellationError() } + catch { return error } +} +@Test func qaInt128ExactProjectionsCoverSignedAndUnsignedBoundaries() throws { + for (text, value) in [ + ("-170141183460469231731687303715884105728", Int128.min), + ("-18446744073709551617", -Int128(UInt64.max)-2), + ("0", Int128(0)), ("1", Int128(1)), + ("170141183460469231731687303715884105727", Int128.max) + ] { + let number = try JSONNumber(parsing: text) + #expect(try number.integer(as: Int128.self) == value) + #expect(JSONNumber(integer: value) == number) + } + for (text,value) in [("0",UInt128(0)),("18446744073709551616",UInt128(UInt64.max)+1), + ("340282366920938463463374607431768211455",UInt128.max)] { + let number = try JSONNumber(parsing: text) + #expect(try number.integer(as: UInt128.self) == value) + #expect(JSONNumber(integer: value) == number) + } +} +@Test func qaInt128ProjectionOverflowIsExactConversionFailure() throws { + for text in ["170141183460469231731687303715884105728", "-170141183460469231731687303715884105729", "1.5"] { + let number = try JSONNumber(parsing: text) + #expect(refusal { _ = try number.integer(as: Int128.self) } as? JSONFailure == JSONFailure(.conversion,code:.inexactConversion)) + } + for text in ["340282366920938463463374607431768211456", "-1", "0.1"] { + let number = try JSONNumber(parsing: text) + #expect(refusal { _ = try number.integer(as: UInt128.self) } as? JSONFailure == JSONFailure(.conversion,code:.inexactConversion)) + } +} +private func encodingUnsupported(_ value: T) { + let error = refusal { _ = try JSONValueEncoder().encode(value) } + guard case EncodingError.invalidValue(let rejected,let context) = error else { Issue.record("Expected SDK unsupported integer encoding"); return } + #expect(rejected as? T == value && context.codingPath.isEmpty && context.underlyingError == nil) +} +@Test func qaInt128NativeCodableEncodingRefusesWithoutLossyCoercion() { + for value in [Int128.min, -1, 0, 1, Int128.max] { encodingUnsupported(value) } + for value in [UInt128(0),1,UInt128.max] { encodingUnsupported(value) } +} +private func decodingUnsupported(_ type: T.Type,_ text: String) throws { + let value = try JSONParser().parse(text) + let error = refusal { _ = try JSONValueDecoder().decode(type,from:value) } + guard case DecodingError.typeMismatch(let requested,let context) = error else { Issue.record("Expected SDK unsupported integer decoding"); return } + #expect(ObjectIdentifier(requested) == ObjectIdentifier(type)) + #expect(context.codingPath.isEmpty && context.underlyingError == nil) +} +@Test func qaInt128NativeCodableDecodingRefusesEvenRepresentableValues() throws { + for text in ["1","-170141183460469231731687303715884105728","170141183460469231731687303715884105727"] { + try decodingUnsupported(Int128.self,text) + } + for text in ["1","340282366920938463463374607431768211455"] { try decodingUnsupported(UInt128.self,text) } +} +@Test func qaInt128ExactJSONAndVersion1PayloadAreSupportedAlternatives() throws { + for text in ["-170141183460469231731687303715884105728", "340282366920938463463374607431768211455"] { + let number = try JSONNumber(parsing:text), value = JSON.number(number) + #expect(try JSONValueEncoder().encode(number) == value) + #expect(try JSONValueDecoder().decode(JSONNumber.self,from:value) == number) + let payload = try JSONCodablePayload(preserving:value) + #expect(try JSONValueDecoder().decode(JSONCodablePayload.self,from:JSONValueEncoder().encode(payload)).value() == value) + } +} From 5dc4f707fb3eb7d5c66e29ee74977368d2eae435 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:30:30 +0200 Subject: [PATCH 05/24] docs: record second-wave qualification and 128-bit limits --- .../Release1.2.1/AcceptanceMatrix.json | 30 +- .../Qualification/Release1.2.1/README.md | 4 + .../Qualification/Release1.2.1/Wave2.md | 22 + .../Release1.2.1/Wave2Outcomes.json | 516 ++++++++++++++++++ 4 files changed, 560 insertions(+), 12 deletions(-) create mode 100644 Documentation/Qualification/Release1.2.1/Wave2Outcomes.json diff --git a/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json b/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json index 0317d8c..b014cff 100644 --- a/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json +++ b/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json @@ -3,7 +3,7 @@ "source_version": "1.2.0", "source_commit": "25fb04f747e583209b4c4511abf39e11ef5b4f80", "target_version": "1.2.1", - "status": "first_wave_native_qualified_remaining_plan", + "status": "second_wave_native_qualified_remaining_plan", "rows": [ { "id": "HAR-01", @@ -46,7 +46,7 @@ "work_package": "T01", "module": "JSON", "topic": "Numeric and primitive type matrix", - "status": "partially_native_qualified_explicit_primitive_subset", + "status": "partially_native_qualified_128_bit_projection_and_typed_refusal", "existing_evidence": [ "machineIntegerBoundaries", "boundedBinaryBridgePreservesLongHalfwayAndSubnormalTails", @@ -59,7 +59,8 @@ "error category/code and path", "native Int/WASI32 scope separate" ], - "first_wave_evidence": "Wave1Outcomes.json" + "first_wave_evidence": "Wave1Outcomes.json", + "second_wave_evidence": "Wave2Outcomes.json" }, { "id": "CORE-02", @@ -126,7 +127,7 @@ "work_package": "T01", "module": "JSON", "topic": "Foreign coder fidelity and fallback", - "status": "planned_extension", + "status": "native_qualified_controlled_foreign_coders", "existing_evidence": [ "qaGeneratedExactRoundTripsAcrossNativeAndForeignPayloads", "stringDictionaryProjectionRejectsScalarCollisions" @@ -138,7 +139,8 @@ "foreign limitations explicit", "Version1 wrapper remains exact", "all six kinds represented" - ] + ], + "second_wave_evidence": "Wave2Outcomes.json" }, { "id": "COD-04", @@ -343,7 +345,7 @@ "work_package": "T04", "module": "JSONStreaming", "topic": "Held source/sink backpressure", - "status": "planned_extension", + "status": "native_qualified_controlled_pull_and_handoff_subset", "existing_evidence": [ "earlyStopDoesNotPullAheadAndChunkAdmissionIsImmediate", "awaitedSinkProducesIdenticalBytesAndStatistics", @@ -356,14 +358,15 @@ "no generation while sink held", "accepted bytes cannot be recalled", "source producer cancellation not invented" - ] + ], + "second_wave_evidence": "Wave2Outcomes.json" }, { "id": "STR-02", "work_package": "T04", "module": "JSONStreaming", "topic": "Framing, partitions and terminal behavior", - "status": "planned_extension", + "status": "partially_native_qualified_held_completion_and_terminal_behavior", "existing_evidence": [ "streamEverySplitAgreesWithNativeParser", "strictNDJSONPreservesPhysicalBoundariesAndLocations", @@ -376,7 +379,8 @@ "no skipped malformed records", "exactly once completion", "no further pull after terminal error" - ] + ], + "second_wave_evidence": "Wave2Outcomes.json" }, { "id": "PAT-01", @@ -482,7 +486,7 @@ "work_package": "T07", "module": "All", "topic": "Actual platform/safety/consumer closure", - "status": "native_first_wave_qualified_other_platform_execution_pending", + "status": "second_wave_native_qualified_other_platform_execution_pending", "existing_evidence": [ "1.2.0 Native/Linux seven passed jobs", "65 WASI commands with explicit per-host scopes" @@ -496,7 +500,8 @@ "physical iPad deferred", "no 1.3 features in patch" ], - "first_wave_evidence": "Wave1Outcomes.json" + "first_wave_evidence": "Wave1Outcomes.json", + "second_wave_evidence": "Wave2Outcomes.json" } ], "proposed_targets": { @@ -517,5 +522,6 @@ "T06", "T07" ], - "first_wave_evidence": "Wave1Outcomes.json" + "first_wave_evidence": "Wave1Outcomes.json", + "second_wave_evidence": "Wave2Outcomes.json" } diff --git a/Documentation/Qualification/Release1.2.1/README.md b/Documentation/Qualification/Release1.2.1/README.md index 94a8b8b..7ca0240 100644 --- a/Documentation/Qualification/Release1.2.1/README.md +++ b/Documentation/Qualification/Release1.2.1/README.md @@ -26,3 +26,7 @@ See [actual outcomes](Wave1Outcomes.json) for receipts and remaining scope. The [second wave](Wave2.md) adds foreign-coder and 128-bit probes plus causal streaming gates. Each wave retains its own source identity and native receipts. + +Second wave natively qualified: 29 new functions, 244/173 roots, 71 QA and 18 TSan +functions. [Actual outcomes](Wave2Outcomes.json) retain unsupported typed 128-bit +Codable and platform/remaining-plan limitations. diff --git a/Documentation/Qualification/Release1.2.1/Wave2.md b/Documentation/Qualification/Release1.2.1/Wave2.md index c627dd5..9dacb93 100644 --- a/Documentation/Qualification/Release1.2.1/Wave2.md +++ b/Documentation/Qualification/Release1.2.1/Wave2.md @@ -75,3 +75,25 @@ outcomes remain. Native coverage/acceptance/QA/safety/docs are the affected lane New Linux/WASI/device runs are separate pending evidence; no runtime or package support expansion is implied. Broader T02/T03/T05/T06 work and global 98%/95% coverage targets remain open. Historical first-wave outcomes are preserved. + +## Actual native outcome + +Tested code commit: `090fffd19b40b6c9812a0add199e4ec880d24cc9`. +[Wave2Outcomes.json](Wave2Outcomes.json) binds exact source/receipt identities, +counts, coverage, qualified limitations and remaining scope. All five native +lanes passed; all approved command outcomes were 0. Coverage, native, QA and docs +CLIs exited 0. The safety CLI exited 1 when ENOSPC prevented opening the TSan log +after full ASAN passed. Source, ASAN outcomes and log hashes were revalidated; +the unchanged CI tail resumed with exit 0 after removing only completed task-owned +caches. The original nonzero result remains in local evidence. +Coverage/root debug/release/full ASAN checked 244/173; QA checked 51/3/15/2, +71 total, in debug/release/ASAN; TSan checked 4/1/1/12, 18 total. Strict memory +safety, 49 cap probes, public consumers, independent policies and 582/582 +DocC symbols passed. No new Linux/WASI/device result is implied. + +The earlier completion-alternative region counter is 0 in this run. +Its denominator remains. Separate causal assertions qualify EOF/closing syntax, +completion exactly once, accepted prefixes and terminal behavior; an execution +counter alone does not certify them. No synthetic private call or branch removal +was used to reach a percentage. Closing documentation changes are separate from +the tested implementation identity. diff --git a/Documentation/Qualification/Release1.2.1/Wave2Outcomes.json b/Documentation/Qualification/Release1.2.1/Wave2Outcomes.json new file mode 100644 index 0000000..654f2db --- /dev/null +++ b/Documentation/Qualification/Release1.2.1/Wave2Outcomes.json @@ -0,0 +1,516 @@ +{ + "schema_version": 1, + "status": "second_wave_native_qualified_remaining_plan", + "tested_code_commit": "090fffd19b40b6c9812a0add199e4ec880d24cc9", + "parent": "21d5428793c4a972f3cfbe34471739784d21c2f0", + "source_base": "25fb04f747e583209b4c4511abf39e11ef5b4f80", + "tested_source_manifest_sha256": "92cd466c4efc0f8e50650e60d56ac5cf4d948f76f014dc05ef2eb7fe8a78206d", + "production_sources_changed": false, + "runtime_dependencies_added": 0, + "new_swift_functions": 29, + "foreign_functions": 12, + "integer128_functions": 5, + "streaming_functions": 12, + "root_function_total": 244, + "parameterized_invocations": 173, + "qa_total": 71, + "tsan_total": 18, + "root_counts": { + "JSONTests": 144, + "JSONSchemaTests": 51, + "JSONRegexRuntimeTests": 12, + "JSONStreamingTests": 26, + "JSONPatchTests": 11 + }, + "authored_coverage": { + "lines": { + "count": 6929, + "covered": 6757, + "percent": 97.51767931880502 + }, + "regions": { + "count": 7691, + "covered": 7163, + "percent": 93.13483292159667 + }, + "functions": { + "count": 1143, + "covered": 1100, + "percent": 96.23797025371829 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "module_coverage": { + "JSON": { + "mapped_files": 50, + "authored": { + "lines": { + "count": 3443, + "covered": 3342, + "percent": 97.06651176299738 + }, + "regions": { + "count": 3427, + "covered": 3179, + "percent": 92.76334986868982 + }, + "functions": { + "count": 595, + "covered": 578, + "percent": 97.14285714285714 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 0, + "covered": 0, + "percent": null + }, + "regions": { + "count": 0, + "covered": 0, + "percent": null + }, + "functions": { + "count": 0, + "covered": 0, + "percent": null + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + "JSONSchema": { + "mapped_files": 32, + "authored": { + "lines": { + "count": 2410, + "covered": 2361, + "percent": 97.96680497925311 + }, + "regions": { + "count": 3088, + "covered": 2891, + "percent": 93.62046632124353 + }, + "functions": { + "count": 384, + "covered": 365, + "percent": 95.05208333333333 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 0, + "covered": 0, + "percent": null + }, + "regions": { + "count": 0, + "covered": 0, + "percent": null + }, + "functions": { + "count": 0, + "covered": 0, + "percent": null + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + "JSONRegexRuntime": { + "mapped_files": 6, + "authored": { + "lines": { + "count": 659, + "covered": 648, + "percent": 98.33080424886191 + }, + "regions": { + "count": 695, + "covered": 646, + "percent": 92.94964028776978 + }, + "functions": { + "count": 88, + "covered": 85, + "percent": 96.5909090909091 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 30, + "covered": 30, + "percent": 100.0 + }, + "regions": { + "count": 44, + "covered": 44, + "percent": 100.0 + }, + "functions": { + "count": 3, + "covered": 3, + "percent": 100.0 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + "JSONStreaming": { + "mapped_files": 10, + "authored": { + "lines": { + "count": 232, + "covered": 227, + "percent": 97.84482758620689 + }, + "regions": { + "count": 269, + "covered": 255, + "percent": 94.79553903345725 + }, + "functions": { + "count": 60, + "covered": 57, + "percent": 95.0 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 0, + "covered": 0, + "percent": null + }, + "regions": { + "count": 0, + "covered": 0, + "percent": null + }, + "functions": { + "count": 0, + "covered": 0, + "percent": null + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + "JSONPatch": { + "mapped_files": 4, + "authored": { + "lines": { + "count": 185, + "covered": 179, + "percent": 96.75675675675676 + }, + "regions": { + "count": 212, + "covered": 192, + "percent": 90.56603773584905 + }, + "functions": { + "count": 16, + "covered": 15, + "percent": 93.75 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 0, + "covered": 0, + "percent": null + }, + "regions": { + "count": 0, + "covered": 0, + "percent": null + }, + "functions": { + "count": 0, + "covered": 0, + "percent": null + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + } + }, + "foreign_bridge_coverage": { + "branches": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "functions": { + "count": 10, + "covered": 10, + "percent": 100 + }, + "instantiations": { + "count": 10, + "covered": 10, + "percent": 100 + }, + "lines": { + "count": 84, + "covered": 83, + "percent": 98.80952380952381 + }, + "mcdc": { + "count": 0, + "covered": 0, + "notcovered": 0, + "percent": 0 + }, + "regions": { + "count": 110, + "covered": 105, + "notcovered": 5, + "percent": 95.45454545454545 + } + }, + "compiler": "swift-driver version: 1.168.6 Apple Swift version 6.4 (swiftlang-6.4.0.34.1 clang-2100.3.34.1)\nTarget: arm64-apple-macosx27.0.0", + "language_mode": 6, + "os": "macOS 27.0", + "sdk": "27.2", + "architecture": "arm64", + "native_cli_exit": 0, + "safety_original_cli_exit": 1, + "safety_continuation_exit": 0, + "safety_resumption": { + "original_cli_exit": 1, + "reason": "ENOSPC; completed task caches removed", + "prefix_commands": 1, + "prior_source_and_log_hashes_verified": true, + "tail": [ + 1, + 2 + ], + "resumed_utc": "2026-10-14T09:25:59.286856+00:00" + }, + "other_lane_cli_exits": 0, + "lanes": { + "coverage": { + "status": "passed", + "receipt_sha256": "d98063182af8ef92762eca7ec7d1bf6e519873b18710d583fd022c6840873766", + "command_count": 2, + "command_exits": [ + 0, + 0 + ] + }, + "native": { + "status": "passed", + "receipt_sha256": "fa01e7b029834ad7ac6f8b1db3e775ef1ff0af8009318fa98c2113698de44297", + "command_count": 16, + "command_exits": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + }, + "qa": { + "status": "passed", + "receipt_sha256": "bb65fae6dd8c625ad36aedb726687449945bd091d91e67a2e5e6e6b6e81f9929", + "command_count": 3, + "command_exits": [ + 0, + 0, + 0 + ] + }, + "safety": { + "status": "passed", + "receipt_sha256": "6176b89f3680c0d63619b0d6879b2eda3cb10ea7ee2ad032148efaac67ca735f", + "command_count": 3, + "command_exits": [ + 0, + 0, + 0 + ] + }, + "docs": { + "status": "passed", + "receipt_sha256": "ae190313614d455bb8141e4aac10077185d10301443a39fbc297cf9a1c9acc1d", + "command_count": 1, + "command_exits": [ + 0 + ] + } + }, + "finite_metrics": 49, + "profiles": 3, + "docc_symbols": 582, + "integer128": { + "exact_number_projection": "supported_boundaries_and_overflow_qualified", + "typed_native_codable": "SDK_defaults_refuse_even_1", + "encoding_error": "EncodingError.invalidValue", + "decoding_error": "DecodingError.typeMismatch", + "lossless_alternatives": [ + "JSONNumber", + "JSON", + "JSONCodablePayload Version1" + ], + "new_support_implemented": false + }, + "streaming_completion": { + "separated_EOF_boolean_number_object": "qualified_exactly_once", + "single_document_and_final_NDJSON": "held_EOF_qualified", + "old_unobserved_region": { + "path": "Sources/JSONStreaming/JSONEventSequence.swift", + "line": 82, + "columns": [ + 97, + 124 + ], + "previous_count": 0, + "current_count": 0, + "disposition": "Retained denominator; causal completion assertions are separate from compiler execution counters. No production branch removed or artificial private caller added." + } + }, + "review_findings": [ + { + "id": "W2R01", + "status": "qualified_limitation", + "note": "Exact 128-bit projections supported; typed native Codable remains unsupported SDK defaults." + }, + { + "id": "W2R02", + "status": "test_expectation_corrected", + "note": "Initial garbage-tail assertion conflated invalidSyntax with trailingData. Final case checks both exact codes/offsets/terminal behavior; original nonzero receipt retained." + }, + { + "id": "W2R03", + "status": "causal_contracts_qualified", + "note": "One controlled pull/handoff, held EOF, acceptance and terminal failure proven through public observations. No caller instruction/RSS/hidden-generation claim." + } + ], + "pending": [ + "Global 98% line/95% region targets", + "T02 source/preview and wider budget cases", + "T03 combined schema/regex decisions", + "T04 remaining framing/Patch matrices", + "T05 public end-to-end workflows", + "T06 adversarial/mutation assurance", + "New-test Linux/WASI/device execution", + "Separate retained-unused-helper review" + ], + "remote_actions": [], + "release_actions": [], + "sibling_edits": 0 +} From ae609f729e0e9a52e26c4941122d8e62d61512cb Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:46:57 +0200 Subject: [PATCH 06/24] test: qualify portable QA and inspect completion alternatives --- .../Qualification/Release1.2.1/Portability.md | 62 +++++++++++++++++ .../Qualification/Release1.2.1/README.md | 5 ++ .../Qualification/TestDiscovery.json | 6 +- Scripts/README.md | 2 + Scripts/ci.py | 25 +++++-- Scripts/test_qualification_harness.py | 15 +++- Scripts/wasm_support.py | 16 ++++- .../CompletionMatrixTests.swift | 68 +++++++++++++++++++ 8 files changed, 188 insertions(+), 11 deletions(-) create mode 100644 Documentation/Qualification/Release1.2.1/Portability.md create mode 100644 Tests/JSONStreamingTests/CompletionMatrixTests.swift diff --git a/Documentation/Qualification/Release1.2.1/Portability.md b/Documentation/Qualification/Release1.2.1/Portability.md new file mode 100644 index 0000000..83e040d --- /dev/null +++ b/Documentation/Qualification/Release1.2.1/Portability.md @@ -0,0 +1,62 @@ +# 1.2.1 portability and completion review + +This increment preserves the published 1.2.0 implementation and both earlier +test waves. It adds one Swift Testing function with 566 fixed completion cases: +eleven root spellings, both framing modes, three trailing-whitespace forms and +every two-chunk byte split, including empty chunks and a split within UTF-8. +The fixed syntax oracles assert the complete event sequence, exactly one selected +completion kind, cumulative record count and repeated terminal nil. + +Current discovery is 245 root functions and 173 parameterized invocations, +72 QA functions and 19 shared-state functions. The 566 inner iterations are +reported separately; they are not 566 discovered Swift Testing functions. + +## Scoped platform lanes + +`Scripts/ci.py --lane wasm-qa` uses the existing pinned Swift 6.4.0 WASI SDK, +Node 26.5.0 and WasmKit 0.3.1. Node executes complete debug/release roots. +Both hosts execute the exact affected QA functions in both configurations. +Strict memory safety compilation remains required. The scoped lane does not +claim fresh independent-suite, cap-consumer, resource-deployment or performance +qualification. The existing full `wasm` lane retains those gates unchanged. + +`Scripts/ci.py --lane linux-qa` requires actual Swift 6.4+ Ubuntu 24.04/glibc. +It executes complete debug/release roots, affected ASan and shared-state TSan +cases plus strict memory safety. It starts Swift Testing only, preserving the +documented upstream empty-XCTest-runner leak policy without suppressions. +The full Linux lanes and their independent inputs remain unchanged. + +Actual results and any local staging blockers are recorded separately; a +prepared lane or a Mac-host Linux preflight rejection is not Linux execution. +The root dependency graph, four products, private regex target, Apple baselines, +finite profiles, notices and fixture policies are unchanged. + +## Completion coverage disposition + +The zero-count alternative is the final `.recordCompleted(summary())` expression +in `Sources/JSONStreaming/JSONEventSequence.swift`, line 82. Source inspection +establishes why that alternative is unreachable for successful current inputs: + +1. `JSONTokenMachine.hasCompleteRoot` is `rootComplete && lexer.idle`. + Every transition setting `rootComplete` returns an event to the driver. +2. At the next loop entry, whitespace-separated framing with a complete root + takes the earlier framing branch. It certifies a whitespace boundary or EOF + and returns record completion there. +3. If the final EOF branch is entered with an incomplete root, `finish()` either + throws or returns the EOF-finalized numeric scalar. The scalar is returned; + the next invocation takes the earlier framing branch. The lexer can return + nil at EOF only in its idle state; token-machine finish then requires an + already complete root, which would have been handled by step 2. +4. Byte-source pulls do not modify token-machine state. After a record reset, + `betweenRecords` permits terminal nil, not a second completion. + +Thus the successful final completion expression serves single-document framing. +The new public matrix covers numeric EOF finalization, all scalar root kinds, +empty/nested containers and closing-delimiter transitions under both modes. +The existing held-EOF tests separately establish causal completion/backpressure. + +Disposition: retain the defensive expression and its unadjusted denominator; +record the structural explanation rather than fabricate a private state or +remove code to improve percentages. This closes semantic investigation of this +region, not a claim of 100% region/branch or exhaustive race coverage. Revisit +the invariant if framing or token-machine transitions change. diff --git a/Documentation/Qualification/Release1.2.1/README.md b/Documentation/Qualification/Release1.2.1/README.md index 7ca0240..6a0f568 100644 --- a/Documentation/Qualification/Release1.2.1/README.md +++ b/Documentation/Qualification/Release1.2.1/README.md @@ -30,3 +30,8 @@ streaming gates. Each wave retains its own source identity and native receipts. Second wave natively qualified: 29 new functions, 244/173 roots, 71 QA and 18 TSan functions. [Actual outcomes](Wave2Outcomes.json) retain unsupported typed 128-bit Codable and platform/remaining-plan limitations. + +The [portability/completion follow-up](Portability.md) adds a 566-case public +completion matrix and narrowly scoped Linux/WASI QA lanes. Its current +discovery is 245/173 roots, 72 QA and 19 shared-state functions; actual platform +results remain separate from the historical native waves. diff --git a/Documentation/Qualification/TestDiscovery.json b/Documentation/Qualification/TestDiscovery.json index 597d1c3..daed2ab 100644 --- a/Documentation/Qualification/TestDiscovery.json +++ b/Documentation/Qualification/TestDiscovery.json @@ -4,17 +4,17 @@ "JSONTests": 144, "JSONSchemaTests": 51, "JSONRegexRuntimeTests": 12, - "JSONStreamingTests": 26, + "JSONStreamingTests": 27, "JSONPatchTests": 11 }, "parameterized_invocations": 173, "qa_functions": { "JSONTests": 51, "JSONSchemaTests": 3, - "JSONStreamingTests": 15, + "JSONStreamingTests": 16, "JSONPatchTests": 2 }, - "shared_state_functions": {"JSONTests": 4, "JSONSchemaTests": 1, "JSONRegexRuntimeTests": 1, "JSONStreamingTests": 12}, + "shared_state_functions": {"JSONTests": 4, "JSONSchemaTests": 1, "JSONRegexRuntimeTests": 1, "JSONStreamingTests": 13}, "coverage_unmapped": { "Sources/JSON/Pointer/JSONLocationComponent.swift": "declaration-only enum", "Sources/JSONSchema/Output/JSONValidationOutput.swift": "declaration-only enum", diff --git a/Scripts/README.md b/Scripts/README.md index a68ac12..fbe10eb 100644 --- a/Scripts/README.md +++ b/Scripts/README.md @@ -11,7 +11,9 @@ separate from the dependency-free library. Native lanes fail on unsupported host | `python3 Scripts/ci.py --lane safety` | Address/Thread Sanitizer and strict memory safety | | `python3 Scripts/ci.py --lane linux` | Actual Linux preflight, all four products, exact root/cap/corpus/consumer counts | | `python3 Scripts/ci.py --lane linux-safety` | Linux Address/Thread Sanitizer and strict memory safety | +| `python3 Scripts/ci.py --lane linux-qa` | Test-only increment: complete debug/release roots, affected ASan/TSan and strict memory safety; no independent-suite refresh claim | | `python3 Scripts/ci.py --lane wasm` | Matched Swift 6.4 WASI debug/release, exact root/corpus/cap/consumer counts, two-host public parity and structured resource failures | +| `python3 Scripts/ci.py --lane wasm-qa` | Test-only increment: complete Node debug/release roots, exact affected QA on Node and WasmKit, strict memory safety; no independent-suite/consumer refresh claim | | `python3 Scripts/qualify_linux_reproduction.py --output PATH` | Actual source archive, byte-equal rebuild, clean extraction, build/tests and public consumer | | `python3 Scripts/ci.py --lane docs` | Four DocC catalogs, authored API/parameter coverage, executed README examples and verified static Pages site | | `python3 Scripts/build_documentation_site.py` | Transform already-qualified DocC archives and verify static routes/assets | diff --git a/Scripts/ci.py b/Scripts/ci.py index 158206f..60060c1 100644 --- a/Scripts/ci.py +++ b/Scripts/ci.py @@ -16,16 +16,29 @@ def snapshot(): def commands(lane, output): # The qualification executable resolves output paths from its repository root. output = output.relative_to(ROOT) - if lane == 'wasm': + if lane in ('wasm', 'wasm-qa'): return [ ['python3', '-B', 'Scripts/public_check.py'], - ['python3', '-B', 'Scripts/wasm_support.py', '--output', str(output / 'probes')], + ['python3', '-B', 'Scripts/wasm_support.py', '--output', str(output / 'probes')] + + (['--scope', 'root-qa'] if lane == 'wasm-qa' else []), ] if lane == 'coverage': return [ ['python3', '-B', 'Scripts/test_qualification_harness.py'], ['python3', '-B', 'Scripts/coverage.py', '--output', str(output / 'export')], ] + if lane == 'linux-qa': + # A test-only increment: complete roots in both configurations, then + # affected QA/shared-state safety. This is not independent-suite refresh. + return [ + ['python3', '-B', 'Scripts/public_check.py'], + ['python3', '-B', 'Scripts/test_qualification_harness.py'], + ['swift', 'test', '--disable-xctest'], + ['swift', 'test', '-c', 'release', '--disable-xctest'], + commands('qa', ROOT / output)[2] + ['--disable-xctest'], + commands('safety', ROOT / output)[1] + ['--disable-xctest'], + commands('safety', ROOT / output)[2], + ] if lane == 'linux': # Root behavior/corpora remain required. SwiftData and Apple SDK/lifetime # probes are separate Apple acceptance; they never become Linux passes. @@ -123,7 +136,7 @@ def check_result(command, code, log): def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--lane', choices=['native', 'safety', 'docs', 'r0', 'qa', 'coverage', 'linux', 'linux-safety', 'wasm'], required=True) + parser.add_argument('--lane', choices=['native', 'safety', 'docs', 'r0', 'qa', 'coverage', 'linux', 'linux-safety', 'linux-qa', 'wasm', 'wasm-qa'], required=True) parser.add_argument('--output') args = parser.parse_args() output = ROOT / (args.output or '.build/ci/' + args.lane) @@ -144,7 +157,7 @@ def save(): log_path = output / ('command-%02d.log' % index) with log_path.open('w') as stream: try: - result = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT, timeout=21600 if args.lane == 'wasm' else 1800) + result = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT, timeout=21600 if args.lane in ('wasm', 'wasm-qa') else 1800) code = result.returncode except (OSError, subprocess.TimeoutExpired) as error: stream.write('\n' + str(error) + '\n') @@ -171,11 +184,13 @@ def save(): receipt['status'] = 'failed' receipt['evidence_error'] = str(error) print('Native evidence failed: ' + str(error), flush=True) - if receipt['status'] == 'passed' and args.lane == 'wasm': + if receipt['status'] == 'passed' and args.lane in ('wasm', 'wasm-qa'): try: wasi = json.loads((output / 'probes/wasi.json').read_text()) if wasi['status'] != 'passed' or wasi['root_counts'] != ROOT_COUNTS: raise ValueError('Missing WASI qualification boundary') + if args.lane == 'wasm-qa' and (wasi['scope'] != 'root-qa' or wasi['qa_counts'] != DISCOVERY['qa_functions']): + raise ValueError('Missing scoped WASI QA boundary') receipt['materialized_wasi'] = {'path':'probes/wasi.json','sha256':hashlib.sha256((output / 'probes/wasi.json').read_bytes()).hexdigest()} except (OSError,KeyError,ValueError,TypeError) as error: receipt['status'] = 'failed'; receipt['evidence_error'] = str(error) diff --git a/Scripts/test_qualification_harness.py b/Scripts/test_qualification_harness.py index d24e2a5..881a922 100644 --- a/Scripts/test_qualification_harness.py +++ b/Scripts/test_qualification_harness.py @@ -7,8 +7,8 @@ import tempfile import unittest from coverage import executable_objects, validate_export -from test_discovery import ROOT_COUNTS, PARAMETER_INVOCATIONS, check_root -from ci import check_result +from test_discovery import DISCOVERY, ROOT_COUNTS, PARAMETER_INVOCATIONS, check_root +from ci import check_result, commands class HarnessTests(unittest.TestCase): def setUp(self): @@ -76,6 +76,17 @@ def test_exact_discovery_rejects_missing_duplicate_and_changed_parameters(self): self.assertFalse(check_root(broken)) self.assertFalse(check_result(['swift','test'],0,broken)[0]) self.assertFalse(check_result(['swift','test'],1,valid)[0]) + # Scoped portability lanes retain exact discovery and safety rather + # than describing an exit-zero subset as full qualification. + output = pathlib.Path(__file__).resolve().parents[1] / 'Artifacts/HarnessProbe' + linux = commands('linux-qa', output) + self.assertEqual(sum(c[:2] == ['swift','test'] for c in linux), 4) + self.assertTrue(all('--disable-xctest' in c for c in linux if c[:2] == ['swift','test'])) + qa = '\n'.join('Test run with '+str(n)+' tests in 0 suites passed' for n in DISCOVERY['qa_functions'].values()) + self.assertTrue(check_result(linux[4], 0, qa)[0]) + self.assertFalse(check_result(linux[4], 0, qa+'\nTest run with 1 test in 0 suites passed')[0]) + self.assertIn('--scope', commands('wasm-qa', output)[1]) + self.assertNotIn('--scope', commands('wasm', output)[1]) def test_darwin_executable_membership_is_exact(self): for name in ROOT_COUNTS: diff --git a/Scripts/wasm_support.py b/Scripts/wasm_support.py index 339abce..212e199 100644 --- a/Scripts/wasm_support.py +++ b/Scripts/wasm_support.py @@ -3,7 +3,7 @@ """Qualify the full-language wasm32 WASI command target with materialized outcomes.""" import argparse, hashlib, json, os, pathlib, re, shutil, signal, subprocess, time from archive_candidate import files -from test_discovery import ROOT_COUNTS, PARAMETER_INVOCATIONS +from test_discovery import DISCOVERY, ROOT_COUNTS, PARAMETER_INVOCATIONS ROOT = pathlib.Path(__file__).resolve().parents[1] SDK = 'swift-6.4.0-RELEASE_wasm' @@ -13,12 +13,18 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--output', required=True) + parser.add_argument('--scope', choices=['full', 'root-qa'], default='full') args = parser.parse_args() output = ROOT / args.output; output.mkdir(parents=True, exist_ok=True) result = {'schema_version': 1, 'status': 'running', 'target': 'wasm32-unknown-wasip1', 'sdk': SDK, 'commands': [], 'modules': [], 'root_counts': COUNTS, 'wasmkit_scope': '13 public consumer functions, 49 caps, 20 rooted vectors, debug/release command parity', 'node_scope': 'complete debug/release root suites, caps, public consumers and independent qualification'} + result['scope'] = args.scope + if args.scope == 'root-qa': + result.update(qa_counts=DISCOVERY['qa_functions'], + wasmkit_scope='exact affected QA functions in debug/release; no full root/corpus claim', + node_scope='complete debug/release root suites and exact affected QA; no separate consumer/cap/independent qualification refresh') def save(): (output / 'wasi.json').write_text(json.dumps(result, indent=2) + '\n') def run(name, command, check=lambda c, t: c == 0, timeout=1800): @@ -72,6 +78,12 @@ def invoke(engine, path, mode='command', arguments=(), repo=True, resources=True text = run('node-' + config + '-' + name, invoke('node', roots[name], 'tests'), tests(count)) invocations += sum(int(n) for n in re.findall(r'with (\d+) test cases passed', text)) if invocations != PARAMETER_INVOCATIONS: raise ValueError('Changed root parameterized discovery: ' + str(invocations)) + if args.scope == 'root-qa': + for name, count in DISCOVERY['qa_functions'].items(): + for engine in ['node', 'wasmkit']: + run(engine + '-' + config + '-qa-' + name, + invoke(engine, roots[name], 'tests', arguments=['--filter', 'qa']), tests(count)) + continue for package, name, scratch in [('Tools/WasmAudit', 'WasmAudit', '.build/wasm-audit-consumer'), ('Tools/Qualification', 'Qualification', '.build/wasm-qualification'), ('Tools/MCPReadiness', 'MCPReadiness', '.build/wasm-readiness')]: @@ -112,6 +124,8 @@ def check(c, t, mode=mode): for name,count in [('authored_public_semantics',82),('public_budget_and_counter_boundaries',9),('deterministic_malformed_utf8_replay',4396)]: if authored[name]['observed'] != count or authored[name]['failed']: raise ValueError('Changed authored outcomes') run('strict-memory', [swift,'build','--swift-sdk',SDK,'--scratch-path','.build/wasm-strict','-Xswiftc','-strict-memory-safety','-Xswiftc','-warnings-as-errors']) + if args.scope == 'root-qa': + result['status'] = 'passed'; save(); return 0 path = modules['release','WasmAudit'] for engine in ['node','wasmkit']: run(engine + '-missing-resources', invoke(engine,path,'missing-resources',repo=False,resources=False), lambda c,t:c == 0 and 'MISSING_RESOURCES structured_metaSchema' in t) diff --git a/Tests/JSONStreamingTests/CompletionMatrixTests.swift b/Tests/JSONStreamingTests/CompletionMatrixTests.swift new file mode 100644 index 0000000..1a51212 --- /dev/null +++ b/Tests/JSONStreamingTests/CompletionMatrixTests.swift @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: Apache-2.0 +import Testing +import JSON +import JSONStreaming + +private enum SyntaxObservation: Equatable { + case objectStart, objectEnd, arrayStart, arrayEnd + case key(JSONKey), scalar(JSON) +} + +@Test(.timeLimit(.minutes(1))) func qaStreamingEveryRootEndingCompletesWithTheSelectedFramingAtEverySplit() async throws { + // Fixed syntax oracles, rather than comparing two consumers of one parser. + // Include numbers finalized only by EOF and containers completed by a delimiter. + let cases: [(String, [SyntaxObservation])] = [ + ("null", [.scalar(.null)]), + ("true", [.scalar(true)]), + ("false", [.scalar(false)]), + ("0", [.scalar(0)]), + ("-17.25e+2", [.scalar(-1725)]), + (#""e\u0301""#, [.scalar("e\u{301}")]), + ("\"😀\"", [.scalar("😀")]), + ("[]", [.arrayStart, .arrayEnd]), + ("{}", [.objectStart, .objectEnd]), + (#"[1,{"k":2}]"#, [.arrayStart, .scalar(1), .objectStart, .key("k"), .scalar(2), .objectEnd, .arrayEnd]), + (#"{"a":[true]}"#, [.objectStart, .key("a"), .arrayStart, .scalar(true), .arrayEnd, .objectEnd]), + ] + var executions = 0 + for framing in [JSONFraming.singleDocument, .whitespaceSeparatedValues] { + for (root, expected) in cases { + for suffix in ["", " ", " \t\r\n"] { + let text = root + suffix + for split in 0...text.utf8.count { + var iterator = JSONEventSequence(Chunks(text, split: split), options: .init(framing: framing)).makeAsyncIterator() + var syntax: [SyntaxObservation] = [] + var documents = 0, records = 0 + while let event = try await iterator.next() { + switch event { + case .documentCompleted(let summary): + #expect(syntax == expected && summary.records == 1) + documents += 1 + case .recordCompleted(let summary): + #expect(syntax == expected && summary.records == 1) + records += 1 + default: + #expect(documents == 0 && records == 0) + switch event { + case .startObject: syntax.append(.objectStart) + case .endObject: syntax.append(.objectEnd) + case .startArray: syntax.append(.arrayStart) + case .endArray: syntax.append(.arrayEnd) + case .key(let text): syntax.append(.key(JSONKey(text))) + case .scalar(let value): syntax.append(.scalar(value)) + default: Issue.record("Unexpected completion in syntax branch") + } + } + } + #expect(syntax == expected) + #expect(documents == (framing == .singleDocument ? 1 : 0)) + #expect(records == (framing == .whitespaceSeparatedValues ? 1 : 0)) + #expect(try await iterator.next() == nil) + #expect(try await iterator.next() == nil) + executions += 1 + } + } + } + } + #expect(executions == 566) +} From 620fc6511921e62162290b3d2204d1a66f424769 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:53:51 +0200 Subject: [PATCH 07/24] test: give foreign fidelity oracle an explicit portable width --- Tests/JSONTests/ForeignCodableTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/JSONTests/ForeignCodableTests.swift b/Tests/JSONTests/ForeignCodableTests.swift index 588e292..35b0f22 100644 --- a/Tests/JSONTests/ForeignCodableTests.swift +++ b/Tests/JSONTests/ForeignCodableTests.swift @@ -273,7 +273,7 @@ private struct AlreadyChangedForeignNumber: Decodable { @Test func qaForeignCoderFidelityCannotRecoverPreviouslyChangedNumbers() throws { let source = try JSONParser().parse("9007199254740993") let decoded = try JSONValueDecoder().decode(AlreadyChangedForeignNumber.self,from:source) - #expect(decoded.number == JSONNumber(integer:9_007_199_254_740_992)) + #expect(decoded.number == JSONNumber(integer:Int64(9_007_199_254_740_992))) #expect(JSON.number(decoded.number) != source) #expect(try JSONValueDecoder().decode(JSON.self,from:source) == source) } From bbc36070502554286a45fe4c49e8d3dde802c470 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:07:35 +0200 Subject: [PATCH 08/24] test: enforce WASI streaming deadlines from the execution host --- .../Qualification/Release1.2.1/Portability.md | 17 +++++++++++++ Scripts/wasm_support.py | 9 ++++--- .../CompletionMatrixTests.swift | 9 ++++++- .../ControlledStreamingTests.swift | 24 +++++++++---------- .../StreamingTestPolicy.swift | 16 +++++++++++++ 5 files changed, 59 insertions(+), 16 deletions(-) create mode 100644 Tests/JSONStreamingTests/StreamingTestPolicy.swift diff --git a/Documentation/Qualification/Release1.2.1/Portability.md b/Documentation/Qualification/Release1.2.1/Portability.md index 83e040d..537adb0 100644 --- a/Documentation/Qualification/Release1.2.1/Portability.md +++ b/Documentation/Qualification/Release1.2.1/Portability.md @@ -31,6 +31,23 @@ prepared lane or a Mac-host Linux preflight rejection is not Linux execution. The root dependency graph, four products, private regex target, Apple baselines, finite profiles, notices and fixture policies are unchanged. +The first actual WASI build rejected an untyped large integer oracle inferred +as 32-bit Int. Its explicit Int64 replacement preserves the same exact expected +value. The next run exposed a pinned Swift Testing timer issue: thirteen async +cases with a one-minute trait timed out together before other async work +progressed. A two-case SDK-only package, with no JSON imports, reproduced this +on both hosts: an immediately returning timed body failed after 60 seconds; +the untimed body passed immediately. + +`StreamingTestPolicy.deadline` therefore preserves the one-minute Swift Testing +trait on native/Linux, and supplies an empty tag trait on WASI. No case or +assertion is disabled. WASI streaming root and affected-QA runners instead have +an interruptible 120-second host process-group watchdog, recorded per command. +The host enforces a real termination deadline even if guest cooperative tasks +stop yielding; it does not grant library operations an elapsed-time budget. +Keep the SDK-only failing/passing receipts and revisit this adjustment with a +new Swift Testing/SDK pin. + ## Completion coverage disposition The zero-count alternative is the final `.recordCompleted(summary())` expression diff --git a/Scripts/wasm_support.py b/Scripts/wasm_support.py index 212e199..1f431c6 100644 --- a/Scripts/wasm_support.py +++ b/Scripts/wasm_support.py @@ -25,6 +25,7 @@ def main(): result.update(qa_counts=DISCOVERY['qa_functions'], wasmkit_scope='exact affected QA functions in debug/release; no full root/corpus claim', node_scope='complete debug/release root suites and exact affected QA; no separate consumer/cap/independent qualification refresh') + result['streaming_timeout_policy'] = 'Identical streaming assertions on every platform. Native/Linux Swift Testing limits stay at one minute. The pinned WASI SDK-only timer reproduction blocks its cooperative executor; streaming runners instead use an interruptible 120-second host process-group watchdog.' def save(): (output / 'wasi.json').write_text(json.dumps(result, indent=2) + '\n') def run(name, command, check=lambda c, t: c == 0, timeout=1800): @@ -39,7 +40,7 @@ def run(name, command, check=lambda c, t: c == 0, timeout=1800): passed = check(code, text) path.write_text(text.replace(str(ROOT), '.')) result['commands'].append({'name': name, 'command': [x.replace(str(ROOT), '.') for x in command], - 'exit_code': code, 'status': 'passed' if passed else 'failed', 'seconds': time.monotonic() - started, + 'exit_code': code, 'status': 'passed' if passed else 'failed', 'seconds': time.monotonic() - started, 'timeout_seconds': timeout, 'log': path.relative_to(ROOT).as_posix(), 'log_sha256': hashlib.sha256(path.read_bytes()).hexdigest()}) save() if not passed: raise ValueError('Failed actual outcome: ' + name) @@ -75,14 +76,16 @@ def invoke(engine, path, mode='command', arguments=(), repo=True, resources=True roots = {name: binary(scratch, config, name + '-test-runner') for name in COUNTS} invocations = 0 for name, count in COUNTS.items(): - text = run('node-' + config + '-' + name, invoke('node', roots[name], 'tests'), tests(count)) + text = run('node-' + config + '-' + name, invoke('node', roots[name], 'tests'), tests(count), + timeout=120 if name == 'JSONStreamingTests' else 1800) invocations += sum(int(n) for n in re.findall(r'with (\d+) test cases passed', text)) if invocations != PARAMETER_INVOCATIONS: raise ValueError('Changed root parameterized discovery: ' + str(invocations)) if args.scope == 'root-qa': for name, count in DISCOVERY['qa_functions'].items(): for engine in ['node', 'wasmkit']: run(engine + '-' + config + '-qa-' + name, - invoke(engine, roots[name], 'tests', arguments=['--filter', 'qa']), tests(count)) + invoke(engine, roots[name], 'tests', arguments=['--filter', 'qa']), tests(count), + timeout=120 if name == 'JSONStreamingTests' else 1800) continue for package, name, scratch in [('Tools/WasmAudit', 'WasmAudit', '.build/wasm-audit-consumer'), ('Tools/Qualification', 'Qualification', '.build/wasm-qualification'), diff --git a/Tests/JSONStreamingTests/CompletionMatrixTests.swift b/Tests/JSONStreamingTests/CompletionMatrixTests.swift index 1a51212..73e4865 100644 --- a/Tests/JSONStreamingTests/CompletionMatrixTests.swift +++ b/Tests/JSONStreamingTests/CompletionMatrixTests.swift @@ -8,7 +8,14 @@ private enum SyntaxObservation: Equatable { case key(JSONKey), scalar(JSON) } -@Test(.timeLimit(.minutes(1))) func qaStreamingEveryRootEndingCompletesWithTheSelectedFramingAtEverySplit() async throws { +@Test(StreamingTestPolicy.deadline) func qaStreamingEveryRootEndingCompletesWithTheSelectedFramingAtEverySplit() async throws { + let traits = try #require(Test.current).traits + #if os(WASI) + #expect(!traits.contains { $0 is TimeLimitTrait }) + #else + let deadline = try #require(traits.compactMap { $0 as? TimeLimitTrait }.first) + #expect(deadline.timeLimit == .seconds(60)) + #endif // Fixed syntax oracles, rather than comparing two consumers of one parser. // Include numbers finalized only by EOF and containers completed by a delimiter. let cases: [(String, [SyntaxObservation])] = [ diff --git a/Tests/JSONStreamingTests/ControlledStreamingTests.swift b/Tests/JSONStreamingTests/ControlledStreamingTests.swift index ce9fc4d..9a44460 100644 --- a/Tests/JSONStreamingTests/ControlledStreamingTests.swift +++ b/Tests/JSONStreamingTests/ControlledStreamingTests.swift @@ -88,7 +88,7 @@ private func arrayTask(_ gate: SourceGate) -> Task { } } -@Test(.timeLimit(.minutes(1))) func qaStreamingRootCompletionWaitsForEOF() async throws { +@Test(StreamingTestPolicy.deadline) func qaStreamingRootCompletionWaitsForEOF() async throws { let gate = SourceGate(["[0] \r\n"]), task = arrayTask(gate) await gate.waitForHeldOrFinished() let before = await gate.snapshot() @@ -101,7 +101,7 @@ private func arrayTask(_ gate: SourceGate) -> Task { #expect(after.documentCompletions == 1 && after.pulls == 2) #expect(after.summary?.records == 1 && after.summary?.statistics[.totalScannedNodes] == 2) } -@Test(.timeLimit(.minutes(1))) func qaStreamingClosingSyntaxAndWhitespacePrecedeCompletion() async { +@Test(StreamingTestPolicy.deadline) func qaStreamingClosingSyntaxAndWhitespacePrecedeCompletion() async { let gate = SourceGate(["[0"]), task = arrayTask(gate) await gate.waitForHeldOrFinished() let before = await gate.snapshot() @@ -112,7 +112,7 @@ private func arrayTask(_ gate: SourceGate) -> Task { #expect(result.error == nil && result.terminalNil && after.values == [0]) #expect(after.documentCompletions == 1 && after.pulls == 3) } -@Test(.timeLimit(.minutes(1))) func qaStreamingMalformedTailNeverCompletesAndStopsPulling() async throws { +@Test(StreamingTestPolicy.deadline) func qaStreamingMalformedTailNeverCompletesAndStopsPulling() async throws { // An invalid token fails lexing; a valid second JSON root fails framing. // Keep both exact diagnoses rather than accepting any JSONFailure. for (tail, code) in [(" garbage", JSONFailure.Code.invalidSyntax), (" false", .trailingData)] { @@ -128,7 +128,7 @@ private func arrayTask(_ gate: SourceGate) -> Task { #expect(result.terminalNil && final.documentCompletions == 0 && final.pulls == 2) } } -@Test(.timeLimit(.minutes(1))) func qaStreamingSourceErrorAfterElementRemainsOperationalAndTerminal() async { +@Test(StreamingTestPolicy.deadline) func qaStreamingSourceErrorAfterElementRemainsOperationalAndTerminal() async { let gate = SourceGate(["[0]"]), task = arrayTask(gate) await gate.waitForHeldOrFinished(); await gate.reply(.failure) let result = await task.value; await gate.close() @@ -136,7 +136,7 @@ private func arrayTask(_ gate: SourceGate) -> Task { let final = await gate.snapshot() #expect(final.values == [0] && final.documentCompletions == 0 && final.pulls == 2) } -@Test(.timeLimit(.minutes(1))) func qaStreamingCancellationWhileEOFHeldSuppressesCompletion() async { +@Test(StreamingTestPolicy.deadline) func qaStreamingCancellationWhileEOFHeldSuppressesCompletion() async { let gate = SourceGate(["[0]"]), task = arrayTask(gate) await gate.waitForHeldOrFinished(); task.cancel(); await gate.reply(.eof) let result = await task.value; await gate.close() @@ -144,7 +144,7 @@ private func arrayTask(_ gate: SourceGate) -> Task { let final = await gate.snapshot() #expect(final.values == [0] && final.documentCompletions == 0 && final.pulls == 2) } -@Test(.timeLimit(.minutes(1))) func qaStreamingFirstElementReturnDoesNotAwaitAnotherSourceChunk() async { +@Test(StreamingTestPolicy.deadline) func qaStreamingFirstElementReturnDoesNotAwaitAnotherSourceChunk() async { let gate = SourceGate(["[0,"]) let task = Task { var iterator = JSONTopLevelArraySequence(ControlledSource(gate:gate)).makeAsyncIterator() @@ -158,7 +158,7 @@ private func arrayTask(_ gate: SourceGate) -> Task { #expect(state.finished && !state.held && state.pulls == 1) await gate.close(); #expect(await task.value) } -@Test(.timeLimit(.minutes(1))) func qaStreamingSeparatedEventsFinishAtEOFExactlyOnce() async { +@Test(StreamingTestPolicy.deadline) func qaStreamingSeparatedEventsFinishAtEOFExactlyOnce() async { for text in ["false", "17", "{}"] { let gate = SourceGate([text]) let task = Task { @@ -186,7 +186,7 @@ private func arrayTask(_ gate: SourceGate) -> Task { #expect(final.summary?.records == 1) } } -@Test(.timeLimit(.minutes(1))) func qaStreamingSingleValueAndFinalNDJSONRecordWaitForEOF() async { +@Test(StreamingTestPolicy.deadline) func qaStreamingSingleValueAndFinalNDJSONRecordWaitForEOF() async { for ndjson in [false,true] { let gate = SourceGate([ndjson ? "1\n2" : "2"]) let task = Task { @@ -243,7 +243,7 @@ private func writerTask(_ gate:SinkGate,limits:JSONLimits = .balanced) -> Task Task Task Date: Wed, 14 Oct 2026 12:31:11 +0200 Subject: [PATCH 09/24] test: use atomic coverage counters and reject saturation --- Documentation/Qualification/Release1.2.1/Portability.md | 9 +++++++++ Scripts/coverage.py | 9 ++++++++- Scripts/test_qualification_harness.py | 4 ++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/Documentation/Qualification/Release1.2.1/Portability.md b/Documentation/Qualification/Release1.2.1/Portability.md index 537adb0..c9c0440 100644 --- a/Documentation/Qualification/Release1.2.1/Portability.md +++ b/Documentation/Qualification/Release1.2.1/Portability.md @@ -72,6 +72,15 @@ The new public matrix covers numeric EOF finalization, all scalar root kinds, empty/nested containers and closing-delimiter transitions under both modes. The existing held-EOF tests separately establish causal completion/backpressure. +The initial follow-up export reported the alternative as covered with an +Int64.max derived counter, while the preceding condition and true branch differed +by one execution. It therefore did not establish execution of the alternative. +Earlier exports also contain saturated async/throwing segment counts. The +installed compiler accepts the profiling-only LLVM atomic-counter option; a +minimal emitted-IR probe confirms `atomicrmw` increments. Coverage now uses this +option for its concurrent tests and rejects saturated library segment counts. +Ordinary builds, package settings and safety/behavior assertions are unchanged. + Disposition: retain the defensive expression and its unadjusted denominator; record the structural explanation rather than fabricate a private state or remove code to improve percentages. This closes semantic investigation of this diff --git a/Scripts/coverage.py b/Scripts/coverage.py index 6497f7e..2b1dca3 100644 --- a/Scripts/coverage.py +++ b/Scripts/coverage.py @@ -48,6 +48,11 @@ def validate_export(export, root=ROOT, policy=DISCOVERY): continue if name not in expected or name in mapped: raise ValueError('Unexpected or duplicate library mapping: ' + name) + # LLVM saturates negative derived counts at Int64.max. Such a + # segment is not evidence of trillions of executions or coverage. + # Concurrent Swift invocations need atomic instrumentation counters. + if any(segment[3] and segment[2] >= 2**63 - 1 for segment in record.get('segments', [])): + raise ValueError('Saturated LLVM coverage counter: ' + name) for kind in ['lines', 'regions', 'functions', 'branches', 'mcdc']: count = record['summary'][kind]['count'] covered = record['summary'][kind]['covered'] @@ -122,7 +127,9 @@ def run(command, name, binary=False): raise ValueError('Swift toolchain llvm-cov required') llvm = [str(tool)] receipt['llvm'] = run(llvm + ['--version'], 'llvm.log').strip() - tests = run(['swift', 'test', '--enable-code-coverage', '--disable-xctest'], 'tests.log') + profile_flags = ['-Xswiftc', '-Xllvm', '-Xswiftc', '-instrprof-atomic-counter-update-all'] + receipt['counter_policy'] = 'Atomic profiling increments for concurrent tests. Reject saturated LLVM segment counts rather than count them as executed paths. Profiling flags do not alter package manifests or ordinary product builds.' + tests = run(['swift', 'test', '--enable-code-coverage', '--disable-xctest', *profile_flags], 'tests.log') if not check_root(tests): raise ValueError('Exact root discovery/parameterized cardinality changed') receipt.update(root_counts=ROOT_COUNTS, parameterized_invocations=PARAMETER_INVOCATIONS) diff --git a/Scripts/test_qualification_harness.py b/Scripts/test_qualification_harness.py index 881a922..bd33b89 100644 --- a/Scripts/test_qualification_harness.py +++ b/Scripts/test_qualification_harness.py @@ -60,6 +60,10 @@ def test_impossible_counters_are_rejected(self): broken['data'][0]['files'][0]['summary']['lines'].update(count=count,covered=covered) with self.assertRaisesRegex(ValueError, 'Invalid coverage counter'): validate_export(broken, self.root, self.policy) + broken = copy.deepcopy(self.export) + broken['data'][0]['files'][0]['segments'] = [[1,1,2**63-1,True,True,False]] + with self.assertRaisesRegex(ValueError, 'Saturated LLVM coverage counter'): + validate_export(broken, self.root, self.policy) def test_stale_classification_cannot_exclude_a_missing_file(self): (self.root/'Sources/JSON/Declaration.swift').unlink() From 5fcc87bef006a5d7791f0dba049f3a43ac3faaec Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:44:56 +0200 Subject: [PATCH 10/24] docs: record portable QA and corrected coverage evidence --- Documentation/Qualification/README.md | 6 + .../Release1.2.1/AcceptanceMatrix.json | 40 +- .../Qualification/Release1.2.1/Portability.md | 22 + .../Release1.2.1/PortabilityOutcomes.json | 460 ++++++++++++++++++ .../Qualification/Release1.2.1/README.md | 4 + 5 files changed, 516 insertions(+), 16 deletions(-) create mode 100644 Documentation/Qualification/Release1.2.1/PortabilityOutcomes.json diff --git a/Documentation/Qualification/README.md b/Documentation/Qualification/README.md index d02aa64..8a551ba 100644 --- a/Documentation/Qualification/README.md +++ b/Documentation/Qualification/README.md @@ -102,3 +102,9 @@ The [second 1.2.1 wave](Release1.2.1/Wave2.md) adds 29 behavior functions: 173 parameterized invocations retained. QA expects 51/3/15/2 functions. Safety's targeted TSan subset adds the 12 controlled streaming functions to the existing six cases, with exact 4/1/1/12 module counts. Historical receipts remain unchanged. + +The 1.2.1 [portability follow-up](Release1.2.1/Portability.md) has 245/173 roots, +72 QA and 19 shared-state functions. ARM64/x86_64 Linux full lanes and scoped +WASI debug/release QA passed. Atomic native profiling reports 97.46% authored +lines / 93.04% regions and rejects saturated LLVM counters; earlier percentages +remain historical measurements, not directly comparable corrected coverage. diff --git a/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json b/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json index b014cff..0606a76 100644 --- a/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json +++ b/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json @@ -3,14 +3,14 @@ "source_version": "1.2.0", "source_commit": "25fb04f747e583209b4c4511abf39e11ef5b4f80", "target_version": "1.2.1", - "status": "second_wave_native_qualified_remaining_plan", + "status": "native_linux_and_scoped_wasi_qualified_remaining_1_2_1_plan", "rows": [ { "id": "HAR-01", "work_package": "T00", "module": "All", "topic": "Complete coverage export", - "status": "native_qualified_linux_execution_pending", + "status": "atomic_native_coverage_qualified_linux_export_execution_pending", "existing_evidence": [ "fresh 192/173 root coverage run", "default export omitted schema/regex/streaming mappings" @@ -22,7 +22,8 @@ "all exact discoveries retained", "branch/MC/DC marked unavailable" ], - "first_wave_evidence": "Wave1Outcomes.json" + "first_wave_evidence": "Wave1Outcomes.json", + "portability_evidence": "PortabilityOutcomes.json" }, { "id": "HAR-02", @@ -46,7 +47,7 @@ "work_package": "T01", "module": "JSON", "topic": "Numeric and primitive type matrix", - "status": "partially_native_qualified_128_bit_projection_and_typed_refusal", + "status": "partially_qualified_128_bit_projection_and_typed_refusal_mac_linux_wasi", "existing_evidence": [ "machineIntegerBoundaries", "boundedBinaryBridgePreservesLongHalfwayAndSubnormalTails", @@ -60,7 +61,8 @@ "native Int/WASI32 scope separate" ], "first_wave_evidence": "Wave1Outcomes.json", - "second_wave_evidence": "Wave2Outcomes.json" + "second_wave_evidence": "Wave2Outcomes.json", + "portability_evidence": "PortabilityOutcomes.json" }, { "id": "CORE-02", @@ -87,7 +89,7 @@ "work_package": "T01", "module": "JSON", "topic": "Explicit native container paths", - "status": "native_qualified_explicit_container_subset", + "status": "portable_qualified_explicit_container_subset", "existing_evidence": [ "nativeCodingUsesValuesAndComposedBytes", "superCodingAndNonSendableModelsWorkSynchronously" @@ -99,7 +101,8 @@ "EOF/wrong-kind coding paths", "successful roundtrip plus independent expected representation" ], - "first_wave_evidence": "Wave1Outcomes.json" + "first_wave_evidence": "Wave1Outcomes.json", + "portability_evidence": "PortabilityOutcomes.json" }, { "id": "COD-02", @@ -127,7 +130,7 @@ "work_package": "T01", "module": "JSON", "topic": "Foreign coder fidelity and fallback", - "status": "native_qualified_controlled_foreign_coders", + "status": "portable_qualified_controlled_foreign_bridge_subset", "existing_evidence": [ "qaGeneratedExactRoundTripsAcrossNativeAndForeignPayloads", "stringDictionaryProjectionRejectsScalarCollisions" @@ -140,7 +143,8 @@ "Version1 wrapper remains exact", "all six kinds represented" ], - "second_wave_evidence": "Wave2Outcomes.json" + "second_wave_evidence": "Wave2Outcomes.json", + "portability_evidence": "PortabilityOutcomes.json" }, { "id": "COD-04", @@ -345,7 +349,7 @@ "work_package": "T04", "module": "JSONStreaming", "topic": "Held source/sink backpressure", - "status": "native_qualified_controlled_pull_and_handoff_subset", + "status": "portable_qualified_causal_completion_and_backpressure_subset", "existing_evidence": [ "earlyStopDoesNotPullAheadAndChunkAdmissionIsImmediate", "awaitedSinkProducesIdenticalBytesAndStatistics", @@ -359,14 +363,15 @@ "accepted bytes cannot be recalled", "source producer cancellation not invented" ], - "second_wave_evidence": "Wave2Outcomes.json" + "second_wave_evidence": "Wave2Outcomes.json", + "portability_evidence": "PortabilityOutcomes.json" }, { "id": "STR-02", "work_package": "T04", "module": "JSONStreaming", "topic": "Framing, partitions and terminal behavior", - "status": "partially_native_qualified_held_completion_and_terminal_behavior", + "status": "partially_portable_qualified_framing_terminal_behavior_and_reviewed_unreachable_completion", "existing_evidence": [ "streamEverySplitAgreesWithNativeParser", "strictNDJSONPreservesPhysicalBoundariesAndLocations", @@ -380,7 +385,8 @@ "exactly once completion", "no further pull after terminal error" ], - "second_wave_evidence": "Wave2Outcomes.json" + "second_wave_evidence": "Wave2Outcomes.json", + "portability_evidence": "PortabilityOutcomes.json" }, { "id": "PAT-01", @@ -486,7 +492,7 @@ "work_package": "T07", "module": "All", "topic": "Actual platform/safety/consumer closure", - "status": "second_wave_native_qualified_other_platform_execution_pending", + "status": "native_linux_and_scoped_wasi_qualified_broader_plan_pending", "existing_evidence": [ "1.2.0 Native/Linux seven passed jobs", "65 WASI commands with explicit per-host scopes" @@ -501,7 +507,8 @@ "no 1.3 features in patch" ], "first_wave_evidence": "Wave1Outcomes.json", - "second_wave_evidence": "Wave2Outcomes.json" + "second_wave_evidence": "Wave2Outcomes.json", + "portability_evidence": "PortabilityOutcomes.json" } ], "proposed_targets": { @@ -523,5 +530,6 @@ "T07" ], "first_wave_evidence": "Wave1Outcomes.json", - "second_wave_evidence": "Wave2Outcomes.json" + "second_wave_evidence": "Wave2Outcomes.json", + "portability_evidence": "PortabilityOutcomes.json" } diff --git a/Documentation/Qualification/Release1.2.1/Portability.md b/Documentation/Qualification/Release1.2.1/Portability.md index c9c0440..44b3ca1 100644 --- a/Documentation/Qualification/Release1.2.1/Portability.md +++ b/Documentation/Qualification/Release1.2.1/Portability.md @@ -86,3 +86,25 @@ record the structural explanation rather than fabricate a private state or remove code to improve percentages. This closes semantic investigation of this region, not a claim of 100% region/branch or exhaustive race coverage. Revisit the invariant if framing or token-machine transitions change. + +## Actual closure + +All four Linux jobs passed at the runtime-tested head, with full debug/release +roots, full ASan, 19 TSan functions, strict memory compilation, all 49 cap +probes, rooted consumers, independent policies and clean-source reproduction. +Both WASI configurations passed the declared per-host scopes; native QA passed +in debug/release/ASan and all 19 shared-state TSan functions passed. + +The corrected atomic export reports **97.46% authored lines / 93.04% regions** +(6753/6929, 7156/7691). Streaming is 226/232 lines and 253/269 regions. +The reviewed alternative has counter zero; no denominator adjustment was made. +The earlier apparent gain was a saturated counter artifact. Lower corrected +percentages do not represent removed assertions; all discoveries remain intact. +The proposed per-module review targets and broader acceptance plan remain open. + +[Machine-readable outcomes](PortabilityOutcomes.json) identify tested heads, +receipt hashes, actual scopes, retained failures and outstanding work. The later +coverage-only commit changes measurement tooling and its notes; runtime source, +tests, manifests, fixtures and CI workflows are byte-identical to the Linux/WASI +qualified head. Closing documentation is a separate successor with those +executable inputs unchanged. No tag, release or sibling implementation is changed. diff --git a/Documentation/Qualification/Release1.2.1/PortabilityOutcomes.json b/Documentation/Qualification/Release1.2.1/PortabilityOutcomes.json new file mode 100644 index 0000000..1480186 --- /dev/null +++ b/Documentation/Qualification/Release1.2.1/PortabilityOutcomes.json @@ -0,0 +1,460 @@ +{ + "schema_version": 1, + "status": "native_linux_and_scoped_wasi_qualified_remaining_1_2_1_plan", + "runtime_tested_head": "bbc36070502554286a45fe4c49e8d3dde802c470", + "measurement_tested_head": "b2bd704dc4d7a82cfde8c0f2b5886b0fe2c0b01f", + "parent_wave": "Wave2Outcomes.json", + "public_runtime_source_changed": false, + "root_functions": 245, + "parameterized_invocations": 173, + "qa_functions": 72, + "shared_state_functions": 19, + "completion_matrix": { + "functions": 1, + "inner_cases": 566, + "root_spellings": 11, + "framing_modes": 2, + "byte_splits": "every two-chunk split including empty chunks" + }, + "native": { + "qa": { + "path": "qa-requalified/receipt.json", + "sha256": "29427d374a464fbdc2e0b9c6b80be99fc7abe195d19daff79eacc8d8ea27736b", + "source_commit": "bbc36070502554286a45fe4c49e8d3dde802c470", + "source_manifest_sha256": "449efd90c6f212d83acb0e4cb7ba1dbf3776f5a3c35e62c01d1f6285b93fd0e6", + "commands": 3 + }, + "affected_tsan": { + "path": "affected-safety.json", + "sha256": "40c10b4c5b62d7537fcde675d0faa8dcc50e83d62e0b2481ef4325f21d3ab54f", + "source_commit": "bbc36070502554286a45fe4c49e8d3dde802c470", + "source_manifest_sha256": null, + "commands": 1 + }, + "qa_counts_each_configuration": [ + 51, + 16, + 3, + 2 + ], + "tsan_counts": [ + 4, + 13, + 1, + 1 + ], + "qa_cli_exit": 0, + "affected_safety_wrapper_exit": 0, + "reuse": "Unchanged production source: previous full ASan, strict library compilation, public consumers and 582/582 authored DocC symbols. New tests have refreshed QA ASan/TSan." + }, + "linux": { + "workflow": "https://github.com/ThorFuchs/Swift-JSON/actions/runs/38547364795", + "head": "bbc36070502554286a45fe4c49e8d3dde802c470", + "jobs": 4, + "status": "success", + "platform": "Swift 6.4 / Ubuntu 24.04 / glibc 2.39", + "architectures": [ + "aarch64", + "x86_64" + ], + "root_functions_each_debug_release_asan": 245, + "parameterized_each": 173, + "tsan_functions_each": 19, + "cap_probes_each": 49, + "rooted_vectors_each": 20, + "operational_errors_separate": 2, + "clean_source_reproduction_each": "passed", + "source_identity": "Local git_commit fields are null. GitHub head SHA/run ID plus byte-equal complete source manifests and verified log hashes establish identity." + }, + "wasi": { + "lane": { + "path": "wasm-requalified/receipt.json", + "sha256": "27dddc82f5cf6e2b509569b9dac2e31d7566515a9200b051111dab338a222df6", + "source_commit": "bbc36070502554286a45fe4c49e8d3dde802c470", + "source_manifest_sha256": "449efd90c6f212d83acb0e4cb7ba1dbf3776f5a3c35e62c01d1f6285b93fd0e6", + "commands": 2 + }, + "commands": 34, + "sdk": "swift-6.4.0-RELEASE_wasm", + "hosts": [ + "Node 26.5.0", + "WasmKit 0.3.1" + ], + "node_scope": "245 root functions / 173 invocations in debug and release; 72 affected QA functions each configuration", + "wasmkit_scope": "72 affected QA functions in debug and release; no full-root/independent-suite refresh claim", + "strict_memory": "passed", + "sanitizers": "unavailable, not passes", + "streaming_host_watchdog_seconds": 120 + }, + "coverage": { + "receipt": { + "path": "coverage-atomic/receipt.json", + "sha256": "5fd163a3d533a747363da8b7b739ae11bf51f4e4527f85912c34d8d459432d4b", + "source_commit": "b2bd704dc4d7a82cfde8c0f2b5886b0fe2c0b01f", + "source_manifest_sha256": "8399bc1c5bf12edc2d73960523a2ccf006f3d246a7e63cb3b6d85dfa644dfa0b", + "commands": 2 + }, + "measurement": "atomic profiling increments; saturated library segment counts rejected", + "authored": { + "lines": { + "count": 6929, + "covered": 6753, + "percent": 97.45995093087025 + }, + "regions": { + "count": 7691, + "covered": 7156, + "percent": 93.04381744896632 + }, + "functions": { + "count": 1143, + "covered": 1100, + "percent": 96.23797025371829 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "modules": { + "JSON": { + "mapped_files": 50, + "authored": { + "lines": { + "count": 3443, + "covered": 3340, + "percent": 97.00842288701713 + }, + "regions": { + "count": 3427, + "covered": 3177, + "percent": 92.7049897869857 + }, + "functions": { + "count": 595, + "covered": 578, + "percent": 97.14285714285714 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 0, + "covered": 0, + "percent": null + }, + "regions": { + "count": 0, + "covered": 0, + "percent": null + }, + "functions": { + "count": 0, + "covered": 0, + "percent": null + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + "JSONSchema": { + "mapped_files": 32, + "authored": { + "lines": { + "count": 2410, + "covered": 2360, + "percent": 97.9253112033195 + }, + "regions": { + "count": 3088, + "covered": 2888, + "percent": 93.52331606217616 + }, + "functions": { + "count": 384, + "covered": 365, + "percent": 95.05208333333333 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 0, + "covered": 0, + "percent": null + }, + "regions": { + "count": 0, + "covered": 0, + "percent": null + }, + "functions": { + "count": 0, + "covered": 0, + "percent": null + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + "JSONRegexRuntime": { + "mapped_files": 6, + "authored": { + "lines": { + "count": 659, + "covered": 648, + "percent": 98.33080424886191 + }, + "regions": { + "count": 695, + "covered": 646, + "percent": 92.94964028776978 + }, + "functions": { + "count": 88, + "covered": 85, + "percent": 96.5909090909091 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 30, + "covered": 30, + "percent": 100.0 + }, + "regions": { + "count": 44, + "covered": 44, + "percent": 100.0 + }, + "functions": { + "count": 3, + "covered": 3, + "percent": 100.0 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + "JSONStreaming": { + "mapped_files": 10, + "authored": { + "lines": { + "count": 232, + "covered": 226, + "percent": 97.41379310344827 + }, + "regions": { + "count": 269, + "covered": 253, + "percent": 94.05204460966543 + }, + "functions": { + "count": 60, + "covered": 57, + "percent": 95.0 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 0, + "covered": 0, + "percent": null + }, + "regions": { + "count": 0, + "covered": 0, + "percent": null + }, + "functions": { + "count": 0, + "covered": 0, + "percent": null + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + "JSONPatch": { + "mapped_files": 4, + "authored": { + "lines": { + "count": 185, + "covered": 179, + "percent": 96.75675675675676 + }, + "regions": { + "count": 212, + "covered": 192, + "percent": 90.56603773584905 + }, + "functions": { + "count": 16, + "covered": 15, + "percent": 93.75 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 0, + "covered": 0, + "percent": null + }, + "regions": { + "count": 0, + "covered": 0, + "percent": null + }, + "functions": { + "count": 0, + "covered": 0, + "percent": null + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + } + }, + "unreachable_completion_alternative_counter": 0, + "denominator_adjusted": false, + "previous_followup_export": "Rejected saturated derived counter; apparent one-region gain is not accepted. Earlier historical percentages remain preserved but are not directly comparable to corrected measurement.", + "branch_mcdc": "unavailable" + }, + "independent_policies": { + "syntax": { + "observed": 318, + "evaluated": 287, + "policy": 31 + }, + "schema": { + "observed": 2337, + "evaluated": 2300, + "policy": 37 + }, + "authored": 82, + "budget_reports": 9, + "malformed_replay": 4396, + "regex_cases": 31824, + "normalization_cases": 18853, + "normalization_comparisons": 94265 + }, + "failures_preserved": [ + "Initial completion-oracle initializer compile failure (exit 1), corrected to fixed integer oracle", + "WASI inferred-Int overflow (exit 1), fixed explicit Int64", + "Pinned SDK time-limit traits: 13 WASI streaming failures (exit 1), SDK-only reproduction fails on both hosts, native controls pass", + "Native release QA interrupted by ENOSPC; wrapper exit 120, unfinished receipt, followed by complete fresh passing QA", + "Superseded Linux run 38545711422 cancelled by existing concurrency policy; not counted as final qualification", + "Initial non-atomic coverage export rejected by new real saturation check (exit 1)" + ], + "remaining": [ + "Broader T01/T02/T03/T04/T05/T06/T07 acceptance rows and proposed per-module 98% line / 95% region targets", + "Actual Linux LLVM coverage export execution", + "Unused private-helper review; denominator unchanged", + "Existing platform limits: physical iPad deferred; no new mobile/device execution", + "Pinned WASI Swift Testing timer issue: preserve reproduction and revisit on SDK upgrade" + ], + "empty_root_dependencies": true, + "public_products": [ + "JSON", + "JSONSchema", + "JSONStreaming", + "JSONPatch" + ], + "private_target": "JSONRegexRuntime", + "sibling_1_3_changed": false, + "tag_or_release_created": false +} diff --git a/Documentation/Qualification/Release1.2.1/README.md b/Documentation/Qualification/Release1.2.1/README.md index 6a0f568..670f91c 100644 --- a/Documentation/Qualification/Release1.2.1/README.md +++ b/Documentation/Qualification/Release1.2.1/README.md @@ -35,3 +35,7 @@ The [portability/completion follow-up](Portability.md) adds a 566-case public completion matrix and narrowly scoped Linux/WASI QA lanes. Its current discovery is 245/173 roots, 72 QA and 19 shared-state functions; actual platform results remain separate from the historical native waves. + +The portability follow-up is [qualified within its declared scopes](PortabilityOutcomes.json). +Atomic coverage corrects saturated async counters to 97.46% lines / 93.04% regions; +the structural completion gap is reviewed with its denominator retained. From 20476dace4626528aba36649df2686b2158c7f9f Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:18:29 +0200 Subject: [PATCH 11/24] test: export Linux coverage and exercise public behavior gaps --- .github/workflows/linux.yml | 2 +- CHANGELOG.md | 5 + .../Release1.2.1/CoverageWave.md | 35 ++++ .../Qualification/Release1.2.1/README.md | 4 + .../Qualification/TestDiscovery.json | 26 +-- Scripts/ci.py | 8 +- Scripts/coverage.py | 18 ++- Scripts/test_qualification_harness.py | 2 + .../Evaluation/JSONSchemaEvaluation.swift | 30 ++-- Tests/JSONPatchTests/CoverageGapTests.swift | 64 ++++++++ .../CoverageGapTests.swift | 57 +++++++ Tests/JSONSchemaTests/CoverageGapTests.swift | 150 ++++++++++++++++++ .../JSONStreamingTests/CoverageGapTests.swift | 42 +++++ Tests/JSONTests/CoverageGapTests.swift | 143 +++++++++++++++++ 14 files changed, 558 insertions(+), 28 deletions(-) create mode 100644 Documentation/Qualification/Release1.2.1/CoverageWave.md create mode 100644 Tests/JSONPatchTests/CoverageGapTests.swift create mode 100644 Tests/JSONRegexRuntimeTests/CoverageGapTests.swift create mode 100644 Tests/JSONSchemaTests/CoverageGapTests.swift create mode 100644 Tests/JSONStreamingTests/CoverageGapTests.swift create mode 100644 Tests/JSONTests/CoverageGapTests.swift diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 5324728..fac3158 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -17,7 +17,7 @@ jobs: strategy: fail-fast: false matrix: - lane: [linux, linux-safety] + lane: [linux, linux-safety, linux-coverage] architecture: [x86_64, arm64] include: - architecture: x86_64 diff --git a/CHANGELOG.md b/CHANGELOG.md index a9d21f4..a0bdebf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +- Preserve ordinary invalidity when best-effort schema diagnostics omit an issue + that exceeds the byte ceiling. Required work and annotation failures remain + terminal. Add 31 behavior and precise failure tests across all five modules, + plus reproducible five-module Linux coverage exports on ARM64 and x86_64. + - Add local full-language wasm32 WASI support, preserving all four products and the empty root dependency graph. Twelve cumulative UInt64 quotas can exceed Int.max without changing profiles; allocation capacities retain Int guards. diff --git a/Documentation/Qualification/Release1.2.1/CoverageWave.md b/Documentation/Qualification/Release1.2.1/CoverageWave.md new file mode 100644 index 0000000..e57a787 --- /dev/null +++ b/Documentation/Qualification/Release1.2.1/CoverageWave.md @@ -0,0 +1,35 @@ +# Linux coverage and behavior-gap wave + +This wave stays on the isolated `codex/1.2.1-test-plan` branch from 1.2.0. +It preserves the prior wave receipts and the sibling 1.3.0 checkout. + +Add 31 Swift Testing functions: JSON 10, JSONSchema 10, private regex 4, +streaming 3 and Patch 4. Exact root discovery becomes 276 functions / 173 +parameterized invocations; affected QA is 103 functions and shared-state safety +is 22. Authored fixed vectors, exact public failures and inclusive/cumulative +limits provide the oracles; tests do not call unreachable private guards. + +A regression case exposed best-effort diagnostic sizing latching a terminal +meter failure before an issue could be omitted. An invocation-local omission +signal now permits the contracted invalid result with incomplete diagnostics. +Required work, annotation, retention, overflow and cancellation still throw; +no meter is reset. Default complete diagnostics retain their strict refusal. +No public signatures, dependencies, products, profiles or fixtures change. + +Add `linux-coverage` to the existing two-architecture pinned Linux matrix. +It retains exact discovery and source-denominator checks, atomic counters and +the actual omitted-schema negative exporter probe. Exports preserve the five +actual objects plus merged profile; the exporter runs against those copies. +Generated tables, unmapped declarations, and unavailable branch/MC/DC counters +remain separate. Existing Linux/native lane names and 49 metric probes remain. + +Initial focused native verification passed 32 functions (including one existing +reader case) on Apple Swift 6.4 / Xcode 27.2 / macOS SDK 27.2. The Python harness +passed all ten negative/discovery cases. Full platform/safety/coverage outcomes +are pending; this document is not a qualification claim. The previous accepted +atomic coverage baseline is 6753/6929 lines and 7156/7691 regions. + +The structural streaming completion alternative and unused private helpers keep +their denominators. Proposed 98% line / 95% region per-module review targets, +remaining public end-to-end/mutation work and deferred device scopes stay open +until evidence justifies closure. diff --git a/Documentation/Qualification/Release1.2.1/README.md b/Documentation/Qualification/Release1.2.1/README.md index 670f91c..2c5629e 100644 --- a/Documentation/Qualification/Release1.2.1/README.md +++ b/Documentation/Qualification/Release1.2.1/README.md @@ -39,3 +39,7 @@ results remain separate from the historical native waves. The portability follow-up is [qualified within its declared scopes](PortabilityOutcomes.json). Atomic coverage corrects saturated async counters to 97.46% lines / 93.04% regions; the structural completion gap is reviewed with its denominator retained. + +The [Linux coverage and behavior-gap wave](CoverageWave.md) extends tests across +all five modules and repairs best-effort diagnostic sizing. Its new native/Linux +qualification is recorded separately from the earlier unchanged-source receipts. diff --git a/Documentation/Qualification/TestDiscovery.json b/Documentation/Qualification/TestDiscovery.json index daed2ab..e70e1b3 100644 --- a/Documentation/Qualification/TestDiscovery.json +++ b/Documentation/Qualification/TestDiscovery.json @@ -1,20 +1,26 @@ { "schema_version": 1, "root_functions": { - "JSONTests": 144, - "JSONSchemaTests": 51, - "JSONRegexRuntimeTests": 12, - "JSONStreamingTests": 27, - "JSONPatchTests": 11 + "JSONTests": 154, + "JSONSchemaTests": 61, + "JSONRegexRuntimeTests": 16, + "JSONStreamingTests": 30, + "JSONPatchTests": 15 }, "parameterized_invocations": 173, "qa_functions": { - "JSONTests": 51, - "JSONSchemaTests": 3, - "JSONStreamingTests": 16, - "JSONPatchTests": 2 + "JSONTests": 61, + "JSONSchemaTests": 13, + "JSONStreamingTests": 19, + "JSONPatchTests": 6, + "JSONRegexRuntimeTests": 4 + }, + "shared_state_functions": { + "JSONTests": 4, + "JSONSchemaTests": 1, + "JSONRegexRuntimeTests": 1, + "JSONStreamingTests": 16 }, - "shared_state_functions": {"JSONTests": 4, "JSONSchemaTests": 1, "JSONRegexRuntimeTests": 1, "JSONStreamingTests": 13}, "coverage_unmapped": { "Sources/JSON/Pointer/JSONLocationComponent.swift": "declaration-only enum", "Sources/JSONSchema/Output/JSONValidationOutput.swift": "declaration-only enum", diff --git a/Scripts/ci.py b/Scripts/ci.py index 60060c1..fe5ea1f 100644 --- a/Scripts/ci.py +++ b/Scripts/ci.py @@ -22,7 +22,7 @@ def commands(lane, output): ['python3', '-B', 'Scripts/wasm_support.py', '--output', str(output / 'probes')] + (['--scope', 'root-qa'] if lane == 'wasm-qa' else []), ] - if lane == 'coverage': + if lane in ('coverage', 'linux-coverage'): return [ ['python3', '-B', 'Scripts/test_qualification_harness.py'], ['python3', '-B', 'Scripts/coverage.py', '--output', str(output / 'export')], @@ -136,7 +136,7 @@ def check_result(command, code, log): def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--lane', choices=['native', 'safety', 'docs', 'r0', 'qa', 'coverage', 'linux', 'linux-safety', 'linux-qa', 'wasm', 'wasm-qa'], required=True) + parser.add_argument('--lane', choices=['native', 'safety', 'docs', 'r0', 'qa', 'coverage', 'linux', 'linux-safety', 'linux-qa', 'linux-coverage', 'wasm', 'wasm-qa'], required=True) parser.add_argument('--output') args = parser.parse_args() output = ROOT / (args.output or '.build/ci/' + args.lane) @@ -194,14 +194,14 @@ def save(): receipt['materialized_wasi'] = {'path':'probes/wasi.json','sha256':hashlib.sha256((output / 'probes/wasi.json').read_bytes()).hexdigest()} except (OSError,KeyError,ValueError,TypeError) as error: receipt['status'] = 'failed'; receipt['evidence_error'] = str(error) - if receipt['status'] == 'passed' and args.lane == 'coverage': + if receipt['status'] == 'passed' and args.lane in ('coverage', 'linux-coverage'): try: coverage = json.loads((output / 'export/coverage.json').read_text()) if (coverage['status'] != 'passed' or coverage['root_counts'] != ROOT_COUNTS or coverage['parameterized_invocations'] != PARAMETER_INVOCATIONS or coverage['omitted_binary_probe']['status'] != 'rejected' or set(coverage['modules']) != {name.removesuffix('Tests') for name in ROOT_COUNTS}): - raise ValueError('Missing complete native coverage boundary') + raise ValueError('Missing complete five-module coverage boundary') receipt['materialized_coverage'] = {'path':'export/coverage.json', 'sha256':hashlib.sha256((output / 'export/coverage.json').read_bytes()).hexdigest()} except (OSError,KeyError,ValueError,TypeError) as error: receipt['status'] = 'failed'; receipt['evidence_error'] = str(error) diff --git a/Scripts/coverage.py b/Scripts/coverage.py index 2b1dca3..7e7b811 100644 --- a/Scripts/coverage.py +++ b/Scripts/coverage.py @@ -141,8 +141,20 @@ def run(command, name, binary=False): raise ValueError('Merged coverage profile missing') shutil.copy2(profile, output / 'default.profdata') receipt['profile_sha256'] = digest(profile) - receipt['binaries'] = {name:dict(path=p.relative_to(ROOT).as_posix(), sha256=digest(p)) for name,p in objects.items()} - first, *others = objects.values() + # Preserve the actual five objects, not just paths into a discarded CI build. + # Together with the merged profile these allow independent LLVM replay. + binary_output = output / 'objects' + binary_output.mkdir(exist_ok=True) + receipt['binaries'] = {} + for name, path in objects.items(): + materialized = binary_output / name + shutil.copy2(path, materialized) + receipt['binaries'][name] = dict(path=path.relative_to(ROOT).as_posix(), + materialized_path=materialized.relative_to(output).as_posix(), sha256=digest(materialized)) + if digest(path) != digest(materialized): + raise ValueError('Instrumented executable copy changed: ' + name) + materialized_objects = {name:output / record['materialized_path'] for name,record in receipt['binaries'].items()} + first, *others = materialized_objects.values() command = llvm + ['export', str(first), '-instr-profile=' + str(profile)] for other in others: command.extend(['-object', str(other)]) @@ -156,7 +168,7 @@ def run(command, name, binary=False): receipt['default_export'] = dict(status='incomplete', error=str(error)) # This actual exporter probe must omit the schema-only executable; # regex code alone is also linked in schema tests and is not a useful omission. - omitted = [p for name,p in objects.items() if name != 'JSONSchemaTests'] + omitted = [p for name,p in materialized_objects.items() if name != 'JSONSchemaTests'] negative_command = llvm + ['export', str(omitted[0]), '-instr-profile=' + str(profile)] for other in omitted[1:]: negative_command.extend(['-object', str(other)]) diff --git a/Scripts/test_qualification_harness.py b/Scripts/test_qualification_harness.py index bd33b89..1d036e6 100644 --- a/Scripts/test_qualification_harness.py +++ b/Scripts/test_qualification_harness.py @@ -91,6 +91,8 @@ def test_exact_discovery_rejects_missing_duplicate_and_changed_parameters(self): self.assertFalse(check_result(linux[4], 0, qa+'\nTest run with 1 test in 0 suites passed')[0]) self.assertIn('--scope', commands('wasm-qa', output)[1]) self.assertNotIn('--scope', commands('wasm', output)[1]) + self.assertEqual(commands('coverage', output), commands('linux-coverage', output)) + self.assertEqual(commands('linux-coverage', output)[1][1:3], ['-B', 'Scripts/coverage.py']) def test_darwin_executable_membership_is_exact(self): for name in ROOT_COUNTS: diff --git a/Sources/JSONSchema/Evaluation/JSONSchemaEvaluation.swift b/Sources/JSONSchema/Evaluation/JSONSchemaEvaluation.swift index f9bb8ab..31a3f27 100644 --- a/Sources/JSONSchema/Evaluation/JSONSchemaEvaluation.swift +++ b/Sources/JSONSchema/Evaluation/JSONSchemaEvaluation.swift @@ -58,28 +58,38 @@ package struct JSONSchemaEvaluation { let result = try JSONPointer.decodedTokens(tokens,meter: &meter) try meter.release(UInt64(tokens.count)*16); return result } + private struct DiagnosticOmission: Error {} + private func outputAdding(_ metric: JSONResourceMetric, _ current: UInt64, _ amount: UInt64, + meter: inout JSONWorkMeter) throws -> UInt64 { + let (next, overflow) = current.addingReportingOverflow(amount) + // Optional diagnostic output must not latch a terminal meter failure. + // Overflow and every owned work/retention/annotation quota still throw. + if !overflow, metric == .diagnosticBytes, options.diagnosticPolicy == .bestEffort, + next > meter.limits[metric] { throw DiagnosticOmission() } + return try meter.adding(metric, current, amount) + } private func outputSize(_ keyword: Keyword?, frame: Frame, metric: JSONResourceMetric = .diagnosticBytes, meter: inout JSONWorkMeter) throws -> UInt64 { // Conservative encoded UTF-8 admission, including both Pointer locations and the absolute URI. var size: UInt64 = 64 for component in frame.path { - try meter.work(); size = try meter.adding(metric,size,16 + 1) + try meter.work(); size = try outputAdding(metric,size,16 + 1,meter: &meter) switch component { - case .index: size = try meter.adding(metric,size,20) + case .index: size = try outputAdding(metric,size,20,meter: &meter) case .key(let key): - for byte in key.string.utf8 { try meter.work(); size = try meter.adding(metric,size,byte == 47 || byte == 126 ? 2 : 1) } + for byte in key.string.utf8 { try meter.work(); size = try outputAdding(metric,size,byte == 47 || byte == 126 ? 2 : 1,meter: &meter) } } } for key in frame.trace { - try meter.work(); size = try meter.adding(metric,size,17) - for byte in key.string.utf8 { try meter.work(); size = try meter.adding(metric,size,byte == 47 || byte == 126 ? 2 : 1) } + try meter.work(); size = try outputAdding(metric,size,17,meter: &meter) + for byte in key.string.utf8 { try meter.work(); size = try outputAdding(metric,size,byte == 47 || byte == 126 ? 2 : 1,meter: &meter) } } let node = plan.nodes[frame.node], root = plan.resources[node.resource].root - for _ in plan.resources[node.resource].uri.string.utf8 { try meter.work(); size = try meter.adding(metric,size,1) } + for _ in plan.resources[node.resource].uri.string.utf8 { try meter.work(); size = try outputAdding(metric,size,1,meter: &meter) } for key in node.pointer.tokens.dropFirst(plan.nodes[root].pointer.tokens.count) { - size = try meter.adding(metric,size,17) - for byte in key.string.utf8 { try meter.work(); size = try meter.adding(metric,size,byte == 47 || byte == 126 ? 6 : 3) } + size = try outputAdding(metric,size,17,meter: &meter) + for byte in key.string.utf8 { try meter.work(); size = try outputAdding(metric,size,byte == 47 || byte == 126 ? 6 : 3,meter: &meter) } } - if let keyword { for _ in keyword.rawValue.utf8 { try meter.work(); size = try meter.adding(metric,size,4) }; size = try meter.adding(metric,size,34) } + if let keyword { for _ in keyword.rawValue.utf8 { try meter.work(); size = try outputAdding(metric,size,4,meter: &meter) }; size = try outputAdding(metric,size,34,meter: &meter) } return size } private func locations(_ key: JSONKey?, frame: Frame, meter: inout JSONWorkMeter) throws -> (JSONPointer,JSONPointer,JSONSchemaURI) { @@ -119,7 +129,7 @@ package struct JSONSchemaEvaluation { if options.output == .flag { return } let size: UInt64 do { size = try outputSize(keyword,frame: frame,meter: &meter) } - catch let error as JSONFailure where error.resource?.metric == .diagnosticBytes && options.diagnosticPolicy == .bestEffort { + catch is DiagnosticOmission { frame.outcome.diagnosticsComplete = false; return } let fits = meter.used(.issues) < meter.limits[.issues] && size <= meter.limits[.diagnosticBytes] - min(meter.used(.diagnosticBytes),meter.limits[.diagnosticBytes]) diff --git a/Tests/JSONPatchTests/CoverageGapTests.swift b/Tests/JSONPatchTests/CoverageGapTests.swift new file mode 100644 index 0000000..48cfb8c --- /dev/null +++ b/Tests/JSONPatchTests/CoverageGapTests.swift @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: Apache-2.0 +import Testing +import JSON +import JSONPatch + +private func patchGapFailure(_ code: JSONFailure.Code, _ body: () throws -> Void) { + do { try body(); Issue.record("Expected patch failure") } + catch let failure as JSONFailure { #expect(failure.category == .patch); #expect(failure.code == code) } + catch { Issue.record("Unexpected failure: \(error)") } +} + +@Test func qaPatchMalformedDeclarationsRefuseBeforeApplication() throws { + for text in ["{}", "[null]", #"[{}]"#, #"[{"op":1,"path":""}]"#, #"[{"op":"add","path":1,"value":null}]"#, + #"[{"op":"add","path":""}]"#, #"[{"op":"copy","path":""}]"#, #"[{"op":"copy","from":false,"path":""}]"#, + #"[{"op":"unknown","path":""}]"#] { + let raw = try JSONParser().parse(text) + patchGapFailure(.invalidOperation) { _ = try JSONPatch(json: raw) } + } + let nullPayload = try JSONPatch(json: JSONParser().parse(#"[{"op":"add","path":"","value":null,"ignored":true}]"#)) + #expect(try nullPayload.applying(to: JSONPatchResult.removed) == .document(.null)) +} + +@Test func qaPatchRemovedRootRefusesEveryOperationRequiringPresence() throws { + let root = try JSONPointer(parsing: ""), member = try JSONPointer(parsing: "/member") + let operations: [JSONPatchOperation] = [.add(path: member,value:1), .test(path:root,value:.null), + .copy(from:root,path:root), .move(from:root,path:root), .remove(path:root), .replace(path:root,value:1)] + for operation in operations { + let patch = try JSONPatch(operations: [operation]) + patchGapFailure(.missingTarget) { _ = try patch.applying(to: JSONPatchResult.removed) } + } + #expect(try JSONPatch(operations: []).applying(to: JSONPatchResult.removed) == .removed) +} + +@Test func qaPatchSamePathMoveRequiresExistingSourceAndChargesCumulatively() throws { + let input = try JSONParser().parse(#"{"a/b":[1,2]}"#) + let path = try JSONPointer(parsing: "/a~1b/0") + let one = try JSONPatch(operations: [.move(from:path,path:path)]).applyingWithReport(to:.document(input)) + #expect(one.result == .document(input) && one.statistics[.operations] == 1) + let twice = try JSONPatch(operations: [.move(from:path,path:path), .move(from:path,path:path)]) + let report = try twice.applyingWithReport(to:.document(input)) + #expect(report.result == .document(input) && report.statistics[.operations] == 2) + #expect(report.statistics[.workUnits] > one.statistics[.workUnits]) + var limits = JSONLimits.balanced; limits[.workUnits] = one.statistics[.workUnits] + do { _ = try twice.applying(to:input,options:.init(limits:limits)); Issue.record("Expected cumulative refusal") } + catch let failure as JSONFailure { #expect(failure.category == .resource && failure.resource?.metric == .workUnits) } + let absent = try JSONPointer(parsing: "/missing") + do { _ = try JSONPatch(operations:[.move(from:absent,path:absent)]).applying(to:input); Issue.record("Expected absent source") } + catch let failure as JSONFailure { #expect(failure.code == .missingTarget) } + #expect(try input == JSONParser().parse(#"{"a/b":[1,2]}"#)) +} + +@Test func qaPatchMoveUnrelatedPrefixAndCopyToRootHaveRFCResults() throws { + let input = try JSONParser().parse(#"{"a":1,"b":{}}"#) + let move = try JSONPatch(operations:[.move(from:JSONPointer(parsing:"/a"),path:JSONPointer(parsing:"/b/c"))]) + #expect(try move.applying(to:input) == .document(JSONParser().parse(#"{"b":{"c":1}}"#))) + for operation in try [JSONPatchOperation.copy(from:JSONPointer(parsing:"/a"),path:JSONPointer(parsing:"")), + .move(from:JSONPointer(parsing:"/a"),path:JSONPointer(parsing:""))] { + #expect(try JSONPatch(operations:[operation]).applying(to:input) == .document(1)) + } + // Moving the entire root into its own descendant is forbidden, including null. + patchGapFailure(.invalidOperation) { + _ = try JSONPatch(operations:[.move(from:JSONPointer(parsing:""),path:JSONPointer(parsing:"/b"))]).applying(to:input) + } +} diff --git a/Tests/JSONRegexRuntimeTests/CoverageGapTests.swift b/Tests/JSONRegexRuntimeTests/CoverageGapTests.swift new file mode 100644 index 0000000..7850368 --- /dev/null +++ b/Tests/JSONRegexRuntimeTests/CoverageGapTests.swift @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: Apache-2.0 +import Testing +import JSON +@testable import JSONRegexRuntime + +@Test func qaRegexControlHexAndAstralEscapesHaveFixedScalarOracles() throws { + // Authored vectors for the shipped ECMA-262 Unicode escape grammar. + let cases: [(String,String)] = [(#"^\n\r\f\v\t$"#, "\n\r\u{c}\u{b}\t"), + (#"^\xAF\xaf$"#, "¯¯"), (#"^\u00e9\u{1f680}$"#, "é🚀"), + (#"^\uD83D\uDE80$"#, "🚀"), (#"^[\n\r\f\v]$"#, "\u{c}"), (#"^\cz$"#, "\u{1a}")] + for (pattern, subject) in cases { + var meter = try JSONWorkMeter(limits:.balanced,domain:.regex) + let program = try RegexCompiler.compile(pattern,meter:&meter) + #expect(try program.search(subject,meter:&meter)) + #expect(try program.search(subject+"x",meter:&meter) == false) + #expect(meter.used(.regexRetainedBytes) == 0 && meter.used(.regexCompileWorkUnits) > 0) + } +} + +@Test func qaRegexMalformedEscapesAreInvalidPatternsRatherThanNoMatch() throws { + let patterns = ["\\", #"\x"#, #"\x0"#, #"\xGG"#, #"\c1"#, #"[\B]"#, + #"\u{}"#, #"\u{g}"#, #"\u{110000}"#, #"\u{ffffff}"#, #"\u{12"#, #"\u12Q4"#, + #"\uD800\uZZZZ"#, #"\01"#, #"[\1]"#, #"\kX"#, #"(?<>a)"#, #"(?<\x61>a)"#, #"(?<\u{31}>a)"#] + for pattern in patterns { + var meter = try JSONWorkMeter(limits:.balanced,domain:.regex) + do { _ = try RegexCompiler.compile(pattern,meter:&meter); Issue.record("Accepted malformed pattern: \(pattern)") } + catch let failure as RegexFailure { + switch failure { case .invalidPattern: break; case .unsupportedPattern: Issue.record("Wrong grammar failure classification") } + } + #expect(meter.used(.regexCompileWorkUnits) > 0) + } +} + +@Test func qaRegexEscapedCaptureNamesAndUnmatchedReferencesRetainScalarIdentity() throws { + var meter = try JSONWorkMeter(limits:.balanced,domain:.regex) + let program = try RegexCompiler.compile(#"^(?<\u{e9}>a)(?b)\k<é>\k$"#,meter:&meter) + #expect(try program.search("abab",meter:&meter)) + #expect(try program.search("abaa",meter:&meter) == false) + let optional = try RegexCompiler.compile(#"^(?:(?a)|b)\k$"#,meter:&meter) + #expect(try optional.search("b",meter:&meter)) + #expect(try optional.search("aa",meter:&meter)) + #expect(try optional.search("ba",meter:&meter) == false) +} + +@Test func qaRegexEscapeCompilationQuotaAndCancellationAreOperational() async throws { + var limits = JSONLimits.balanced; limits[.regexCompileWorkUnits] = 1 + var meter = try JSONWorkMeter(limits:limits,domain:.regex) + do { _ = try RegexCompiler.compile(#"\u{1f680}"#,meter:&meter); Issue.record("Expected quota refusal") } + catch let failure as JSONFailure { #expect(failure.category == .resource && failure.resource?.metric == .regexCompileWorkUnits) } + let task = Task.detached { + var cancelled = try JSONWorkMeter(limits:.balanced,domain:.regex) + withUnsafeCurrentTask { $0?.cancel() } + do { _ = try RegexCompiler.compile(#"\u{1f680}"#,meter:&cancelled); return false } + catch is CancellationError { return true } + } + #expect(try await task.value) +} diff --git a/Tests/JSONSchemaTests/CoverageGapTests.swift b/Tests/JSONSchemaTests/CoverageGapTests.swift new file mode 100644 index 0000000..1ca21c5 --- /dev/null +++ b/Tests/JSONSchemaTests/CoverageGapTests.swift @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: Apache-2.0 +import Testing +import JSON +import JSONSchema + +private func gapPlan(_ text: String) throws -> CompiledJSONSchema { + try JSONSchemaCompiler().compile(JSONSchemaDocument(JSONParser().parse(text))) +} +private func gapCompilation(_ category: JSONSchemaCompilationFailure.Category, _ body: () throws -> Void) { + do { try body(); Issue.record("Expected compilation failure \(category)") } + catch let failure as JSONSchemaCompilationFailure { #expect(failure.category == category) } + catch { Issue.record("Unexpected operational error: \(error)") } +} + +@Test func qaSchemaMalformedStandardKeywordShapesAreOperationalFailures() throws { + let values = [#"{"type":[]}"#, #"{"type":["string",1]}"#, #"{"title":1}"#, #"{"$defs":[]}"#, + #"{"$vocabulary":[]}"#, #"{"$vocabulary":{"https://e.test/vocab":1}}"#, #"{"enum":true}"#, + #"{"examples":{}}"#, #"{"dependentRequired":[]}"#, #"{"dependentRequired":{"x":[1]}}"#, + #"{"minimum":false}"#, #"{"maximum":"2"}"#, #"{"exclusiveMinimum":null}"#, + #"{"exclusiveMaximum":[]}"#, #"{"uniqueItems":0}"#, #"{"deprecated":"yes"}"#, + #"{"readOnly":1}"#, #"{"writeOnly":null}"#] + for text in values { gapCompilation(.schemaSyntax) { _ = try gapPlan(text) } } + gapCompilation(.rootKind) { _ = try gapPlan("[]") } + // A successfully compiled assertion produces ordinary invalidity, not an error. + let verdict = try JSONSchemaValidator().validate("wrong", against: gapPlan(#"{"type":"number"}"#)) + #expect(!verdict.isValid && verdict.diagnosticsComplete && !verdict.issues.isEmpty) +} + +@Test func qaSchemaMalformedAnchorsAndReferencesHaveExactFailureCategories() throws { + for anchor in ["", "a!", "a/b", "a b"] { + let raw = JSON.object(try JSONObject(members:[.init("$anchor",value:.string(anchor))])) + gapCompilation(.anchor) { _ = try JSONSchemaCompiler().compile(JSONSchemaDocument(raw)) } + } + for text in [##"{"$ref":"#/$defs/~2"}"##, ##"{"$ref":"#missing"}"##, + ##"{"$defs":{"value":false},"$ref":"#/$defs/value/type"}"##] { + gapCompilation(.reference) { _ = try gapPlan(text) } + } + gapCompilation(.identifier) { _ = try gapPlan(#"{"$ref":"https://e.test/%ZZ"}"#) } + gapCompilation(.schemaSyntax) { _ = try gapPlan(#"{"pattern":"[z-a]"}"#) } +} + +@Test func qaSchemaURIIdentityFragmentRemovalAndRelativeResolutionAreOffline() throws { + let a = try JSONSchemaURI("HTTPS://EXAMPLE.TEST:443/a/../b/%7e#/$defs/value") + let b = try JSONSchemaURI("https://example.test:443/b/~#/$defs/value") + #expect(a == b && a.hashValue == b.hashValue && Set([a,b]).count == 1) + #expect(a.fragment == "/$defs/value") + let resource = try a.removingFragment() + #expect(resource.string == "https://example.test:443/b/~" && resource.fragment == nil) + #expect(try resource.removingFragment() == resource) + #expect(try JSONSchemaURI("../c#anchor",relativeTo:resource).string == "https://example.test:443/c#anchor") + for text in ["https://e.test/a b", "https://e.test/\\a", "https://e.test/\u{7f}", "relative/path", "https://e.test/%"] { + gapCompilation(.identifier) { _ = try JSONSchemaURI(text) } + } + var limits = JSONLimits.balanced; limits[.stringBytes] = 3 + do { _ = try a.removingFragment(limits:limits); Issue.record("Expected bounded URI refusal") } + catch let failure as JSONFailure { #expect(failure.category == .resource && failure.resource?.metric == .stringBytes) } +} + +@Test func qaSchemaRegistryConflictingNormalizedIDsCannotReplaceAPlan() throws { + let first = try JSONSchemaDocument(JSONParser().parse(#"{"$id":"https://e.test:443/value","type":"string"}"#),retrievalURI:JSONSchemaURI("https://e.test/first")) + let second = try JSONSchemaDocument(JSONParser().parse(#"{"$id":"HTTPS://E.TEST:443/value","type":"number"}"#),retrievalURI:JSONSchemaURI("https://e.test/second")) + let registry = try JSONSchemaRegistry(documents:[first,second]) + gapCompilation(.identifier) { _ = try JSONSchemaCompiler().compile(JSONSchemaDocument(true),registry:registry) } + let stable = try JSONSchemaCompiler().compile(first) + #expect(try JSONSchemaValidator().validate("valid",against:stable).isValid) + #expect(try JSONSchemaValidator().validate(1,against:stable).isValid == false) +} + +@Test func qaSchemaEscapedDiagnosticPathsIdentifyExactScalarMembers() throws { + let plan = try gapPlan(#"{"$id":"https://e.test/root","properties":{"/~":{"type":"number"},"é":false,"e\u0301":true}}"#) + let input = try JSONParser().parse(#"{"/~":"wrong","é":1,"e\u0301":2}"#) + let result = try JSONSchemaValidator().validate(input,against:plan) + #expect(!result.isValid && result.issues.count == 2) + let issue = try #require(result.issues.first { $0.instanceLocation.string == "/~1~0" }) + #expect(issue.keywordLocation.string == "/properties/~1~0/type") + #expect(issue.absoluteKeywordLocation?.string == "https://e.test/root#/properties/~1~0/type") + #expect(result.issues.contains { $0.instanceLocation.tokens == [JSONKey("é")] }) + #expect(!result.issues.contains { $0.instanceLocation.tokens == [JSONKey("e\u{301}")] }) +} + +@Test func qaSchemaEmptyArrayAndNonStringAnnotationApplicability() throws { + let arrayPlan = try gapPlan(#"{"prefixItems":[true],"items":true,"unevaluatedItems":false}"#) + let validator = JSONSchemaValidator(options:.init(collectAnnotations:true)) + let empty = try validator.validate(.array([]),against:arrayPlan) + #expect(empty.isValid) + #expect(empty.annotations.contains { $0.keyword == JSONKey("prefixItems") && $0.value == true }) + let nonEmpty = try validator.validate(.array([1,2]),against:arrayPlan) + #expect(nonEmpty.isValid) + let contentPlan = try gapPlan(#"{"format":"email","contentEncoding":"base64","contentMediaType":"application/json","contentSchema":false}"#) + let numeric = try validator.validate(1,against:contentPlan) + #expect(numeric.isValid && numeric.annotations.isEmpty) + let text = try validator.validate("not asserted",against:contentPlan) + #expect(text.isValid) + #expect(text.annotations.contains { $0.keyword == JSONKey("contentEncoding") && $0.value == "base64" }) +} + +@Test func qaSchemaBestEffortByteRefusalPreservesInvalidityAndExactAccounting() throws { + let plan = try gapPlan(#"{"properties":{"/~":false}}"#), value = try JSONParser().parse(#"{"/~":null}"#) + var limits = JSONLimits.balanced; limits[.diagnosticBytes] = 0 + do { _ = try JSONSchemaValidator(options:.init(limits:limits)).validate(value,against:plan); Issue.record("Expected diagnostic refusal") } + catch let failure as JSONFailure { #expect(failure.category == .resource && failure.resource?.metric == .diagnosticBytes) } + let report = try JSONSchemaValidator(options:.init(limits:limits,diagnosticPolicy:.bestEffort)).validateWithReport(value,against:plan) + #expect(!report.result.isValid && !report.result.diagnosticsComplete && report.result.issues.isEmpty) + #expect(report.statistics[.diagnosticBytes] == 0 && report.statistics[.evaluationWorkUnits] > 0) +} + +@Test func qaSchemaURIFormatsExerciseIPvFutureAuthorityAndPrivateIRIRanges() throws { + let cases: [(String,String,Bool)] = [ + ("uri","https://[v1.a:b]/",true), ("uri","https://[Vf.a!]/",true), + ("uri","https://[v.a]/",false), ("uri","https://[v1.]/",false), + ("uri","https://[v1.a^]/",false), ("uri","https://[v1.a",false), + ("uri","https://[::1]x/",false), ("uri","https://host:abc/",false), + ("uri","https://[v1.a]:80/",true), ("uri","https://user@other@host/",false), + ("iri","https://e.test/?q=\u{f0000}",true), ("iri","https://e.test/#\u{f0000}",false), + ("iri","https://e.test/\u{e000}",false), ("iri","https://e.test/\u{ffff}",false), + ("uri","https://e.test/é",false)] + let validator = JSONSchemaValidator(options:.init(formatAssertion:true)) + for (format,input,expected) in cases { + let raw = JSON.object(try JSONObject(members:[.init("format",value:.string(format))])) + let plan = try JSONSchemaCompiler().compile(JSONSchemaDocument(raw)) + let result = try validator.validate(.string(input),against:plan) + #expect(result.isValid == expected) + #expect(result.diagnosticsComplete) + } +} + +@Test func qaSchemaURITemplatePrefixLimitsAndPercentVariablesUseStrictGrammar() throws { + let plan = try gapPlan(#"{"format":"uri-template"}"#) + let validator = JSONSchemaValidator(options:.init(formatAssertion:true)) + for (text,valid) in [("{value:9999}",true), ("{value:10000}",false), ("{value:0}",false), + ("{a.%41,b*}",true), ("{a.%GG}",false), ("{.a..b}",false), ("{value:}",false), ("{%41}",true)] { + #expect(try validator.validate(.string(text),against:plan).isValid == valid) + } +} + +@Test func qaSchemaBestEffortDiagnosticsCannotSwallowOwnedWorkOrAnnotations() throws { + let plan = try gapPlan(#"{"allOf":[{"properties":{"/~":false}},false]}"#) + let value = try JSONParser().parse(#"{"/~":null}"#) + var limits = JSONLimits.balanced; limits[.diagnosticBytes] = 0 + let report = try JSONSchemaValidator(options:.init(limits:limits,diagnosticPolicy:.bestEffort)).validateWithReport(value,against:plan) + #expect(!report.result.isValid && !report.result.diagnosticsComplete) + #expect(report.result.issues.isEmpty && report.statistics[.diagnosticBytes] == 0) + limits[.evaluationWorkUnits] = report.statistics[.evaluationWorkUnits] - 1 + do { _ = try JSONSchemaValidator(options:.init(limits:limits,diagnosticPolicy:.bestEffort)).validate(value,against:plan); Issue.record("Work exhaustion became invalidity") } + catch let failure as JSONFailure { #expect(failure.category == .resource && failure.resource?.metric == .evaluationWorkUnits) } + let annotationPlan = try gapPlan(#"{"title":"metadata"}"#) + limits = .balanced; limits[.annotationBytes] = 0 + do { _ = try JSONSchemaValidator(options:.init(limits:limits,diagnosticPolicy:.bestEffort,collectAnnotations:true)).validate(value,against:annotationPlan); Issue.record("Annotation exhaustion was omitted") } + catch let failure as JSONFailure { #expect(failure.category == .resource && failure.resource?.metric == .annotationBytes) } +} diff --git a/Tests/JSONStreamingTests/CoverageGapTests.swift b/Tests/JSONStreamingTests/CoverageGapTests.swift new file mode 100644 index 0000000..d6d613d --- /dev/null +++ b/Tests/JSONStreamingTests/CoverageGapTests.swift @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: Apache-2.0 +import Testing +import JSON +import JSONStreaming + +@Test(StreamingTestPolicy.deadline) func qaStreamingWriterFacadePreservesGoldenChunksWithoutEmptyHandoffs() async throws { + // Independent strict-JSON bytes, including a scalar split across chunks. + let value = try JSONParser().parse(#"{"s":"🚀","a":[null,false,-1]}"#) + for size in [1,2,4,64] { + let sink = SinkBytes() + try await JSONAsyncWriter(chunkSize:size).write(value) { try await sink.accept($0) } + let bytes = await sink.bytes, calls = await sink.calls + #expect(bytes == Array(#"{"s":"🚀","a":[null,false,-1]}"#.utf8)) + #expect(calls == (bytes.count+size-1)/size) + } +} + +@Test(StreamingTestPolicy.deadline) func qaStreamingWriterInvalidConfigurationNeverCallsSink() async throws { + var limits = JSONLimits.balanced; limits[.chunkBytes] = 2 + let writers = [JSONAsyncWriter(chunkSize:0), JSONAsyncWriter(chunkSize:-1), + JSONAsyncWriter(options:.init(limits:limits),chunkSize:3), + JSONAsyncWriter(options:.init(indentation:-1)), JSONAsyncWriter(options:.init(indentation:9))] + for writer in writers { + let sink = SinkBytes() + do { try await writer.write(.null) { try await sink.accept($0) }; Issue.record("Expected invalid configuration") } + catch let failure as JSONFailure { #expect(failure.category == .configuration && failure.code == .invalidLimits) } + let calls = await sink.calls; #expect(calls == 0) + } +} + +@Test(StreamingTestPolicy.deadline) func qaStreamingWriterExactChunkAndIndentationBoundsHaveFiniteReports() async throws { + var limits = JSONLimits.balanced; limits[.chunkBytes] = 4 + let sink = SinkBytes() + let stats = try await JSONAsyncWriter(options:.init(limits:limits,indentation:0),chunkSize:4).writeWithReport(.null) { try await sink.accept($0) } + let bytes = await sink.bytes, calls = await sink.calls + #expect(bytes == Array("null".utf8) && calls == 1) + #expect(stats[.outputBytes] == 4 && stats[.retainedBytes] == 0) + let pretty = SinkBytes() + try await JSONAsyncWriter(options:.init(indentation:8),chunkSize:2).write(.array([true])) { try await pretty.accept($0) } + let output = await pretty.bytes + #expect(output == Array("[\n true\n]".utf8)) +} diff --git a/Tests/JSONTests/CoverageGapTests.swift b/Tests/JSONTests/CoverageGapTests.swift new file mode 100644 index 0000000..c9b7790 --- /dev/null +++ b/Tests/JSONTests/CoverageGapTests.swift @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation +import Testing +import JSON + +private func gapFailure(_ category: JSONFailure.Category, _ code: JSONFailure.Code, _ body: () throws -> Void) { + do { try body(); Issue.record("Expected \(category)/\(code)") } + catch let failure as JSONFailure { #expect(failure.category == category); #expect(failure.code == code) } + catch { Issue.record("Unexpected error: \(error)") } +} +private func gapAccess(_ path: [JSONLocationComponent], _ code: JSONFailure.Code = .wrongKind, _ body: () throws -> Void) { + do { try body(); Issue.record("Expected access failure") } + catch let failure as JSONAccessFailure { #expect(failure.path == path); #expect(failure.cause.code == code) } + catch { Issue.record("Unexpected error: \(error)") } +} + +@Test func qaReadWrongKindProjectionsRecoverWithinOneScope() throws { + let value = try JSONParser().parse(#"{"value":true,"good":17}"#) + let report = try value.readWithReport { root in + let wrong = try root["value"] + gapAccess([.key("value")]) { _ = try wrong.requiredString() } + gapAccess([.key("value")]) { _ = try wrong.requiredNumber() } + gapAccess([.key("value")]) { _ = try wrong.requiredObject() } + gapAccess([.key("value")]) { _ = try wrong.requiredArray() } + gapAccess([.key("value")]) { _ = try wrong.requiredDouble(policy: .exact) } + gapAccess([.key("value")]) { _ = try wrong.requiredDecimal(policy: .exact) } + return try root["good"].requiredNumber() + } + #expect(report.result == JSONNumber(integer: 17)) + #expect(report.statistics[.workUnits] > 0 && report.statistics[.retainedBytes] == 0) +} + +@Test func qaReadRawAndTypedPresenceUseLiteralScalarKeys() throws { + let value = try JSONParser().parse(#"{"é":1,"e\u0301":2,"nil":null,"wrong":[],"empty":""}"#) + try value.read { (root: JSONReadCursor) throws -> Void in + #expect(try root.presence(of: "é") == .value(1)) + #expect(try root.presence(of: JSONKey("e\u{301}")) == .value(2)) + #expect(try root.presence(of: "nil") == .null) + #expect(try root.presence(of: "absent") == .absent) + gapAccess([.key("wrong")]) { _ = try root.stringPresence(of: "wrong") } + gapAccess([.key("wrong")]) { _ = try root.booleanPresence(of: JSONKey("wrong")) } + gapAccess([.key("wrong")]) { _ = try root.numberPresence(of: "wrong") } + gapAccess([.key("wrong"), .key("member")]) { _ = try root["wrong"].presence(of: "member") } + #expect(try root.stringPresence(of: "empty") == .value("")) + } +} + +@Test func qaReadFailedDecimalProjectionReleasesTemporaryRetention() throws { + let value = try JSONParser().parse("999999999999999999999999999999999999999") + let once = try value.readWithReport { root in + gapAccess([], .inexactConversion) { _ = try root.requiredDecimal(policy: .exact) } + return try root.requiredNumber() + } + var limits = JSONLimits.balanced; limits[.retainedBytes] = once.statistics.peakLogicalRetainedBytes + let repeated = try value.readWithReport(limits: limits) { root in + for _ in 0..<32 { gapAccess([], .inexactConversion) { _ = try root.requiredDecimal(policy: .exact) } } + return try root.requiredNumber() + } + #expect(repeated.result == once.result) + #expect(repeated.statistics.peakLogicalRetainedBytes == once.statistics.peakLogicalRetainedBytes) + #expect(repeated.statistics[.retainedBytes] == 0) + #expect(repeated.statistics[.numericWorkUnits] > once.statistics[.numericWorkUnits]) +} + +@Test func qaPointerCompositionPreservesLiteralTokensAndInclusiveLimits() throws { + let base = try JSONPointer(tokens: ["a/b"]), tail = try JSONPointer(tokens: ["~", "é", "e\u{301}"]) + let combined = try base.appending(tail) + #expect(combined.tokens == [JSONKey("a/b"), "~", "é", "e\u{301}"]) + #expect(combined.string == "/a~1b/~0/é/e\u{301}") + #expect(try combined.appending(JSONPointer(parsing: "")) == combined) + #expect(try JSONPointer(parsing: "").appending(tail) == tail) + #expect(try base.appending(key: "/~").string == "/a~1b/~1~0") + var limits = JSONLimits.balanced; limits[.pointerTokens] = 4 + #expect(try base.appending(tail, limits: limits) == combined) + limits[.pointerTokens] = 3 + gapFailure(.resource, .limitExceeded) { _ = try base.appending(tail, limits: limits) } + gapFailure(.pointer, .invalidIndex) { _ = try base.appending(index: -1) } +} + +@Test func qaPointerFragmentRefusalsAndNavigationHavePreciseCauses() throws { + for fragment in ["#/%", "#/%A", "#/%G0", "#/%0g", "#/%ff", "/a"] { + gapFailure(.pointer, .invalidPointer) { _ = try JSONPointer.uriFragment(fragment) } + } + let object = try JSONParser().parse(#"{"present":null}"#) + gapFailure(.pointer, .missingTarget) { _ = try JSONPointer(parsing: "/missing").resolve(in: object) } + gapFailure(.typeMismatch, .wrongKind) { _ = try JSONPointer(parsing: "/present/child").resolve(in: object) } + gapFailure(.pointer, .invalidIndex) { _ = try JSONPointer(parsing: "/999999999999999999999999999999999999").resolve(in: .array([1])) } +} + +@Test func qaPreviewScalarEscapesAndNormalizedNumbersAreStrictWhenComplete() throws { + let cases: [(JSON, String)] = [(.null, "null"), (.boolean(true), "true"), (.boolean(false), "false"), + (.string("\"\\\n\t\u{0}"), #""\"\\\u000a\u0009\u0000""#), + (try JSONParser().parse("-12.00e3"), "-12e3")] + for (value, expected) in cases { + let preview = try JSONPreview.render(value) + #expect(preview.complete && preview.text == expected) + #expect(try JSONParser().parse(preview.text) == value) + } +} + +@Test func qaPreviewOmissionKeepsUTF8WholeAtEveryScalarByteBoundary() throws { + for capacity in 8...12 { + let preview = try JSONPreview.render(.string("🚀x"), options: .init(maxBytes: UInt64(capacity))) + #expect(!preview.complete && preview.text.hasSuffix("[...]")) + #expect(preview.text.utf8.count <= capacity && !preview.text.contains("�")) + #expect(preview.text == (capacity < 12 ? "\"[...]" : "\"🚀[...]")) + } + let object = try JSONParser().parse(#"{"x":{"private":"secret"}}"#) + let shallow = try JSONPreview.render(object, options: .init(maxDepth: 1)) + #expect(!shallow.complete && shallow.text == #"{"x":"[...]"}[...]"#) + let zero = try JSONPreview.render(.null, options: .init(maxNodes: 0)) + #expect(!zero.complete && zero.text == "[...]") +} + +@Test func qaPreviewInvalidOptionsAreConfigurationFailures() { + for options in [JSONPreviewOptions(maxDepth: -1), .init(maxBytes: 6)] { + gapFailure(.configuration, .invalidLimits) { _ = try JSONPreview.render(.null, options: options) } + } +} + +@Test func qaValueHashLawsCoverEveryKindAndExactObjectIdentity() throws { + let equivalents: [(JSON, JSON)] = [(.null,.null), (true,true), (false,false), ("e\u{301}","e\u{301}"), + (try JSONParser().parse("1.00"), 1), (.array([.null,true,"x"]), .array([.null,true,"x"])), + (try JSONParser().parse(#"{"é":1,"e\u0301":2}"#), try JSONParser().parse(#"{"e\u0301":2.0,"é":1e0}"#))] + for (a,b) in equivalents { #expect(a == b); #expect(a.hashValue == b.hashValue); #expect(Set([a,b]).count == 1) } + #expect(JSON.boolean(true) != false) + #expect(JSON.array([1]) != .array([1,2])) + #expect(try JSONParser().parse(#"{"a":1}"#) != JSONParser().parse(#"{"b":1}"#)) + #expect(try JSONParser().parse(#"{"a":1}"#) != JSONParser().parse(#"{"a":1,"b":2}"#)) + let nilValue: JSON = nil, array: JSON = [nil,true,"x"] + #expect(nilValue == .null && array == .array([.null,true,"x"])) +} + +@Test func qaDecodeConvenienceReportPreservesExactIntegerAndWorkRefusal() throws { + let value = try JSONParser().parse("9007199254740993") + let report = try value.decodeWithReport(UInt64.self) + #expect(report.result == 9_007_199_254_740_993) + #expect(report.statistics.domain == .coding && report.statistics[.numericWorkUnits] > 0) + var coding = JSONCodingOptions(); coding.limits[.workUnits] = report.statistics[.workUnits] + #expect(try value.decodeWithReport(UInt64.self, coding: coding).result == report.result) + coding.limits[.workUnits] -= 1 + gapFailure(.resource, .limitExceeded) { _ = try value.decodeWithReport(UInt64.self, coding: coding) } +} From 708c26389ff17a46e327c03d61f32947a4104415 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:27:57 +0200 Subject: [PATCH 12/24] fix: include Swift Build shared objects in Linux coverage --- .github/workflows/linux.yml | 8 ++++++- Scripts/coverage.py | 33 +++++++++++++++++++++++---- Scripts/test_qualification_harness.py | 9 +++++++- 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index fac3158..785406b 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -4,6 +4,12 @@ on: branches: [main] pull_request: workflow_dispatch: + inputs: + scope: + description: Full qualification or coverage-only exporter refresh + type: choice + options: [full, coverage] + default: full permissions: contents: read concurrency: @@ -17,7 +23,7 @@ jobs: strategy: fail-fast: false matrix: - lane: [linux, linux-safety, linux-coverage] + lane: ${{ fromJSON(inputs.scope == 'coverage' && '["linux-coverage"]' || '["linux", "linux-safety", "linux-coverage"]') }} architecture: [x86_64, arm64] include: - architecture: x86_64 diff --git a/Scripts/coverage.py b/Scripts/coverage.py index 7e7b811..983eecf 100644 --- a/Scripts/coverage.py +++ b/Scripts/coverage.py @@ -29,6 +29,20 @@ def executable_objects(bin_directory): result[name] = found[0] return result + +def shared_objects(bin_directory): + """Swift Build's Linux runners load instrumented product/test DSOs.""" + result = {} + targets = list(ROOT_COUNTS) + [name.removesuffix('Tests') for name in ROOT_COUNTS] + for target in targets: + found = [bin_directory / name for name in [target + '.so', 'lib' + target + '.so'] + if (bin_directory / name).is_file()] + if len(found) > 1: + raise ValueError('Ambiguous instrumented library: ' + target) + if found: + result[target] = found[0] + return result + def validate_export(export, root=ROOT, policy=DISCOVERY): """Validate the executable source denominator before calculating percentages.""" expected = {p.relative_to(root).as_posix() for p in (root / 'Sources').rglob('*.swift')} @@ -135,6 +149,8 @@ def run(command, name, binary=False): receipt.update(root_counts=ROOT_COUNTS, parameterized_invocations=PARAMETER_INVOCATIONS) bin_directory = pathlib.Path(run(['swift', 'build', '--show-bin-path'], 'bin-path.log').strip()) objects = executable_objects(bin_directory) + libraries = shared_objects(bin_directory) + receipt['build_product_inventory'] = sorted(p.name for p in bin_directory.iterdir() if p.is_file()) coverage_path = pathlib.Path(run(['swift', 'test', '--show-codecov-path'], 'profile-path.log').strip()) profile = coverage_path.parent / 'default.profdata' if not profile.is_file(): @@ -153,7 +169,16 @@ def run(command, name, binary=False): materialized_path=materialized.relative_to(output).as_posix(), sha256=digest(materialized)) if digest(path) != digest(materialized): raise ValueError('Instrumented executable copy changed: ' + name) - materialized_objects = {name:output / record['materialized_path'] for name,record in receipt['binaries'].items()} + receipt['libraries'] = {} + for name, path in libraries.items(): + materialized = binary_output / path.name + shutil.copy2(path, materialized) + receipt['libraries'][name] = dict(path=path.relative_to(ROOT).as_posix(), + materialized_path=materialized.relative_to(output).as_posix(), sha256=digest(materialized)) + if digest(path) != digest(materialized): + raise ValueError('Instrumented library copy changed: ' + name) + materialized_objects = {name + '/runner':output / record['materialized_path'] for name,record in receipt['binaries'].items()} + materialized_objects.update({name:output / record['materialized_path'] for name,record in receipt['libraries'].items()}) first, *others = materialized_objects.values() command = llvm + ['export', str(first), '-instr-profile=' + str(profile)] for other in others: @@ -166,9 +191,9 @@ def run(command, name, binary=False): receipt['default_export'] = dict(status='complete') except ValueError as error: receipt['default_export'] = dict(status='incomplete', error=str(error)) - # This actual exporter probe must omit the schema-only executable; + # This actual exporter probe must omit the schema runner and its DSOs; # regex code alone is also linked in schema tests and is not a useful omission. - omitted = [p for name,p in materialized_objects.items() if name != 'JSONSchemaTests'] + omitted = [p for name,p in materialized_objects.items() if name not in ('JSONSchemaTests/runner', 'JSONSchemaTests', 'JSONSchema')] negative_command = llvm + ['export', str(omitted[0]), '-instr-profile=' + str(profile)] for other in omitted[1:]: negative_command.extend(['-object', str(other)]) @@ -178,7 +203,7 @@ def run(command, name, binary=False): except ValueError as error: if 'Incomplete library export:' not in str(error): raise - receipt['omitted_binary_probe'] = dict(status='rejected', omitted='JSONSchemaTests', reason=str(error)) + receipt['omitted_binary_probe'] = dict(status='rejected', omitted='JSONSchemaTests runner and schema/test shared objects when present', reason=str(error)) else: raise ValueError('Harness accepted export missing the schema executable') # Public summaries and normalized export contain no local source path. diff --git a/Scripts/test_qualification_harness.py b/Scripts/test_qualification_harness.py index 1d036e6..88244f5 100644 --- a/Scripts/test_qualification_harness.py +++ b/Scripts/test_qualification_harness.py @@ -6,7 +6,7 @@ import pathlib import tempfile import unittest -from coverage import executable_objects, validate_export +from coverage import executable_objects, shared_objects, validate_export from test_discovery import DISCOVERY, ROOT_COUNTS, PARAMETER_INVOCATIONS, check_root from ci import check_result, commands @@ -107,6 +107,13 @@ def test_linux_runner_and_ambiguous_layout(self): for name in ROOT_COUNTS: (self.root/(name+'-test-runner')).touch() self.assertEqual(set(executable_objects(self.root)),set(ROOT_COUNTS)) + self.assertEqual(shared_objects(self.root), {}) # Static-linked backend remains valid. + targets = set(ROOT_COUNTS) | {name.removesuffix('Tests') for name in ROOT_COUNTS} + for target in targets: (self.root/(target+'.so')).touch() + self.assertEqual(set(shared_objects(self.root)), targets) + (self.root/'libJSON.so').touch() + with self.assertRaisesRegex(ValueError, 'Ambiguous instrumented library'): + shared_objects(self.root) (self.root/(name+'.xctest')).touch() with self.assertRaisesRegex(ValueError,'Missing or ambiguous'): executable_objects(self.root) From 288dc626d7a44240ffca2fa4e12ea4ce68662c66 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Wed, 14 Oct 2026 14:12:31 +0200 Subject: [PATCH 13/24] docs: record Linux coverage and behavior-wave qualification --- .../Release1.2.1/AcceptanceMatrix.json | 48 +- .../Release1.2.1/CoverageWave.md | 57 ++- .../Release1.2.1/CoverageWaveOutcomes.json | 466 ++++++++++++++++++ 3 files changed, 548 insertions(+), 23 deletions(-) create mode 100644 Documentation/Qualification/Release1.2.1/CoverageWaveOutcomes.json diff --git a/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json b/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json index 0606a76..f7e040b 100644 --- a/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json +++ b/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json @@ -10,7 +10,7 @@ "work_package": "T00", "module": "All", "topic": "Complete coverage export", - "status": "atomic_native_coverage_qualified_linux_export_execution_pending", + "status": "atomic_native_and_both_linux_exports_qualified", "existing_evidence": [ "fresh 192/173 root coverage run", "default export omitted schema/regex/streaming mappings" @@ -23,7 +23,8 @@ "branch/MC/DC marked unavailable" ], "first_wave_evidence": "Wave1Outcomes.json", - "portability_evidence": "PortabilityOutcomes.json" + "portability_evidence": "PortabilityOutcomes.json", + "coverage_wave_evidence": "CoverageWaveOutcomes.json" }, { "id": "HAR-02", @@ -190,7 +191,7 @@ "work_package": "T02", "module": "JSON", "topic": "Preview/redaction/completeness", - "status": "planned_extension", + "status": "partially_portable_qualified_preview_scalar_boundaries_and_invalid_options", "existing_evidence": [ "previewRedactsExactKeysBeforeTraversingValues", "previewCapsTotalTraversalAndOutputWithoutAdmittingWholeDocument" @@ -202,7 +203,8 @@ "completeness explicit", "no replacement-character corruption", "no serializer claim" - ] + ], + "coverage_wave_evidence": "CoverageWaveOutcomes.json" }, { "id": "BUD-01", @@ -249,7 +251,7 @@ "work_package": "T03", "module": "JSONSchema", "topic": "Applicators/evaluated locations/output", - "status": "planned_extension", + "status": "partially_portable_qualified_malformed_shapes_operational_categories", "existing_evidence": [ "evaluatedLocationsFlowThroughSuccessfulReferencesAndComposition", "arrayContainsAndConditionalAnnotationsRespectTheirBranches", @@ -262,14 +264,15 @@ "no quota-as-invalidity", "output conversion does not reevaluate", "incompleteness explicit" - ] + ], + "coverage_wave_evidence": "CoverageWaveOutcomes.json" }, { "id": "SCH-02", "work_package": "T03", "module": "JSONSchema", "topic": "References and immutable registries", - "status": "planned_extension", + "status": "partially_portable_qualified_uri_identity_collision_and_escaped_locations", "existing_evidence": [ "productiveRecursionProgressesAndNonproductiveCyclesThrow", "URIResolutionUsesRFC3986AndPercentFragmentsRemainEncoded", @@ -282,14 +285,15 @@ "cycles operational", "no online resolution", "no global mutable cache/state leak" - ] + ], + "coverage_wave_evidence": "CoverageWaveOutcomes.json" }, { "id": "SCH-03", "work_package": "T03", "module": "JSONSchema", "topic": "Raw/compiled inspection and compilation shapes", - "status": "planned_extension", + "status": "partially_portable_qualified_annotations_and_best_effort_terminal_controls", "existing_evidence": [ "rawDeclarationInspectionNeverInfersOrNarrowsConstraints", "compiledInspectionExposesStaticTargetsAndDynamicAmbiguity", @@ -302,14 +306,15 @@ "dynamic target ambiguity preserved", "selected builder phases cumulative", "compile error versus resource/cancel distinguished" - ] + ], + "coverage_wave_evidence": "CoverageWaveOutcomes.json" }, { "id": "REG-01", "work_package": "T03", "module": "JSONRegexRuntime", "topic": "VM semantic decisions", - "status": "planned_extension", + "status": "partially_portable_qualified_control_hex_named_scalar_escapes", "existing_evidence": [ "deterministicIndependentRegexDifferentialCases", "scalarBacktrackingSemantics", @@ -322,14 +327,15 @@ "unsupported/malformed compiled errors explicit", "newer engine grammar not adopted", "immutable program exclusive state" - ] + ], + "coverage_wave_evidence": "CoverageWaveOutcomes.json" }, { "id": "REG-02", "work_package": "T03", "module": "JSONRegexRuntime", "topic": "Regex budget/cleanup decisions", - "status": "planned_extension", + "status": "partially_portable_qualified_escape_compile_quota_and_cancellation", "existing_evidence": [ "adversarialRegexPathsExhaustWorkInsteadOfReturningInvalidity", "regexSearchLimitResetsButAggregateLimitNeverResets", @@ -342,7 +348,8 @@ "no branch swallows resource/cancel", "scratch released", "new invocation recovers independently" - ] + ], + "coverage_wave_evidence": "CoverageWaveOutcomes.json" }, { "id": "STR-01", @@ -393,7 +400,7 @@ "work_package": "T04", "module": "JSONPatch", "topic": "RFC6902 operation/state matrix", - "status": "planned_extension", + "status": "partially_portable_qualified_malformed_removed_root_same_path_move_copy", "existing_evidence": [ "allRFC6902OperationsAndArrayMoveOrdering", "rootRemovalRecreationAndNullRemainDistinct", @@ -406,7 +413,8 @@ "remove before target resolution", "input unchanged on failure", "irrelevant fields treated per RFC" - ] + ], + "coverage_wave_evidence": "CoverageWaveOutcomes.json" }, { "id": "PAT-02", @@ -492,7 +500,7 @@ "work_package": "T07", "module": "All", "topic": "Actual platform/safety/consumer closure", - "status": "native_linux_and_scoped_wasi_qualified_broader_plan_pending", + "status": "native_linux_and_scoped_wasi_qualified_performance_variance_followup_broader_plan_pending", "existing_evidence": [ "1.2.0 Native/Linux seven passed jobs", "65 WASI commands with explicit per-host scopes" @@ -508,7 +516,8 @@ ], "first_wave_evidence": "Wave1Outcomes.json", "second_wave_evidence": "Wave2Outcomes.json", - "portability_evidence": "PortabilityOutcomes.json" + "portability_evidence": "PortabilityOutcomes.json", + "coverage_wave_evidence": "CoverageWaveOutcomes.json" } ], "proposed_targets": { @@ -531,5 +540,6 @@ ], "first_wave_evidence": "Wave1Outcomes.json", "second_wave_evidence": "Wave2Outcomes.json", - "portability_evidence": "PortabilityOutcomes.json" + "portability_evidence": "PortabilityOutcomes.json", + "coverage_wave_evidence": "CoverageWaveOutcomes.json" } diff --git a/Documentation/Qualification/Release1.2.1/CoverageWave.md b/Documentation/Qualification/Release1.2.1/CoverageWave.md index e57a787..dde8758 100644 --- a/Documentation/Qualification/Release1.2.1/CoverageWave.md +++ b/Documentation/Qualification/Release1.2.1/CoverageWave.md @@ -19,17 +19,66 @@ No public signatures, dependencies, products, profiles or fixtures change. Add `linux-coverage` to the existing two-architecture pinned Linux matrix. It retains exact discovery and source-denominator checks, atomic counters and the actual omitted-schema negative exporter probe. Exports preserve the five -actual objects plus merged profile; the exporter runs against those copies. +runners and matching Linux shared objects plus the merged profile; the exporter +runs against those copies. Generated tables, unmapped declarations, and unavailable branch/MC/DC counters remain separate. Existing Linux/native lane names and 49 metric probes remain. Initial focused native verification passed 32 functions (including one existing reader case) on Apple Swift 6.4 / Xcode 27.2 / macOS SDK 27.2. The Python harness -passed all ten negative/discovery cases. Full platform/safety/coverage outcomes -are pending; this document is not a qualification claim. The previous accepted -atomic coverage baseline is 6753/6929 lines and 7156/7691 regions. +passed all ten negative/discovery cases. The prior accepted atomic coverage baseline is 6753/6929 lines and 7156/7691 +regions. [Actual outcomes](CoverageWaveOutcomes.json) retain commands, source +identities, materialized receipt hashes and distinct platform scopes. The structural streaming completion alternative and unused private helpers keep their denominators. Proposed 98% line / 95% region per-module review targets, remaining public end-to-end/mutation work and deferred device scopes stay open until evidence justifies closure. + +## Qualified outcomes + +Native/debug/release/full ASan passed 276 functions / 173 parameterized +invocations; targeted TSan passed 22 functions. Native/docs passed all 582 +authored public symbols and compiled examples/catalogs. Both Linux architectures +passed full qualification/safety, 49 caps, consumers, corpus policies and clean +source reproduction. Both corrected coverage jobs passed; their original +runners-only exports remain failed evidence. Swift Build's Linux test/shared +objects contain the library code, so exports preserve five runners plus five +DSOs. Actual LLVM replay from preserved objects/profiles reproduces the totals. + +WASI's scoped lane passed 38 commands: Node full debug/release roots, 103 +QA functions on each Node/WasmKit configuration, and strict-memory compilation. +This scoped lane does not refresh separate independent corpora/consumer/cap +qualification; it does not claim unavailable WASI sanitizers. + +| Module | Authored lines | Authored regions | +| --- | ---: | ---: | +| JSON | 3369/3443 (97.85%) | 3252/3427 (94.89%) | +| JSONSchema | 2389/2417 (98.84%) | 2953/3091 (95.54%) | +| JSONRegexRuntime | 650/659 (98.63%) | 668/695 (96.12%) | +| JSONStreaming | 228/232 (98.28%) | 256/269 (95.17%) | +| JSONPatch | 184/185 (99.46%) | 210/212 (99.06%) | +| Total | 6820/6936 (98.33%) | 7339/7694 (95.39%) | + +Native and ARM64/x86_64 Linux totals agree. No denominator was reduced. Core's +unused JSONFingerprint and JSONOperationContext remain included; other public +and guarded paths still need separate review/tests. Execution percentages do +not establish assertion completeness; branch and MC/DC counters are unavailable. + +## Performance review and remaining work + +The required correctness fix was measured against its parent on an authored +32-issue complete-diagnostic workload. Three short paired runs have a +1.19% +median with a noisy +149% pair. Five longer paired runs have a +11.98% median, +crossing the unchanged 10% review trigger; paired changes range from -0.53% +to +73.21%. All samples remain retained. Logical peaks and owned work are equal. +The correctness repair is retained; timing attribution is inconclusive amid +large variance. Controlled quiet-host recalibration remains pending before a +1.2.1 release decision. No optimization, threshold or quota was changed. +This does not replace the full historical performance baseline or streaming +regression disposition. + +The 98/95 per-module targets remain open for core JSON, and the broader +end-to-end/mutation acceptance plan is unfinished. This closes the Linux export +and current behavior wave, not the whole proposed 1.2.1 program. Closing records +are a documentation successor; runtime/tests match the qualified heads. diff --git a/Documentation/Qualification/Release1.2.1/CoverageWaveOutcomes.json b/Documentation/Qualification/Release1.2.1/CoverageWaveOutcomes.json new file mode 100644 index 0000000..8bc7a17 --- /dev/null +++ b/Documentation/Qualification/Release1.2.1/CoverageWaveOutcomes.json @@ -0,0 +1,466 @@ +{ + "schema_version": 1, + "status": "functional_native_linux_wasi_qualified_performance_variance_followup", + "parent_wave": "PortabilityOutcomes.json", + "runtime_tested_head": "20476dace4626528aba36649df2686b2158c7f9f", + "measurement_tested_head": "708c26389ff17a46e327c03d61f32947a4104415", + "new_functions": 31, + "root_functions": 276, + "parameterized_invocations": 173, + "qa_functions": 103, + "shared_state_functions": 22, + "source_changes": [ + "Sources/JSONSchema/Evaluation/JSONSchemaEvaluation.swift" + ], + "preserved": [ + "four public products", + "private regex runtime", + "empty root dependency graph", + "Swift tools/compiler 6.4 and language mode 6", + "qualified Apple/Linux/WASI platform boundaries", + "all 49 caps", + "fixture pins and policy exclusions", + "prior W/G/RC/release/test-wave history", + "sibling 1.3.0 checkout" + ], + "native": { + "workflow": "https://github.com/ThorFuchs/Swift-JSON/actions/runs/38555689649", + "status": "success", + "lanes": [ + "native", + "safety", + "docs" + ], + "root_each_configuration": 276, + "parameterized_each_configuration": 173, + "full_asan": 276, + "tsan": 22, + "authored_public_docc": 582, + "documented_public_docc": 582, + "verification": { + "path": "NativeReceiptVerification.json", + "sha256": "f5be4ed3ac4802a04f15d9e7f34efe9fd67fefe78ec427b79660fcc812aa168d" + } + }, + "linux": { + "qualification_workflow": "https://github.com/ThorFuchs/Swift-JSON/actions/runs/38555685386", + "overall_status": "failure due to initial incomplete coverage exports", + "passed_jobs": [ + "arm64/linux", + "x86_64/linux", + "arm64/linux-safety", + "x86_64/linux-safety" + ], + "coverage_workflow": "https://github.com/ThorFuchs/Swift-JSON/actions/runs/38558125723", + "coverage_status": "success", + "coverage_jobs": 2, + "instrumented_objects_each_architecture": 10, + "qualification_verification": { + "path": "LinuxReceiptVerification.json", + "sha256": "7b39f0077bb0575061f83100d88cfa56a34b719a857944356fab532ede912c16" + }, + "coverage_verification": { + "path": "LinuxCoverageVerification.json", + "sha256": "6df9c86cfa0d397bc96ad4dc9f055cb76a44598b680dd4a36ea78c931efadf2d" + }, + "independent_llvm_replay": { + "path": "LinuxReplay.json", + "sha256": "55e05dec41b757dbe87a45cb6ad955637856b93c5bde2368d531cf77b938d741" + }, + "reproduction": "Both qualification jobs extracted and built/tested actual byte-reproducible source archives at runtime head." + }, + "wasi": { + "receipt": { + "path": "wasi/receipt.json", + "sha256": "ffd98420525598ce22ed03467a890cd3e6f655219519317d3b643bcbad87238d" + }, + "probe_receipt": { + "path": "wasi/probes/wasi.json", + "sha256": "72a819f46238d2cfa9109fc898fa4cff2274c06e859af6ffafab476edce86682" + }, + "status": "passed", + "commands": 38, + "scope": "Node full debug/release roots 276/173. Node and WasmKit affected QA 103 per configuration; strict-memory compilation. No independent consumer/cap/corpus refresh or sanitizer pass claim in scoped lane.", + "materialization": "WASIMaterialization.json" + }, + "coverage": { + "accepted_baseline": { + "lines": { + "count": 6929, + "covered": 6753 + }, + "regions": { + "count": 7691, + "covered": 7156 + } + }, + "current": { + "lines": { + "count": 6936, + "covered": 6820, + "percent": 98.3275663206459 + }, + "regions": { + "count": 7694, + "covered": 7339, + "percent": 95.38601507668314 + }, + "functions": { + "count": 1143, + "covered": 1123, + "percent": 98.25021872265967 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "modules": { + "JSON": { + "mapped_files": 50, + "authored": { + "lines": { + "count": 3443, + "covered": 3369, + "percent": 97.85071158873076 + }, + "regions": { + "count": 3427, + "covered": 3252, + "percent": 94.89349285088998 + }, + "functions": { + "count": 595, + "covered": 588, + "percent": 98.82352941176471 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 0, + "covered": 0, + "percent": null + }, + "regions": { + "count": 0, + "covered": 0, + "percent": null + }, + "functions": { + "count": 0, + "covered": 0, + "percent": null + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + "JSONSchema": { + "mapped_files": 32, + "authored": { + "lines": { + "count": 2417, + "covered": 2389, + "percent": 98.84153909805543 + }, + "regions": { + "count": 3091, + "covered": 2953, + "percent": 95.53542542866386 + }, + "functions": { + "count": 384, + "covered": 375, + "percent": 97.65625 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 0, + "covered": 0, + "percent": null + }, + "regions": { + "count": 0, + "covered": 0, + "percent": null + }, + "functions": { + "count": 0, + "covered": 0, + "percent": null + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + "JSONRegexRuntime": { + "mapped_files": 6, + "authored": { + "lines": { + "count": 659, + "covered": 650, + "percent": 98.63429438543247 + }, + "regions": { + "count": 695, + "covered": 668, + "percent": 96.11510791366906 + }, + "functions": { + "count": 88, + "covered": 86, + "percent": 97.72727272727273 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 30, + "covered": 30, + "percent": 100.0 + }, + "regions": { + "count": 44, + "covered": 44, + "percent": 100.0 + }, + "functions": { + "count": 3, + "covered": 3, + "percent": 100.0 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + "JSONStreaming": { + "mapped_files": 10, + "authored": { + "lines": { + "count": 232, + "covered": 228, + "percent": 98.27586206896552 + }, + "regions": { + "count": 269, + "covered": 256, + "percent": 95.16728624535315 + }, + "functions": { + "count": 60, + "covered": 58, + "percent": 96.66666666666667 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 0, + "covered": 0, + "percent": null + }, + "regions": { + "count": 0, + "covered": 0, + "percent": null + }, + "functions": { + "count": 0, + "covered": 0, + "percent": null + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + "JSONPatch": { + "mapped_files": 4, + "authored": { + "lines": { + "count": 185, + "covered": 184, + "percent": 99.45945945945945 + }, + "regions": { + "count": 212, + "covered": 210, + "percent": 99.05660377358491 + }, + "functions": { + "count": 16, + "covered": 16, + "percent": 100.0 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 0, + "covered": 0, + "percent": null + }, + "regions": { + "count": 0, + "covered": 0, + "percent": null + }, + "functions": { + "count": 0, + "covered": 0, + "percent": null + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + } + }, + "native_and_both_linux_totals_equal": true, + "counter_policy": "Atomic increments; reject saturated segments. Actual omitted-schema negative probe.", + "classification_policy": "Generated tables separate; declaration-only unmapped files explicit; unused/unreachable paths retained. Branch and MC/DC unavailable, not percentages.", + "review_target": "Core remains below proposed per-module 98% lines / 95% regions. All other modules exceed both targets." + }, + "independent_suites": { + "strict_json": { + "observed": 318, + "evaluated_passed": 287, + "policy_cases": 31, + "failed": 0 + }, + "schema": { + "observed": 2337, + "evaluated_passed": 2300, + "policy_cases": 37, + "failed": 0 + }, + "authored_public_semantics": 82, + "public_budget_counter_cases": 9, + "malformed_replay": 4396, + "malformed_seed": "0x53574946544a534f", + "rooted_consumer": { + "explicit_cases": 20, + "valid": 8, + "invalid": 12, + "operational_errors": 2, + "mcp_server_conformance_claim": false + } + }, + "performance": { + "review": { + "path": "Calibration/Review.json", + "sha256": "abfb74a8e367f6580787352c69b216b6574b67dad21ef25cb941398e7393e6c9" + }, + "short_median_change_percent": 1.193816893582933, + "long_median_change_percent": 11.98325099056705, + "trigger_percent": 10, + "disposition": "Required correctness fix retained after review; wall-clock attribution remains inconclusive amid high variance. Controlled quiet-host recalibration is pending before release. Logical retention/work equal; no quota or optimization changes." + }, + "retained_failures": [ + "Initial compile/oracle corrections in focused logs", + "Focused-5 ENOSPC build exit 1, followed by successful run", + "Initial ARM64/x86 Linux exports correctly rejected runners-only mapping", + "Initial independent replay needed exact path-field rebasing for validation", + "Short and long noisy timing windows, no sample discarded" + ], + "remaining": [ + "Core per-module review targets not reached; unused helpers remain included", + "Broader T01-T07 acceptance/e2e/mutation work remains open", + "Performance variance followup before any 1.2.1 release decision", + "WASI sanitizers unavailable and physical iPad deferred under existing policy" + ], + "no_merge_tag_publication": true, + "evidence_paths": "Relative paths resolve within the materialized Evidence directory of the private wave bundle; hashes bind bytes, GitHub links bind executions." +} From df3e382526d799963806ef9c73ecbb237981e9bc Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Wed, 14 Oct 2026 14:49:14 +0200 Subject: [PATCH 14/24] test: cover reachable core conversion and quota boundaries --- .../Qualification/TestDiscovery.json | 4 +- Tests/JSONTests/CoreClosureTests.swift | 284 ++++++++++++++++++ 2 files changed, 286 insertions(+), 2 deletions(-) create mode 100644 Tests/JSONTests/CoreClosureTests.swift diff --git a/Documentation/Qualification/TestDiscovery.json b/Documentation/Qualification/TestDiscovery.json index e70e1b3..04719e0 100644 --- a/Documentation/Qualification/TestDiscovery.json +++ b/Documentation/Qualification/TestDiscovery.json @@ -1,7 +1,7 @@ { "schema_version": 1, "root_functions": { - "JSONTests": 154, + "JSONTests": 171, "JSONSchemaTests": 61, "JSONRegexRuntimeTests": 16, "JSONStreamingTests": 30, @@ -9,7 +9,7 @@ }, "parameterized_invocations": 173, "qa_functions": { - "JSONTests": 61, + "JSONTests": 78, "JSONSchemaTests": 13, "JSONStreamingTests": 19, "JSONPatchTests": 6, diff --git a/Tests/JSONTests/CoreClosureTests.swift b/Tests/JSONTests/CoreClosureTests.swift new file mode 100644 index 0000000..53a7f64 --- /dev/null +++ b/Tests/JSONTests/CoreClosureTests.swift @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation +import Testing +import JSON + +private func closureFailure(_ category: JSONFailure.Category, _ code: JSONFailure.Code, + metric: JSONResourceMetric? = nil, _ body: () throws -> Void) { + do { try body(); Issue.record("Expected \(category)/\(code)") } + catch let failure as JSONFailure { + #expect(failure.category == category && failure.code == code) + if let metric { #expect(failure.resource?.metric == metric) } + } catch { Issue.record("Unexpected error: \(error)") } +} + +@Test func qaCoreNumericTextPreservesLexemesAndFallsBackUnderInclusiveOutputCaps() throws { + let parsed = try JSONNumber(parsing: "-12.00e3") + #expect(try parsed.numericText(style: .normalized) == "-12e3") + #expect(try parsed.numericText(style: .preserved) == "-12.00e3") + #expect(try JSONNumber(integer: -12).numericText(style: .preserved) == "-12") + var limits = JSONLimits.balanced; limits[.outputBytes] = 8 + #expect(try parsed.numericText(style: .preserved, limits: limits) == "-12.00e3") + limits[.outputBytes] = 7 + closureFailure(.resource, .limitExceeded, metric: .outputBytes) { + _ = try parsed.numericText(style: .preserved, limits: limits) + } +} + +@Test func qaCoreBinaryZeroSubnormalAndNonfiniteConversionsHaveExactOutcomes() throws { + for text in ["0", "-0", "0e-999"] { + #expect(try JSONNumber(parsing: text).doubleValue(policy: .exact) == 0) + } + for text in ["5e-324", "-5e-324"] { + let number = try JSONNumber(parsing: text) + let expected = text.hasPrefix("-") ? -Double.leastNonzeroMagnitude : Double.leastNonzeroMagnitude + #expect(try number.doubleValue(policy: .rounded).bitPattern == expected.bitPattern) + closureFailure(.conversion, .inexactConversion) { _ = try number.doubleValue(policy: .exact) } + } + #expect(try JSONNumber(parsing: "1e-1000").doubleValue(policy: .rounded) == 0) + for value in [Double.nan, Double.infinity, -Double.infinity] { + closureFailure(.conversion, .inexactConversion) { _ = try JSONNumber.fromFiniteDouble(value) } + } +} + +@Test func qaCoreSourceDocumentFinishesNumbersAtEOFWithCoherentSpans() throws { + for (text, expected) in [("123", JSON.number(JSONNumber(integer: 123))), + ("-12.50e2", JSON.number(JSONNumber(integer: -1250))), ("0", JSON.number(JSONNumber(integer: 0)))] { + let bytes = Data(text.utf8) + let document = try JSONParser().parseDocument(bytes, source: "authored-eof") + #expect(document.root == expected) + #expect(document.source == "authored-eof" && document.sourceBytes == bytes) + let map = try #require(document.sourceMap) + let span = try #require(try map.span(at: [])) + #expect(span.start == 0 && span.end == UInt64(bytes.count)) + #expect(try map.span(at: [], kind: .memberKey) == nil) + let replacement = document.replacingRoot(.null) + #expect(replacement.source == nil && replacement.sourceBytes == nil && replacement.sourceMap == nil) + } +} + +@Test func qaCoreSourceMapDistinguishesTypedKeyAndIndexPaths() throws { + let document = try JSONParser().parseDocument(Data(#"{"0":[17]}"#.utf8)) + let map = try #require(document.sourceMap) + let key = try #require(try map.span(at: [.key("0")], kind: .memberKey)) + let element = try #require(try map.span(at: [.key("0"), .index(0)])) + #expect(key.start == 1 && key.end == 4 && element.start == 6 && element.end == 8) + #expect(try map.span(at: [.index(0)]) == nil) + #expect(try map.span(at: [.key("0"), .key("0")]) == nil) +} + +@Test func qaCoreValueMemberAndPresenceRefusalsKeepKindsDistinct() throws { + closureFailure(.typeMismatch, .wrongKind) { _ = try JSON.null.requiredMember("x") } + let value = try JSONParser().parse(#"{"null":null,"é":1,"e\u0301":2}"#) + #expect(try value.presence(of: "null") == .null) + #expect(try value.presence(of: "missing") == .absent) + #expect(try value.requiredMember("null") == .null) + #expect(try value.requiredMember("é") == 1 && value.requiredMember("e\u{301}") == 2) + closureFailure(.missingValue, .missingMember) { _ = try value.requiredMember("missing") } +} + +@Test func qaCoreJSONKeyForeignAndNativeCodablePreserveScalarSpelling() throws { + for text in ["", "a/~", "é", "e\u{301}", "🚀"] { + let key = JSONKey(text) + let foreign = try Foundation.JSONEncoder().encode(key) + let decoded = try Foundation.JSONDecoder().decode(JSONKey.self, from: foreign) + #expect(Array(decoded.string.utf8) == Array(text.utf8)) + let native = try JSONValueEncoder().encode(key) + #expect(native == .string(text)) + #expect(try JSONValueDecoder().decode(JSONKey.self, from: native) == key) + } +} + +private struct DelegatedExactNumber: Codable { + let number: JSONNumber + init(_ number: JSONNumber) { self.number = number } + init(from decoder: any Decoder) throws { number = try JSONNumber(from: decoder) } + func encode(to encoder: any Encoder) throws { try number.encode(to: encoder) } +} +private struct NumberEnvelope: Codable { let payload: DelegatedExactNumber } + +@Test func qaCoreCustomModelCanDelegateExactNumberCodingWithoutForeignProjection() throws { + let number = try JSONNumber(parsing: "900719925474099312345678901234567890123456789.00") + let model = NumberEnvelope(payload: DelegatedExactNumber(number)) + let encoded = try JSONValueEncoder().encode(model) + #expect(try encoded.requiredMember("payload") == .number(number)) + let decoded = try encoded.decodeWithReport(NumberEnvelope.self) + #expect(decoded.result.payload.number == number && decoded.statistics[.workUnits] > 0) + #expect(try decoded.result.payload.number.numericText(style: .preserved) == "900719925474099312345678901234567890123456789.00") + do { + _ = try JSONValueDecoder().decode(NumberEnvelope.self, from: JSONParser().parse(#"{"payload":false}"#)) + Issue.record("Boolean must not decode as a number") + } catch DecodingError.typeMismatch(let type, let context) { + #expect(type == JSONNumber.self && context.codingPath.map(\.stringValue) == ["payload"]) + #expect((context.underlyingError as? JSONFailure)?.code == .wrongKind) + } +} + +private struct DataEnvelope: Decodable { let payload: Data } +private struct DateEnvelope: Decodable { let payload: Date } +private struct IntegerEnvelope: Decodable { let payload: Int } +private func wrongKind(_ type: T.Type, value: JSON, options: JSONCodingOptions = .init()) throws { + do { _ = try JSONValueDecoder(options: options).decode(type, from: value); Issue.record("Expected wrong kind") } + catch DecodingError.typeMismatch(_, let context) { + #expect(context.codingPath.map(\.stringValue) == ["payload"]) + #expect((context.underlyingError as? JSONFailure)?.category == .typeMismatch) + #expect((context.underlyingError as? JSONFailure)?.code == .wrongKind) + } +} + +@Test func qaCoreNativeDataDateAndIntegerWrongKindsKeepNestedPaths() throws { + let value = try JSONParser().parse(#"{"payload":false}"#) + try wrongKind(DataEnvelope.self, value: value) + try wrongKind(DateEnvelope.self, value: value, options: .init(dateStrategy: .iso8601)) + try wrongKind(IntegerEnvelope.self, value: value) +} + +@Test func qaCoreMalformedBase64ReportsSyntaxAtTheNestedField() throws { + for text in ["AA!A", "AAA!"] { + let value = JSON.object(try JSONObject(members: [.init("payload", value: .string(text))])) + do { _ = try value.decode(DataEnvelope.self); Issue.record("Expected malformed base64") } + catch DecodingError.dataCorrupted(let context) { + #expect(context.codingPath.map(\.stringValue) == ["payload"]) + #expect((context.underlyingError as? JSONFailure)?.category == .syntax) + #expect((context.underlyingError as? JSONFailure)?.code == .invalidSyntax) + } + } +} + +@Test func qaCoreMalformedISO8601IsCorruptionRatherThanWrongKind() throws { + let value = try JSONParser().parse(#"{"payload":"not a date"}"#) + do { + _ = try value.decode(DateEnvelope.self, coding: .init(dateStrategy: .iso8601)) + Issue.record("Expected malformed date") + } catch DecodingError.dataCorrupted(let context) { + #expect(context.codingPath.map(\.stringValue) == ["payload"]) + } +} + +@Test func qaCoreMalformedBase64ReleasesScratchAndKeepsScopeWorkCumulative() throws { + let value = try JSONParser().parse(#"{"bad":"Zm9vAAA!","good":17}"#) + func probe(_ attempts: Int, _ limits: JSONLimits) throws -> JSONOperationReport { + try value.readWithReport(limits: limits) { root in + let bad = try root["bad"] + for _ in 0.. once.statistics[.workUnits]) +} + +@Test func qaCoreScalarSortedPrefixKeysHaveIndependentGoldenAndCumulativeWork() throws { + let keys = ["🚀", "é", "e\u{301}", "aa", "a/", "a", ""] + let object = try JSONObject(members: keys.map { JSONMember($0, value: .null) }) + let golden = #"{"":null,"a":null,"a/":null,"aa":null,"é":null,"é":null,"🚀":null}"# + let writer = JSONWriter(options: .init(keyOrder: .scalar)) + let baseline = try writer.writeWithReport(.object(object)) + #expect(Array(baseline.result) == Array(golden.utf8)) + #expect(object.members.map(\.key) == keys.map { JSONKey($0) }) + var limits = JSONLimits.balanced; limits[.workUnits] = baseline.statistics[.workUnits] + #expect(try JSONWriter(options: .init(limits: limits, keyOrder: .scalar)).write(.object(object)) == baseline.result) + limits[.workUnits] -= 1 + closureFailure(.resource, .limitExceeded, metric: .workUnits) { + _ = try JSONWriter(options: .init(limits: limits, keyOrder: .scalar)).write(.object(object)) + } +} + +@Test func qaCoreWriterRejectsInvalidIndentationBeforeOutput() { + for indentation in [-1, 9] { + closureFailure(.configuration, .invalidLimits) { + _ = try JSONWriter(options: .init(indentation: indentation)).write(.null) + } + } +} + +@Test func qaCoreRemovingAbsentMemberPreservesSmallAndIndexedObjectsButStillAdmits() throws { + for count in [0, 1, 31, 32, 64] { + let object = try JSONObject(members: (0.. 0 && work <= 8192) + let golden = try JSONObject(members: (0..<64).map { JSONMember("k\($0)", value: .null) }) + #expect(baseline.result == .object(golden)) + for quota in 0.. quota) + } + } + var limits = JSONLimits.balanced; limits[.workUnits] = work + let accepted = try JSONValueEncoder(options: .init(limits: limits)).encodeWithReport(BoundedWideFields()) + #expect(accepted.result == baseline.result && accepted.statistics[.workUnits] == work) + #expect(golden.count == 64 && golden["k63"] == .null) +} From 2b4ab659050b7d20917db0de1bf7d29bb1d6dfb4 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Wed, 14 Oct 2026 15:09:54 +0200 Subject: [PATCH 15/24] test: exercise remaining public core cleanup paths --- .../Qualification/TestDiscovery.json | 8 +-- Tests/JSONPatchTests/CoverageGapTests.swift | 15 +++++ Tests/JSONTests/CoreClosureTests.swift | 63 +++++++++++++++++++ Tests/JSONTests/ForeignCodableTests.swift | 10 +++ 4 files changed, 92 insertions(+), 4 deletions(-) diff --git a/Documentation/Qualification/TestDiscovery.json b/Documentation/Qualification/TestDiscovery.json index 04719e0..d1e8a60 100644 --- a/Documentation/Qualification/TestDiscovery.json +++ b/Documentation/Qualification/TestDiscovery.json @@ -1,18 +1,18 @@ { "schema_version": 1, "root_functions": { - "JSONTests": 171, + "JSONTests": 175, "JSONSchemaTests": 61, "JSONRegexRuntimeTests": 16, "JSONStreamingTests": 30, - "JSONPatchTests": 15 + "JSONPatchTests": 16 }, "parameterized_invocations": 173, "qa_functions": { - "JSONTests": 78, + "JSONTests": 82, "JSONSchemaTests": 13, "JSONStreamingTests": 19, - "JSONPatchTests": 6, + "JSONPatchTests": 7, "JSONRegexRuntimeTests": 4 }, "shared_state_functions": { diff --git a/Tests/JSONPatchTests/CoverageGapTests.swift b/Tests/JSONPatchTests/CoverageGapTests.swift index 48cfb8c..dcb788b 100644 --- a/Tests/JSONPatchTests/CoverageGapTests.swift +++ b/Tests/JSONPatchTests/CoverageGapTests.swift @@ -62,3 +62,18 @@ private func patchGapFailure(_ code: JSONFailure.Code, _ body: () throws -> Void _ = try JSONPatch(operations:[.move(from:JSONPointer(parsing:""),path:JSONPointer(parsing:"/b"))]).applying(to:input) } } + +@Test func qaPatchNestedArrayEditsRebuildAncestorsAndRefuseScalarParentsTransactionally() throws { + let input = try JSONParser().parse(#"{"a/b":[{"~":[1,2]},null]}"#) + let path = try JSONPointer(parsing: "/a~1b/0/~0/1") + let patch = try JSONPatch(operations: [.replace(path: path, value: 17), + .add(path: JSONPointer(parsing: "/a~1b/0/~0/-"), value: 3), .remove(path: JSONPointer(parsing: "/a~1b/1"))]) + let expected = try JSONParser().parse(#"{"a/b":[{"~":[1,17,3]}]}"#) + #expect(try patch.applying(to: input) == .document(expected)) + for badPath in ["/a~1b/1/child", "/a~1b/1/child/deeper"] { + let failing = try JSONPatch(operations: [.replace(path: path, value: 17), .add(path: JSONPointer(parsing: badPath), value: 3)]) + do { _ = try failing.applying(to: input); Issue.record("Expected scalar parent refusal") } + catch let failure as JSONFailure { #expect(failure.category == .typeMismatch && failure.code == .wrongKind) } + #expect(try input == JSONParser().parse(#"{"a/b":[{"~":[1,2]},null]}"#)) + } +} diff --git a/Tests/JSONTests/CoreClosureTests.swift b/Tests/JSONTests/CoreClosureTests.swift index 53a7f64..120a96b 100644 --- a/Tests/JSONTests/CoreClosureTests.swift +++ b/Tests/JSONTests/CoreClosureTests.swift @@ -117,6 +117,15 @@ private struct NumberEnvelope: Codable { let payload: DelegatedExactNumber } private struct DataEnvelope: Decodable { let payload: Data } private struct DateEnvelope: Decodable { let payload: Date } private struct IntegerEnvelope: Decodable { let payload: Int } +private struct ExplicitIntegerEnvelope: Decodable { + enum Key: String, CodingKey { case payload } + let payload: Int8 + init(from decoder: any Decoder) throws { + let object = try decoder.container(keyedBy: Key.self) + let child = try object.superDecoder(forKey: .payload) + payload = try child.singleValueContainer().decode(Int8.self) + } +} private func wrongKind(_ type: T.Type, value: JSON, options: JSONCodingOptions = .init()) throws { do { _ = try JSONValueDecoder(options: options).decode(type, from: value); Issue.record("Expected wrong kind") } catch DecodingError.typeMismatch(_, let context) { @@ -131,6 +140,7 @@ private func wrongKind(_ type: T.Type, value: JSON, options: JSONC try wrongKind(DataEnvelope.self, value: value) try wrongKind(DateEnvelope.self, value: value, options: .init(dateStrategy: .iso8601)) try wrongKind(IntegerEnvelope.self, value: value) + try wrongKind(ExplicitIntegerEnvelope.self, value: value) } @Test func qaCoreMalformedBase64ReportsSyntaxAtTheNestedField() throws { @@ -282,3 +292,56 @@ private struct BoundedWideFields: Encodable { #expect(accepted.result == baseline.result && accepted.statistics[.workUnits] == work) #expect(golden.count == 64 && golden["k63"] == .null) } + +@Test func qaCoreFiniteDoubleRoundTripsAndDecimalOverflowHaveExplicitFidelity() throws { + for value in [0.0, -12.5, 0.1, Double.greatestFiniteMagnitude] { + let number = try JSONNumber.fromFiniteDouble(value) + #expect(try number.doubleValue(policy: .rounded) == value) + } + let decimalTenth = try JSONNumber.fromFiniteDouble(0.1) + #expect(decimalTenth == (try JSONNumber(parsing: "0.1"))) + closureFailure(.conversion, .inexactConversion) { _ = try decimalTenth.doubleValue(policy: .exact) } + for policy in [JSONNumberConversionPolicy.exact, .rounded] { + closureFailure(.conversion, .inexactConversion) { _ = try JSONNumber(parsing: "9e165").decimalValue(policy: policy) } + } +} + +@Test func qaCoreMappedErrorConstructionSharesWorkAndCannotMaskQuotaFailure() throws { + let document = try JSONParser().parseDocument(Data(#"{"x":false}"#.utf8)) + func probe(_ limits: JSONLimits) throws -> JSONOperationReport { + try document.readWithReport(limits: limits) { root in + do { _ = try root["x"].requiredNumber(); Issue.record("Expected wrong kind") } + catch let failure as JSONAccessFailure { + #expect(failure.path == [.key("x")] && failure.cause.code == .wrongKind) + #expect(failure.cause.location?.byteOffset == 5) + } + return 17 + } + } + let baseline = try probe(.balanced); let work = baseline.statistics[.workUnits] + try #require(work > 0 && work <= 1024) + for quota in 0.. 0 && work <= 2048) + for quota in 0.. Date: Wed, 14 Oct 2026 15:33:14 +0200 Subject: [PATCH 16/24] test: qualify escaped output pointers and value presence --- Tests/JSONSchemaTests/CoverageGapTests.swift | 17 +++++++++++++++++ Tests/JSONTests/CoreClosureTests.swift | 2 ++ 2 files changed, 19 insertions(+) diff --git a/Tests/JSONSchemaTests/CoverageGapTests.swift b/Tests/JSONSchemaTests/CoverageGapTests.swift index 1ca21c5..07d768c 100644 --- a/Tests/JSONSchemaTests/CoverageGapTests.swift +++ b/Tests/JSONSchemaTests/CoverageGapTests.swift @@ -76,6 +76,23 @@ private func gapCompilation(_ category: JSONSchemaCompilationFailure.Category, _ #expect(issue.absoluteKeywordLocation?.string == "https://e.test/root#/properties/~1~0/type") #expect(result.issues.contains { $0.instanceLocation.tokens == [JSONKey("é")] }) #expect(!result.issues.contains { $0.instanceLocation.tokens == [JSONKey("e\u{301}")] }) + let output = try result.outputWithReport(.basic) + let errors = try #require(try output.result.requiredMember("errors").arrayValue) + #expect(errors.count == 2) + let escaped = try #require(errors.first { $0.objectValue?["instanceLocation"] == .string("/~1~0") }) + #expect(try escaped.requiredMember("keywordLocation") == .string("/properties/~1~0/type")) + #expect(try escaped.requiredMember("absoluteKeywordLocation") == .string("https://e.test/root#/properties/~1~0/type")) + #expect(try escaped.requiredMember("valid") == .boolean(false)) + var limits = JSONLimits.balanced; limits[.workUnits] = output.statistics[.workUnits] + #expect(try result.output(.basic, limits: limits) == output.result) + limits[.workUnits] -= 1 + do { _ = try result.output(.basic, limits: limits); Issue.record("Expected cumulative output work refusal") } + catch let failure as JSONFailure { #expect(failure.category == .resource && failure.resource?.metric == .workUnits) } + limits = .balanced; limits[.pointerBytes] = UInt64("/properties/~1~0/type".utf8.count) + #expect(try result.output(.basic, limits: limits) == output.result) + limits[.pointerBytes] -= 1 + do { _ = try result.output(.basic, limits: limits); Issue.record("Expected encoded Pointer byte refusal") } + catch let failure as JSONFailure { #expect(failure.category == .resource && failure.resource?.metric == .pointerBytes) } } @Test func qaSchemaEmptyArrayAndNonStringAnnotationApplicability() throws { diff --git a/Tests/JSONTests/CoreClosureTests.swift b/Tests/JSONTests/CoreClosureTests.swift index 120a96b..a7d4375 100644 --- a/Tests/JSONTests/CoreClosureTests.swift +++ b/Tests/JSONTests/CoreClosureTests.swift @@ -72,6 +72,8 @@ private func closureFailure(_ category: JSONFailure.Category, _ code: JSONFailur let value = try JSONParser().parse(#"{"null":null,"é":1,"e\u0301":2}"#) #expect(try value.presence(of: "null") == .null) #expect(try value.presence(of: "missing") == .absent) + #expect(try value.presence(of: "é") == .value(1)) + #expect(try value.presence(of: "e\u{301}") == .value(2)) #expect(try value.requiredMember("null") == .null) #expect(try value.requiredMember("é") == 1 && value.requiredMember("e\u{301}") == 2) closureFailure(.missingValue, .missingMember) { _ = try value.requiredMember("missing") } From c11d96ff8197362537e8353e1ad3199d4f95df3f Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:00:23 +0200 Subject: [PATCH 17/24] docs: record core coverage closure and quiet-host blocker --- .../Release1.2.1/AcceptanceMatrix.json | 41 +- .../Qualification/Release1.2.1/CoreClosure.md | 84 ++ .../Release1.2.1/CoreClosureOutcomes.json | 941 ++++++++++++++++++ .../Qualification/Release1.2.1/README.md | 4 + 4 files changed, 1056 insertions(+), 14 deletions(-) create mode 100644 Documentation/Qualification/Release1.2.1/CoreClosure.md create mode 100644 Documentation/Qualification/Release1.2.1/CoreClosureOutcomes.json diff --git a/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json b/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json index f7e040b..3e4f1a9 100644 --- a/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json +++ b/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json @@ -3,7 +3,7 @@ "source_version": "1.2.0", "source_commit": "25fb04f747e583209b4c4511abf39e11ef5b4f80", "target_version": "1.2.1", - "status": "native_linux_and_scoped_wasi_qualified_remaining_1_2_1_plan", + "status": "core_public_paths_qualified_quiet_diagnostic_calibration_blocked_broader_plan_open", "rows": [ { "id": "HAR-01", @@ -24,7 +24,8 @@ ], "first_wave_evidence": "Wave1Outcomes.json", "portability_evidence": "PortabilityOutcomes.json", - "coverage_wave_evidence": "CoverageWaveOutcomes.json" + "coverage_wave_evidence": "CoverageWaveOutcomes.json", + "core_closure_evidence": "CoreClosureOutcomes.json" }, { "id": "HAR-02", @@ -41,7 +42,8 @@ "no artificial test-only callers", "cleanup separate from tests-only scope" ], - "first_wave_evidence": "Wave1Outcomes.json" + "first_wave_evidence": "Wave1Outcomes.json", + "core_closure_evidence": "CoreClosureOutcomes.json" }, { "id": "CORE-01", @@ -63,7 +65,8 @@ ], "first_wave_evidence": "Wave1Outcomes.json", "second_wave_evidence": "Wave2Outcomes.json", - "portability_evidence": "PortabilityOutcomes.json" + "portability_evidence": "PortabilityOutcomes.json", + "core_closure_evidence": "CoreClosureOutcomes.json" }, { "id": "CORE-02", @@ -83,7 +86,8 @@ "equal values hash equally within a process", "order stable on replace/remove/reinsert", "no persistent hash claim" - ] + ], + "core_closure_evidence": "CoreClosureOutcomes.json" }, { "id": "COD-01", @@ -103,7 +107,8 @@ "successful roundtrip plus independent expected representation" ], "first_wave_evidence": "Wave1Outcomes.json", - "portability_evidence": "PortabilityOutcomes.json" + "portability_evidence": "PortabilityOutcomes.json", + "core_closure_evidence": "CoreClosureOutcomes.json" }, { "id": "COD-02", @@ -145,7 +150,8 @@ "all six kinds represented" ], "second_wave_evidence": "Wave2Outcomes.json", - "portability_evidence": "PortabilityOutcomes.json" + "portability_evidence": "PortabilityOutcomes.json", + "core_closure_evidence": "CoreClosureOutcomes.json" }, { "id": "COD-04", @@ -163,7 +169,8 @@ "correct admitted result", "configured limits intersection", "no duplicate/reset work" - ] + ], + "core_closure_evidence": "CoreClosureOutcomes.json" }, { "id": "DIA-01", @@ -184,7 +191,8 @@ "source replacement invalidates mapping", "caller description not traversed" ], - "first_wave_evidence": "Wave1Outcomes.json" + "first_wave_evidence": "Wave1Outcomes.json", + "core_closure_evidence": "CoreClosureOutcomes.json" }, { "id": "DIA-02", @@ -244,7 +252,8 @@ "consumed work retained", "caller work excluded" ], - "first_wave_evidence": "Wave1Outcomes.json" + "first_wave_evidence": "Wave1Outcomes.json", + "core_closure_evidence": "CoreClosureOutcomes.json" }, { "id": "SCH-01", @@ -307,7 +316,8 @@ "selected builder phases cumulative", "compile error versus resource/cancel distinguished" ], - "coverage_wave_evidence": "CoverageWaveOutcomes.json" + "coverage_wave_evidence": "CoverageWaveOutcomes.json", + "core_closure_evidence": "CoreClosureOutcomes.json" }, { "id": "REG-01", @@ -414,7 +424,8 @@ "input unchanged on failure", "irrelevant fields treated per RFC" ], - "coverage_wave_evidence": "CoverageWaveOutcomes.json" + "coverage_wave_evidence": "CoverageWaveOutcomes.json", + "core_closure_evidence": "CoreClosureOutcomes.json" }, { "id": "PAT-02", @@ -517,7 +528,8 @@ "first_wave_evidence": "Wave1Outcomes.json", "second_wave_evidence": "Wave2Outcomes.json", "portability_evidence": "PortabilityOutcomes.json", - "coverage_wave_evidence": "CoverageWaveOutcomes.json" + "coverage_wave_evidence": "CoverageWaveOutcomes.json", + "core_closure_evidence": "CoreClosureOutcomes.json" } ], "proposed_targets": { @@ -541,5 +553,6 @@ "first_wave_evidence": "Wave1Outcomes.json", "second_wave_evidence": "Wave2Outcomes.json", "portability_evidence": "PortabilityOutcomes.json", - "coverage_wave_evidence": "CoverageWaveOutcomes.json" + "coverage_wave_evidence": "CoverageWaveOutcomes.json", + "core_closure_evidence": "CoreClosureOutcomes.json" } diff --git a/Documentation/Qualification/Release1.2.1/CoreClosure.md b/Documentation/Qualification/Release1.2.1/CoreClosure.md new file mode 100644 index 0000000..d6fa4ea --- /dev/null +++ b/Documentation/Qualification/Release1.2.1/CoreClosure.md @@ -0,0 +1,84 @@ +# Reachable core coverage and diagnostic calibration + +This wave adds 22 meaningful Swift Testing functions on the isolated 1.2.1 branch: +21 in JSON and one public Patch integration case. Existing schema-output and +value-presence cases gain exact assertions. Root discovery is 298 functions and +173 parameterized invocations; QA is 125 functions and shared-state safety 22. + +The cases cover preserved numeric spelling, finite/nonfinite/subnormal conversion, +EOF source documents and typed spans, exact foreign keys and delegated native +numbers, nested conversion failures, scratch release, scalar prefix sorting, +absent-member edits, reused containers, integer dictionaries, exhaustive bounded +work-quota sweeps, persistence sink refusal, mapped-error work, nested array +editing, and escaped schema-output Pointers. Production source and APIs are +unchanged from the previously qualified best-effort diagnostic repair. + +A deliberately bounded exhaustive sweep asserts every insufficient work quota +and the exact successful boundary. Source-mapped error construction, payload +field construction, selection, projection and ordinary refusals retain one owned +ledger. Native model Codable paths are relative to the selected projection root; +read-access failures retain the complete typed JSON path. + +The source receipts preserve the initial compiler mistakes and an incorrect +selected-root path assumption. That failing run produced 33 assertions; correction +follows the inspected existing API rather than weakening a released contract. +The first WASI setup also failed because a concurrent SwiftPM lock message entered +manifest JSON. It ran no WASI behavior tests and is retained beside the sequential +rerun. No production test or corpus case was excluded to obtain a pass. + +## Performance disposition + +No new timing comparison ran. Five native observations failed the predeclared +quiet-host gate: load1 31.14–35.19 on ten logical cores; CPU idle 0.24–10.22%. +The readiness command exited 2, and no other session/application was stopped. +The user explicitly authorized recording a blocker if competing activity persisted. +The frozen original short/long runs, binaries and input hashes remain preserved. + +The +11.983% longer diagnostic median remains above the unchanged 10% review +trigger. Its paired changes range from -0.53% to +73.21%; attribution is unresolved. +Retain the required correctness repair and recalibrate on a genuinely quiet host +before the 1.2.1 release decision. No threshold, quota or optimization was changed. +Historical streaming performance disposition is unchanged. Logical retained bytes +and owned work are equal in the original pairs; neither describes process RSS +or arbitrary caller instruction quotas. + +## Scope and remaining work + +Coverage retains its full denominator, including 53 authored lines in unused +JSONFingerprint/JSONOperationContext helpers. Guarded states, arithmetic defenses, +generic-expression continuations and unavailable branch/MC/DC counters are +explicitly distinguished from asserted public behavior. Execution percentages +alone establish no exhaustive assertion or decision coverage. + +The broader E2E/mutation acceptance plan remains open. No Apple-device scope, +WASI sanitizer, backend, API, dependency, optimization, sibling 1.3.0 file, +merge, tag or release publication is added by this wave. + +## Qualified outcomes + +Native debug/release/full ASan passed 298 functions / 173 parameterized +invocations; TSan passed 22 functions. All 49 cap probes, strict-memory +compilation, public consumers, independent policies and 582-symbol DocC passed. +Both Linux architectures passed full qualification/safety and complete atomic +exports. Node/WasmKit scoped WASI passed 38 commands. Actual source and every +downloaded command log were compared with the final tested commit. + +| Module | Authored lines | Authored regions | +| --- | ---: | ---: | +| JSON | 3381/3443 (98.20%) | 3316/3427 (96.76%) | +| JSONSchema | 2389/2417 (98.84%) | 2954/3091 (95.57%) | +| JSONRegexRuntime | 650/659 (98.63%) | 668/695 (96.12%) | +| JSONStreaming | 228/232 (98.28%) | 256/269 (95.17%) | +| JSONPatch | 184/185 (99.46%) | 210/212 (99.06%) | + +All five modules meet the proposed 98/95 review targets without denominator +changes. Native and both Linux totals agree. The residual review retains 111 +never-hit code-region mappings, including unused helpers and generic variants. +This does not prove branch or MC/DC completeness. + +[Actual outcomes and evidence index](CoreClosureOutcomes.json) distinguish the +scopes and include the final [Native run](https://github.com/ThorFuchs/Swift-JSON/actions/runs/38572760920) +and [Linux run](https://github.com/ThorFuchs/Swift-JSON/actions/runs/38572755403). +The materialized companion bundle supplies profiles, native/Linux objects, WASI +modules, logs, source identities and original calibration inputs/binaries; +private build artifacts remain outside Git and public source archives. diff --git a/Documentation/Qualification/Release1.2.1/CoreClosureOutcomes.json b/Documentation/Qualification/Release1.2.1/CoreClosureOutcomes.json new file mode 100644 index 0000000..87784d8 --- /dev/null +++ b/Documentation/Qualification/Release1.2.1/CoreClosureOutcomes.json @@ -0,0 +1,941 @@ +{ + "schema_version": 1, + "status": "core_public_paths_qualified_quiet_diagnostic_calibration_blocked_broader_plan_open", + "baseline_head": "288dc626d7a44240ffca2fa4e12ea4ce68662c66", + "test_head": "9184481e30e4f7faf262f335f775f8732c634040", + "commits": [ + "df3e382526d799963806ef9c73ecbb237981e9bc", + "2b4ab659050b7d20917db0de1bf7d29bb1d6dfb4", + "9184481e30e4f7faf262f335f775f8732c634040" + ], + "new_functions": { + "JSONTests": 21, + "JSONPatchTests": 1 + }, + "discovery": { + "schema_version": 1, + "root_functions": { + "JSONTests": 175, + "JSONSchemaTests": 61, + "JSONRegexRuntimeTests": 16, + "JSONStreamingTests": 30, + "JSONPatchTests": 16 + }, + "parameterized_invocations": 173, + "qa_functions": { + "JSONTests": 82, + "JSONSchemaTests": 13, + "JSONStreamingTests": 19, + "JSONPatchTests": 7, + "JSONRegexRuntimeTests": 4 + }, + "shared_state_functions": { + "JSONTests": 4, + "JSONSchemaTests": 1, + "JSONRegexRuntimeTests": 1, + "JSONStreamingTests": 16 + }, + "coverage_unmapped": { + "Sources/JSON/Pointer/JSONLocationComponent.swift": "declaration-only enum", + "Sources/JSONSchema/Output/JSONValidationOutput.swift": "declaration-only enum", + "Sources/JSONSchema/Formats/JSONIDNAData.swift": "generated data, no executable mapping", + "Sources/JSONStreaming/JSONFraming.swift": "declaration-only enum", + "Sources/JSONStreaming/JSONTopLevelArrayEvent.swift": "declaration-only enum", + "Sources/JSONPatch/JSONPatchResult.swift": "declaration-only enum" + }, + "coverage_generated": [ + "Sources/JSONSchema/Formats/JSONIDNAData.swift", + "Sources/JSONRegexRuntime/RegexUnicodeTables.swift" + ] + }, + "coverage": { + "JSON": { + "lines": { + "count": 3443, + "covered": 3381, + "percent": 98.19924484461225 + }, + "regions": { + "count": 3427, + "covered": 3316, + "percent": 96.76101546542165 + }, + "functions": { + "count": 595, + "covered": 589, + "percent": 98.99159663865547 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "JSONSchema": { + "lines": { + "count": 2417, + "covered": 2389, + "percent": 98.84153909805543 + }, + "regions": { + "count": 3091, + "covered": 2954, + "percent": 95.56777741831122 + }, + "functions": { + "count": 384, + "covered": 375, + "percent": 97.65625 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "JSONRegexRuntime": { + "lines": { + "count": 659, + "covered": 650, + "percent": 98.63429438543247 + }, + "regions": { + "count": 695, + "covered": 668, + "percent": 96.11510791366906 + }, + "functions": { + "count": 88, + "covered": 86, + "percent": 97.72727272727273 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "JSONStreaming": { + "lines": { + "count": 232, + "covered": 228, + "percent": 98.27586206896552 + }, + "regions": { + "count": 269, + "covered": 256, + "percent": 95.16728624535315 + }, + "functions": { + "count": 60, + "covered": 58, + "percent": 96.66666666666667 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "JSONPatch": { + "lines": { + "count": 185, + "covered": 184, + "percent": 99.45945945945945 + }, + "regions": { + "count": 212, + "covered": 210, + "percent": 99.05660377358491 + }, + "functions": { + "count": 16, + "covered": 16, + "percent": 100.0 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + "native_run": "https://github.com/ThorFuchs/Swift-JSON/actions/runs/38572760920", + "linux_run": "https://github.com/ThorFuchs/Swift-JSON/actions/runs/38572755403", + "wasi_commands": 38, + "wasi_scope": { + "Node": "full roots debug/release 298 functions, 173 parameterized invocations; QA 125 each configuration", + "WasmKit": "QA 125 each configuration; no full independent/cap/consumer refresh; no sanitizer claim" + }, + "receipts": [ + { + "receipt": "NativeClosure/native-safety/ci/safety/receipt.json", + "sha256": "0fc375de64c602eeaad2526f69b3241e9b530d248a081bdad1bced72b8222bd6", + "lane": "safety", + "status": "passed", + "run_id": 38572760920, + "source_sha": "9184481e30e4f7faf262f335f775f8732c634040", + "git_commit_field": "9184481e30e4f7faf262f335f775f8732c634040", + "source_inputs_verified": 849, + "logs": [ + { + "log": "NativeClosure/native-safety/ci/safety/command-00.log", + "sha256": "f5270941d7a738f1621cf4f629651b7e58b652a625d86d1cd6b1188fc8e3adf4", + "exit_code": 0, + "status": "passed", + "function_counts": [ + 175, + 30, + 61, + 16, + 16 + ], + "parameterized_invocations": 173 + }, + { + "log": "NativeClosure/native-safety/ci/safety/command-01.log", + "sha256": "3ab955904dad8d3626c2377aca7942fb57dac412b9a3bbe8647c431693402748", + "exit_code": 0, + "status": "passed", + "function_counts": [ + 4, + 16, + 1, + 1 + ], + "parameterized_invocations": 0 + }, + { + "log": "NativeClosure/native-safety/ci/safety/command-02.log", + "sha256": "55f9f31ef3525e25388f783952c30b50668a03c5afc5fc15a4d9874243c5f3a1", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + } + ] + }, + { + "receipt": "NativeClosure/native-docs/ci/docs/receipt.json", + "sha256": "2d9ec374cb6c775efbe97b8de4ed1c483c78685291af98543813f4d83b35f16f", + "lane": "docs", + "status": "passed", + "run_id": 38572760920, + "source_sha": "9184481e30e4f7faf262f335f775f8732c634040", + "git_commit_field": "9184481e30e4f7faf262f335f775f8732c634040", + "source_inputs_verified": 849, + "logs": [ + { + "log": "NativeClosure/native-docs/ci/docs/command-00.log", + "sha256": "337649f4007cf99c7e5c0a0d5717d78c4bc09e5648f74a9c7de431a6bc9984f0", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + } + ] + }, + { + "receipt": "NativeClosure/native-native/ci/native/receipt.json", + "sha256": "4f8f60d0e8ae7284517157aafaf640b1803bdddcef653a443db7184ab4b71e47", + "lane": "native", + "status": "passed", + "run_id": 38572760920, + "source_sha": "9184481e30e4f7faf262f335f775f8732c634040", + "git_commit_field": "9184481e30e4f7faf262f335f775f8732c634040", + "source_inputs_verified": 849, + "logs": [ + { + "log": "NativeClosure/native-native/ci/native/command-00.log", + "sha256": "251f744310f08a1ab78099cd98b604effb6aceb5fafc473c4b6533fa93cc5819", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "NativeClosure/native-native/ci/native/command-01.log", + "sha256": "9e58bd4efc46eb0dd551d9b374db5c6cb8c14cc88ed7bfa24478fbfe9efa4da6", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "NativeClosure/native-native/ci/native/command-02.log", + "sha256": "b72c0f1f6043e4de041fec96a5e9f780656e41827002ed9a1718e349c992492c", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "NativeClosure/native-native/ci/native/command-03.log", + "sha256": "e930f0d635e4616ee257bdcd0139d1f0c2f06080ea7c5765a90c247c92bf9794", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "NativeClosure/native-native/ci/native/command-04.log", + "sha256": "f2f7bff67651eed40b52dca0a00d1d7bac395c6f7dbf6a0e69dd8f36ee5ddbfe", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "NativeClosure/native-native/ci/native/command-05.log", + "sha256": "b0ffd27ae64bc25cd47715dc4619ce1922255918875f4a2236032520fb02088f", + "exit_code": 0, + "status": "passed", + "function_counts": [ + 175, + 30, + 61, + 16, + 16 + ], + "parameterized_invocations": 173 + }, + { + "log": "NativeClosure/native-native/ci/native/command-06.log", + "sha256": "5ef0b1a1539d883cdc350a525b0832e3ba70e36947e95c42bad479ce7d94e039", + "exit_code": 0, + "status": "passed", + "function_counts": [ + 175, + 30, + 61, + 16, + 16 + ], + "parameterized_invocations": 173 + }, + { + "log": "NativeClosure/native-native/ci/native/command-07.log", + "sha256": "56bd52bdd6eec22ddafa5b309538d8b03cbf76bcd12f73d392bcb271722bfc17", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "NativeClosure/native-native/ci/native/command-08.log", + "sha256": "a1cf992dbb5c583900cc8f7e6a3abe3978659842acd51c6b50b2984bb4e983cc", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "NativeClosure/native-native/ci/native/command-09.log", + "sha256": "abdb994c10d1bf2f31217a7d55cf35f6d3d7bb8497af1f832c78cee76a743e09", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "NativeClosure/native-native/ci/native/command-10.log", + "sha256": "39c0297006b694e1b63a8199fa9ff44b983af6466d2cc5e80f98b48c26466cbc", + "exit_code": 0, + "status": "passed", + "function_counts": [ + 3 + ], + "parameterized_invocations": 0 + }, + { + "log": "NativeClosure/native-native/ci/native/command-11.log", + "sha256": "cd6ec9a1a581df413ab37e6aca6aa7c21690891d138b8c40c1b311465fbe392e", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "NativeClosure/native-native/ci/native/command-12.log", + "sha256": "c24200ac2f8e8ae8ab608a22581102c506ec5f2715cd553d54d02fe80205df85", + "exit_code": 0, + "status": "passed", + "function_counts": [ + 3 + ], + "parameterized_invocations": 0 + }, + { + "log": "NativeClosure/native-native/ci/native/command-13.log", + "sha256": "457879e410b8f4b90843a7703a5dec786666174a3cc1a59b1374faca61454b8e", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "NativeClosure/native-native/ci/native/command-14.log", + "sha256": "10c3242a4ea259c62391305d7760d3895fd9506d980a9962c56c56ca981a1514", + "exit_code": 0, + "status": "passed", + "function_counts": [ + 1 + ], + "parameterized_invocations": 0 + }, + { + "log": "NativeClosure/native-native/ci/native/command-15.log", + "sha256": "8ce77664334abc787dcd514057378614b5a10cce619bfca463dc6357929a41e4", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + } + ] + }, + { + "receipt": "LinuxClosure/linux-x86_64-linux/linux/receipt.json", + "sha256": "5e603ea28bccb65f4ec760c7855d506752cbbf4885f905d0e4ab578d45299be5", + "lane": "linux", + "status": "passed", + "run_id": 38572755403, + "source_sha": "9184481e30e4f7faf262f335f775f8732c634040", + "git_commit_field": null, + "source_inputs_verified": 849, + "logs": [ + { + "log": "LinuxClosure/linux-x86_64-linux/linux/command-00.log", + "sha256": "c6306a13a4dde0b9e0ef638e1bf83c2cb3f5b8079eb09001c3d235f74cd058e4", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-x86_64-linux/linux/command-01.log", + "sha256": "a8b5499bbea4b95fa412bbaa0de8738297cb53bd48263b1c3c62702c72a59dd2", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-x86_64-linux/linux/command-02.log", + "sha256": "b72c0f1f6043e4de041fec96a5e9f780656e41827002ed9a1718e349c992492c", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-x86_64-linux/linux/command-03.log", + "sha256": "e930f0d635e4616ee257bdcd0139d1f0c2f06080ea7c5765a90c247c92bf9794", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-x86_64-linux/linux/command-04.log", + "sha256": "c655769ce723aff613887e5be4dd07be2f1470a271e35881fd69a13c48ac68b0", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-x86_64-linux/linux/command-05.log", + "sha256": "00039fd6eeb02f8d766e4798ca417c4755d2505d7eb97557d290e61ab5b30577", + "exit_code": 0, + "status": "passed", + "function_counts": [ + 175, + 30, + 61, + 16, + 16 + ], + "parameterized_invocations": 173 + }, + { + "log": "LinuxClosure/linux-x86_64-linux/linux/command-06.log", + "sha256": "b2a0204685183902cb1503b0ccdb02aa54b8040ef318a16472e15f8a9b505678", + "exit_code": 0, + "status": "passed", + "function_counts": [ + 175, + 30, + 61, + 16, + 16 + ], + "parameterized_invocations": 173 + }, + { + "log": "LinuxClosure/linux-x86_64-linux/linux/command-07.log", + "sha256": "f52e298d3caf54402c275a47830bf72811c1c67d0afc0a25e95cb2ad6fa7880a", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-x86_64-linux/linux/command-08.log", + "sha256": "00f852637ccbd91b729bc2e9b2cc7e2e7d8a4fecf1724877a6f460ffa03b4e6b", + "exit_code": 0, + "status": "passed", + "function_counts": [ + 3 + ], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-x86_64-linux/linux/command-09.log", + "sha256": "00d8b106fae4950bf8c4afb8d007f89f8588d997ac85d6db36dfb772a03a87ba", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-x86_64-linux/linux/command-10.log", + "sha256": "72337f0220e371461ae1eccc1938402e9a088eaf9681dc74a5f7cb7452608e52", + "exit_code": 0, + "status": "passed", + "function_counts": [ + 1 + ], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-x86_64-linux/linux/command-11.log", + "sha256": "cf4ef20402f54d9a51dbf8a0abf4cf64988271a24055de45a75877d61d293dc2", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-x86_64-linux/linux/command-12.log", + "sha256": "361f0d9069dbc0ce43713ce6f83a5e366a9e1f1bd0f3f1509bf2c91d567c0165", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + } + ] + }, + { + "receipt": "LinuxClosure/linux-x86_64-linux-safety/linux-safety/receipt.json", + "sha256": "2ac13de0afa61f9d27bb1d795c0fd85e53809c7559e740bac0cf54d44433dc76", + "lane": "linux-safety", + "status": "passed", + "run_id": 38572755403, + "source_sha": "9184481e30e4f7faf262f335f775f8732c634040", + "git_commit_field": null, + "source_inputs_verified": 849, + "logs": [ + { + "log": "LinuxClosure/linux-x86_64-linux-safety/linux-safety/command-00.log", + "sha256": "f70d757ff636b6c4d53ff9fd1f969b37fb63e5dd869803aa79f4b001a2f4a2a6", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-x86_64-linux-safety/linux-safety/command-01.log", + "sha256": "c358c020e06099ed901ccc2a105bb79038fbefd3dcf87aa187e8ff482d314cfb", + "exit_code": 0, + "status": "passed", + "function_counts": [ + 175, + 30, + 61, + 16, + 16 + ], + "parameterized_invocations": 173 + }, + { + "log": "LinuxClosure/linux-x86_64-linux-safety/linux-safety/command-02.log", + "sha256": "9b28e5679f08a0d6010008876aebf480fbe3a8322d04f477da15d6929bb798c3", + "exit_code": 0, + "status": "passed", + "function_counts": [ + 4, + 16, + 1, + 1 + ], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-x86_64-linux-safety/linux-safety/command-03.log", + "sha256": "896daaad3a2800e2f29bb03ab3d9390211fd122c72aafbc8356729c6069a96c6", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + } + ] + }, + { + "receipt": "LinuxClosure/linux-arm64-linux-coverage/linux-coverage/receipt.json", + "sha256": "7c6c4b8af66c6c4f3e44c7e30f54741e3e30dd223f604a2959b0c2ba5b18e97e", + "lane": "linux-coverage", + "status": "passed", + "run_id": 38572755403, + "source_sha": "9184481e30e4f7faf262f335f775f8732c634040", + "git_commit_field": null, + "source_inputs_verified": 849, + "logs": [ + { + "log": "LinuxClosure/linux-arm64-linux-coverage/linux-coverage/command-00.log", + "sha256": "a3321c34e484b9eaabe711057619cc12495f10ae4bbeb5fc8fb244cae6f7180b", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-arm64-linux-coverage/linux-coverage/command-01.log", + "sha256": "e3e094a6bcb9a5d9553e3fec03764d44a38b02f0a090ddccb74dad5bccc0197e", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + } + ] + }, + { + "receipt": "LinuxClosure/linux-arm64-linux-safety/linux-safety/receipt.json", + "sha256": "a6d1f5ce7fc0cd6c99d814ca3789a2169e0219bd1465cc3b3d3f0e707e6a2da0", + "lane": "linux-safety", + "status": "passed", + "run_id": 38572755403, + "source_sha": "9184481e30e4f7faf262f335f775f8732c634040", + "git_commit_field": null, + "source_inputs_verified": 849, + "logs": [ + { + "log": "LinuxClosure/linux-arm64-linux-safety/linux-safety/command-00.log", + "sha256": "80dd8017ae34fd9d4b24df469013075ebb834a81037779f816dc038ca1de3ed0", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-arm64-linux-safety/linux-safety/command-01.log", + "sha256": "2a826fbd2526e689c78905d004560d0ea9b46e6bf51ccbcd71e21617988137d4", + "exit_code": 0, + "status": "passed", + "function_counts": [ + 175, + 30, + 61, + 16, + 16 + ], + "parameterized_invocations": 173 + }, + { + "log": "LinuxClosure/linux-arm64-linux-safety/linux-safety/command-02.log", + "sha256": "5d3840afa4f0fa7126412deecdadbae3f9038f6788b8171777f94336c329b303", + "exit_code": 0, + "status": "passed", + "function_counts": [ + 4, + 16, + 1, + 1 + ], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-arm64-linux-safety/linux-safety/command-03.log", + "sha256": "7ac24e7b875fe6b04feb3a2b6e1458e3c8cf1739ebc0647811cc213ce995e15f", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + } + ] + }, + { + "receipt": "LinuxClosure/linux-arm64-linux/linux/receipt.json", + "sha256": "145293e279c2aed7bf463b9517f5577ec51063dad5871c17f334ad1f111082d8", + "lane": "linux", + "status": "passed", + "run_id": 38572755403, + "source_sha": "9184481e30e4f7faf262f335f775f8732c634040", + "git_commit_field": null, + "source_inputs_verified": 849, + "logs": [ + { + "log": "LinuxClosure/linux-arm64-linux/linux/command-00.log", + "sha256": "c6306a13a4dde0b9e0ef638e1bf83c2cb3f5b8079eb09001c3d235f74cd058e4", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-arm64-linux/linux/command-01.log", + "sha256": "a3321c34e484b9eaabe711057619cc12495f10ae4bbeb5fc8fb244cae6f7180b", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-arm64-linux/linux/command-02.log", + "sha256": "b72c0f1f6043e4de041fec96a5e9f780656e41827002ed9a1718e349c992492c", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-arm64-linux/linux/command-03.log", + "sha256": "e930f0d635e4616ee257bdcd0139d1f0c2f06080ea7c5765a90c247c92bf9794", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-arm64-linux/linux/command-04.log", + "sha256": "22953251d63c65076c2a95677e5815a622f024a8af8d271ed6cde733aa774104", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-arm64-linux/linux/command-05.log", + "sha256": "faed345c850bb5321151b9565e6281fa7f3f4f56446d5fed42bd888faa5c03ba", + "exit_code": 0, + "status": "passed", + "function_counts": [ + 175, + 30, + 61, + 16, + 16 + ], + "parameterized_invocations": 173 + }, + { + "log": "LinuxClosure/linux-arm64-linux/linux/command-06.log", + "sha256": "38209d6cb640bc670fb6144a12535890a8855627d0d7a02aa57c8a45b03eaf7d", + "exit_code": 0, + "status": "passed", + "function_counts": [ + 175, + 30, + 61, + 16, + 16 + ], + "parameterized_invocations": 173 + }, + { + "log": "LinuxClosure/linux-arm64-linux/linux/command-07.log", + "sha256": "c5cb8b2b86686e4feb2704c80fae7928231710a92a99e7a54a311345d7436a8f", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-arm64-linux/linux/command-08.log", + "sha256": "4d6340863817107eba015020890731aff5c34c6fad7cff599b4809c5c7a2883d", + "exit_code": 0, + "status": "passed", + "function_counts": [ + 3 + ], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-arm64-linux/linux/command-09.log", + "sha256": "ee00f896d17bb311006c406e8174972f258f2a2fcfe16cfed958dbe97497a522", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-arm64-linux/linux/command-10.log", + "sha256": "89193960f96b94f5856825c5fb175bee7a06577d7e523a424582e72377b3430b", + "exit_code": 0, + "status": "passed", + "function_counts": [ + 1 + ], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-arm64-linux/linux/command-11.log", + "sha256": "72f9b4854185b61f4a4dde24dc4a5abfe997564ee5e62ddd167201f24427a536", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-arm64-linux/linux/command-12.log", + "sha256": "339b7eb803f83f8edc22f6d685727b600ab0d1937e819f143bff738be140647d", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + } + ] + }, + { + "receipt": "LinuxClosure/linux-x86_64-linux-coverage/linux-coverage/receipt.json", + "sha256": "123fa95cc19cbde5fc61c2111b7aa6e88c815d114609a8de311a29d45d013a0e", + "lane": "linux-coverage", + "status": "passed", + "run_id": 38572755403, + "source_sha": "9184481e30e4f7faf262f335f775f8732c634040", + "git_commit_field": null, + "source_inputs_verified": 849, + "logs": [ + { + "log": "LinuxClosure/linux-x86_64-linux-coverage/linux-coverage/command-00.log", + "sha256": "a253326fe856602c0bcde1236fe41bfe3c0720ca90cf17ce623e7b0b7744545b", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + }, + { + "log": "LinuxClosure/linux-x86_64-linux-coverage/linux-coverage/command-01.log", + "sha256": "e3e094a6bcb9a5d9553e3fec03764d44a38b02f0a090ddccb74dad5bccc0197e", + "exit_code": 0, + "status": "passed", + "function_counts": [], + "parameterized_invocations": 0 + } + ] + }, + { + "receipt": "closure-native-qa/receipt.json", + "sha256": "b72a7bee47a9b488a3436eb73f247a43c7a090cf26b38089eb28ca43838d5414", + "lane": "qa", + "status": "passed", + "source_sha": "9184481e30e4f7faf262f335f775f8732c634040", + "commands": 3, + "command_exits": [ + 0, + 0, + 0 + ] + }, + { + "receipt": "closure-native-coverage/receipt.json", + "sha256": "8080c0e5a1ac96ab2dfda86c1eae50c325e51ecf5db389248d54d1eccef7e6a7", + "lane": "coverage", + "status": "passed", + "source_sha": "9184481e30e4f7faf262f335f775f8732c634040", + "commands": 2, + "command_exits": [ + 0, + 0 + ] + }, + { + "receipt": "closure-wasi/receipt.json", + "sha256": "1f80f59524bf7ceec763ba0efbab8c5719ac562b1d65383c56cb2695d8fb5559", + "lane": "wasm-qa", + "status": "passed", + "source_sha": "9184481e30e4f7faf262f335f775f8732c634040", + "commands": 2, + "command_exits": [ + 0, + 0 + ] + } + ], + "evidence_paths_base": "Evidence/CoreClosure in the delivered local companion bundle; build artifacts are not committed to Git", + "independent_policy_counts": { + "syntax": { + "observed": 318, + "evaluated_pass": 287, + "policy": 31, + "evaluated_fail": 0 + }, + "schema": { + "observed": 2337, + "evaluated_pass": 2300, + "policy": 37, + "evaluated_fail": 0 + } + }, + "qualification_counts": { + "public_semantics": 82, + "budget_counter": 9, + "malformed_replay": 4396, + "seed": "0x53574946544a534f", + "metric_probes": 49, + "rooted_consumer_valid": 8, + "rooted_consumer_invalid": 12, + "rooted_consumer_operational": 2 + }, + "performance": { + "status": "blocked", + "readiness_exit": 2, + "original_long_change_percent": 11.98325099056705, + "review_trigger_percent": 10, + "disposition": "Correctness repair retained; timing attribution unresolved; no new comparison ran because quiet-host criteria failed. Historical streaming disposition unchanged. Logical retained bytes and owned work are not RSS/caller instruction caps." + }, + "remaining": [ + "Quiet native-host diagnostic recalibration before the 1.2.1 release decision", + "Broader E2E/mutation acceptance plan", + "Unused private helpers and guarded/specialized mapping regions retained; no 100% decision-coverage claim", + "Existing Apple device scopes and WASI sanitizer limitations unchanged" + ], + "evidence_index": [ + "CoreResidualReview.json", + "RuntimeEquivalenceFinal.json", + "CoverageAgreement.json", + "WASIMaterialization.json", + "Calibration/Recalibration.json", + "Calibration/HostBefore.json", + "NativeClosure-verified.json", + "LinuxClosure-verified.json" + ], + "preserved_prior_bundles": { + "Swift-JSON-1.2.1-Portability-Evidence.zip": "41b2446430cc891627cc0a77d53d335eb6df01c410a9336af320a9c927d04a9c", + "Swift-JSON-1.2.1-Coverage-Evidence.zip": "baa386331c89d2bcdd7036f72916e72fdb2256a9054d0991b1a00bd8ce409b75" + } +} diff --git a/Documentation/Qualification/Release1.2.1/README.md b/Documentation/Qualification/Release1.2.1/README.md index 2c5629e..c531c44 100644 --- a/Documentation/Qualification/Release1.2.1/README.md +++ b/Documentation/Qualification/Release1.2.1/README.md @@ -43,3 +43,7 @@ the structural completion gap is reviewed with its denominator retained. The [Linux coverage and behavior-gap wave](CoverageWave.md) extends tests across all five modules and repairs best-effort diagnostic sizing. Its new native/Linux qualification is recorded separately from the earlier unchanged-source receipts. + +The [reachable core follow-up](CoreClosure.md) adds 22 public behavior functions +and extends escaped schema-output/presence assertions. Unadjusted coverage meets +the five-module review targets; quiet-host diagnostic recalibration stays blocked. From 85a7f4f191053bc25c7b0020e0a58fb92f2caf0a Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Wed, 14 Oct 2026 20:45:07 +0200 Subject: [PATCH 18/24] test: add public end-to-end acceptance and bounded mutation pilot --- .../Release1.2.1/AcceptanceReplays.json | 71 ++++++ .../Release1.2.1/MutationPilot.json | 198 ++++++++++++++++ .../Qualification/TestDiscovery.json | 16 +- Package.swift | 1 + Scripts/ci.py | 5 +- Scripts/coverage.py | 7 +- Scripts/mutation_acceptance.py | 71 ++++++ Scripts/qualify_distribution.py | 46 ++++ Scripts/test_qualification_harness.py | 11 +- .../PublicWorkflowTests.swift | 221 ++++++++++++++++++ .../SeededReplayTests.swift | 71 ++++++ .../StagedImportTests.swift | 121 ++++++++++ Tools/DistributionAcceptance/Package.swift | 12 + .../Sources/DistributionAcceptance/main.swift | 26 +++ 14 files changed, 867 insertions(+), 10 deletions(-) create mode 100644 Documentation/Qualification/Release1.2.1/AcceptanceReplays.json create mode 100644 Documentation/Qualification/Release1.2.1/MutationPilot.json create mode 100644 Scripts/mutation_acceptance.py create mode 100644 Scripts/qualify_distribution.py create mode 100644 Tests/JSONIntegrationTests/PublicWorkflowTests.swift create mode 100644 Tests/JSONIntegrationTests/SeededReplayTests.swift create mode 100644 Tests/JSONIntegrationTests/StagedImportTests.swift create mode 100644 Tools/DistributionAcceptance/Package.swift create mode 100644 Tools/DistributionAcceptance/Sources/DistributionAcceptance/main.swift diff --git a/Documentation/Qualification/Release1.2.1/AcceptanceReplays.json b/Documentation/Qualification/Release1.2.1/AcceptanceReplays.json new file mode 100644 index 0000000..6ecb209 --- /dev/null +++ b/Documentation/Qualification/Release1.2.1/AcceptanceReplays.json @@ -0,0 +1,71 @@ +{ + "schema_version": 1, + "seed": "0x53574a313231", + "valid_generated_models": 128, + "invalid_generated_leading_zero_mutations": 128, + "minimized_replays": [ + { + "id": "trailing-comma", + "hex": "5b302c5d", + "sha256": "a3e9007c4356922ae64718791f03bb64b52d2c7b759d42f33a24669005e75688", + "category": "syntax", + "code": "invalidSyntax" + }, + { + "id": "leading-zero", + "hex": "5b30315d", + "sha256": "8e955b12c5bd485a2f5ad9ec07f09bf0cd19d368b6fed043e8049f6dc17cc899", + "category": "syntax", + "code": "invalidNumber" + }, + { + "id": "comment", + "hex": "5b302f2a782a2f5d", + "sha256": "9f0d758dd43c10cce32f4e8398de5ef438ec2250bb56615133b3110fcbe4b108", + "category": "syntax", + "code": "invalidSyntax" + }, + { + "id": "raw-control", + "hex": "220a22", + "sha256": "71f6bb6ee37cb1df013db8d8a675a947b8b5253d571b6533a30671ddd37243fc", + "category": "syntax", + "code": "invalidSyntax" + }, + { + "id": "invalid-utf8", + "hex": "22c0af22", + "sha256": "bfc25cfd85c49686ec04294e6de47c5dd57651e75084d1c08d3d124cd1dd471d", + "category": "syntax", + "code": "invalidUTF8" + }, + { + "id": "decoded-duplicate", + "hex": "7b2261223a302c225c7530303631223a317d", + "sha256": "961266c2e42a000802c6eb1d325e7455ece518e0228ced748990ea6d586b7277", + "category": "syntax", + "code": "duplicateKey" + }, + { + "id": "unpaired-surrogate", + "hex": "225c754438303022", + "sha256": "e2abb8b90faadf0e98d5e6382fa5f354e814aaa72ce1155de80231de88c7724d", + "category": "syntax", + "code": "invalidSyntax" + }, + { + "id": "truncated-escape", + "hex": "225c753030", + "sha256": "cbc010641b736066c61900cfd7c61f6213341f567864b7c6f46ad832ec484144", + "category": "syntax", + "code": "unexpectedEOF" + }, + { + "id": "second-root", + "hex": "302031", + "sha256": "5cc3a6551605a0b4e9c3334f5eb5554c404973daf0b1a58655fa29c0ba3d47b0", + "category": "syntax", + "code": "trailingData" + } + ] +} diff --git a/Documentation/Qualification/Release1.2.1/MutationPilot.json b/Documentation/Qualification/Release1.2.1/MutationPilot.json new file mode 100644 index 0000000..a7f5316 --- /dev/null +++ b/Documentation/Qualification/Release1.2.1/MutationPilot.json @@ -0,0 +1,198 @@ +{ + "schema_version": 1, + "scope": "Single-site assertion-quality pilot; no mutant is shipped", + "mutants": [ + { + "id": "M01", + "path": "Sources/JSON/Limits/JSONWorkMeter.swift", + "before": "guard attempted <= limits[metric] else", + "after": "guard attempted < limits[metric] else", + "family": "inclusive-cap", + "filter": "qaE2E" + }, + { + "id": "M02", + "path": "Sources/JSON/Limits/JSONWorkMeter.swift", + "before": "try Task.checkCancellation()\n } catch", + "after": "/* dropped checkpoint */\n } catch", + "family": "cancellation", + "filter": "qaE2E" + }, + { + "id": "M03", + "path": "Sources/JSON/Value/JSONComparison.swift", + "before": "var l = lhs.utf8.makeIterator(), r = rhs.utf8.makeIterator()", + "after": "if lhs == rhs { return true }; var l = lhs.utf8.makeIterator(), r = rhs.utf8.makeIterator()", + "family": "canonical-key-collapse", + "filter": "qaE2E" + }, + { + "id": "M04", + "path": "Sources/JSON/Codec/JSONTokenMachine.swift", + "before": "frame.expectation == .arrayFirstValue || frame.expectation == .arrayEnd", + "after": "frame.expectation == .arrayFirstValue || frame.expectation == .arrayEnd || frame.expectation == .arrayValue", + "family": "grammar", + "filter": "qaE2E" + }, + { + "id": "M05", + "path": "Sources/JSON/Codec/JSONLexer.swift", + "before": "guard byte >= 32 else", + "after": "guard byte >= 10 else", + "family": "grammar", + "filter": "qaE2E" + }, + { + "id": "M06", + "path": "Sources/JSON/Codec/JSONTokenMachine.swift", + "before": "guard !rootStarted else { throw failure(token, .trailingData) }", + "after": "guard !rootStarted || frames.isEmpty else { throw failure(token, .trailingData) }", + "family": "trailing-root", + "filter": "qaE2E" + }, + { + "id": "M07", + "path": "Sources/JSON/Coding/JSONCodablePayload.swift", + "before": "version == 1 else", + "after": "version <= 2 else", + "family": "version-envelope", + "filter": "qaE2E" + }, + { + "id": "M08", + "path": "Sources/JSON/Codec/JSONWriter.swift", + "before": "if options.numberStyle == .preserved, let lexeme", + "after": "if false, let lexeme", + "family": "exact-lexeme", + "filter": "qaE2E" + }, + { + "id": "M09", + "path": "Sources/JSONPatch/JSONPatch.swift", + "before": "guard try JSONComparison.equals(selected, expected, meter: &meter) else", + "after": "guard try JSONComparison.equals(selected, selected, meter: &meter) else", + "family": "staged-publication", + "filter": "qaE2E" + }, + { + "id": "M10", + "path": "Sources/JSONPatch/JSONPatch.swift", + "before": "staged.map { .document($0) } ?? .removed", + "after": "staged.map { .document($0) } ?? .document(.null)", + "family": "root-absence", + "filter": "qaE2E" + }, + { + "id": "M11", + "path": "Sources/JSONPatch/JSONMergePatch.swift", + "before": "if existing != nil {", + "after": "if existing == nil {", + "family": "merge-removal", + "filter": "qaE2E" + }, + { + "id": "M12", + "path": "Sources/JSONStreaming/JSONTopLevelArraySequence.swift", + "before": "closed = true; continue", + "after": "closed = true; terminal = true; return .documentCompleted(.init(records: input.meter.used(.totalRecords), statistics: input.meter.statistics()))", + "family": "early-completion", + "filter": "qaE2E" + }, + { + "id": "M13", + "path": "Sources/JSONStreaming/JSONTopLevelArraySequence.swift", + "before": "try input.meter.charge(.totalRecords); assembler", + "after": "/* dropped record charge */ assembler", + "family": "dropped-charge", + "filter": "qaE2E" + }, + { + "id": "M14", + "path": "Sources/JSONStreaming/JSONTopLevelArraySequence.swift", + "before": "catch { terminal = true; input.failed = true; assembler = JSONValueAssembler(); throw error }", + "after": "catch { terminal = true; input.failed = true; assembler = JSONValueAssembler(); return nil }", + "family": "swallowed-operational-error", + "filter": "qaE2E" + }, + { + "id": "M15", + "path": "Sources/JSONStreaming/NDJSON/NDJSONValueSequence.swift", + "before": "try input.meter.charge(.totalRecords)", + "after": "/* dropped record charge */", + "family": "dropped-charge", + "filter": "qaE2E" + }, + { + "id": "M16", + "path": "Sources/JSONStreaming/NDJSON/NDJSONValueSequence.swift", + "before": "catch { terminal = true; input.failed = true; throw error }", + "after": "catch { terminal = true; input.failed = true; return nil }", + "family": "swallowed-operational-error", + "filter": "qaE2E" + }, + { + "id": "M17", + "path": "Sources/JSONStreaming/JSONAsyncWriter.swift", + "before": "try meter.checkpoint(); try await sink(JSONByteChunk(bytes)); try meter.checkpoint()\n try meter.release", + "after": "try meter.checkpoint(); try await sink(JSONByteChunk(bytes)); /* dropped after-await checkpoint */\n try meter.release", + "family": "await-cancellation", + "filter": "qaE2E" + }, + { + "id": "M18", + "path": "Sources/JSONStreaming/JSONAsyncWriter.swift", + "before": "try meter.charge(.outputBytes); try meter.retain(1)", + "after": "/* dropped output charge */ try meter.retain(1)", + "family": "dropped-charge", + "filter": "qaE2E" + }, + { + "id": "M19", + "path": "Sources/JSONSchema/Evaluation/JSONSchemaAssertions.swift", + "before": "case .minimum: return comparison != .orderedAscending", + "after": "case .minimum: return comparison == .orderedDescending", + "family": "inclusive-numeric-bound", + "filter": "qaE2E" + }, + { + "id": "M20", + "path": "Sources/JSONSchema/Evaluation/JSONSchemaEvaluation.swift", + "before": "if child.valid { try merge(&child,into: &frame.branch,evaluated: true,meter: &meter) }", + "after": "if true { try merge(&child,into: &frame.branch,evaluated: true,meter: &meter) }", + "family": "failed-branch-annotation-merge", + "filter": "qaE2E" + }, + { + "id": "M21", + "path": "Sources/JSONSchema/Evaluation/JSONSchemaAssertions.swift", + "before": "case \"integer\": return instance.numberValue?.isInteger ?? false", + "after": "case \"integer\": return instance.numberValue != nil", + "family": "integer-kind", + "filter": "qaE2E" + }, + { + "id": "M22", + "path": "Sources/JSON/Reading/JSONReadCursor.swift", + "before": "if value.isNull { return .null }", + "after": "if value.isNull { return .absent }", + "family": "presence", + "filter": "qaE2E" + }, + { + "id": "M23", + "path": "Sources/JSONSchema/Document/JSONSchemaBuiltins.swift", + "before": "guard hash.utf8.elementsEqual(expectedHash.utf8) else", + "after": "guard true else", + "family": "builtin-integrity", + "filter": "builtinIntegrity" + }, + { + "id": "M24", + "path": "Sources/JSONSchema/Evaluation/JSONSchemaEvaluation.swift", + "before": "frame.outcome.valid = false\n if options.output", + "after": "frame.outcome.valid = true\n if options.output", + "family": "ordinary-invalidity", + "filter": "qaE2E" + } + ] +} diff --git a/Documentation/Qualification/TestDiscovery.json b/Documentation/Qualification/TestDiscovery.json index d1e8a60..7800fb0 100644 --- a/Documentation/Qualification/TestDiscovery.json +++ b/Documentation/Qualification/TestDiscovery.json @@ -5,7 +5,8 @@ "JSONSchemaTests": 61, "JSONRegexRuntimeTests": 16, "JSONStreamingTests": 30, - "JSONPatchTests": 16 + "JSONPatchTests": 16, + "JSONIntegrationTests": 20 }, "parameterized_invocations": 173, "qa_functions": { @@ -13,13 +14,15 @@ "JSONSchemaTests": 13, "JSONStreamingTests": 19, "JSONPatchTests": 7, - "JSONRegexRuntimeTests": 4 + "JSONRegexRuntimeTests": 4, + "JSONIntegrationTests": 20 }, "shared_state_functions": { "JSONTests": 4, "JSONSchemaTests": 1, "JSONRegexRuntimeTests": 1, - "JSONStreamingTests": 16 + "JSONStreamingTests": 16, + "JSONIntegrationTests": 1 }, "coverage_unmapped": { "Sources/JSON/Pointer/JSONLocationComponent.swift": "declaration-only enum", @@ -32,5 +35,12 @@ "coverage_generated": [ "Sources/JSONSchema/Formats/JSONIDNAData.swift", "Sources/JSONRegexRuntime/RegexUnicodeTables.swift" + ], + "coverage_modules": [ + "JSON", + "JSONSchema", + "JSONRegexRuntime", + "JSONStreaming", + "JSONPatch" ] } diff --git a/Package.swift b/Package.swift index c372695..3074fda 100644 --- a/Package.swift +++ b/Package.swift @@ -17,6 +17,7 @@ let package = Package( .target(name: "JSONSchema", dependencies: ["JSON", "JSONRegexRuntime"], resources: [.copy("Resources/Draft2020-12")]), .target(name: "JSONStreaming", dependencies: ["JSON"]), .target(name: "JSONPatch", dependencies: ["JSON"]), + .testTarget(name: "JSONIntegrationTests", dependencies: ["JSON", "JSONSchema", "JSONStreaming", "JSONPatch"]), .testTarget(name: "JSONTests", dependencies: ["JSON"], exclude: ["ImplementationPlan.md"]), .testTarget(name: "JSONSchemaTests", dependencies: ["JSONSchema"], exclude: ["ImplementationPlan.md"]), .testTarget(name: "JSONStreamingTests", dependencies: ["JSONStreaming"], exclude: ["ImplementationPlan.md"]), diff --git a/Scripts/ci.py b/Scripts/ci.py index fe5ea1f..708b612 100644 --- a/Scripts/ci.py +++ b/Scripts/ci.py @@ -7,7 +7,7 @@ from test_discovery import DISCOVERY, ROOT_COUNTS, PARAMETER_INVOCATIONS ROOT = pathlib.Path(__file__).resolve().parents[1] -SHARED_TESTS = 'independentCodecFacadesAndExactValuesCanBeSharedAcrossTasks|concurrentSchemaEvaluationsHaveExclusiveDynamicRegexAndAnnotationState|sharedRegexProgramHasExclusiveConcurrentSearchState|immutableIndexedObjectCanBeSharedAcrossTasks|qaSharedPayloadAndRootHaveIndependentConcurrentReadScopes|sha256InputCapacityAndConcurrentInvocationsRemainIndependent|qaStreaming' +SHARED_TESTS = 'independentCodecFacadesAndExactValuesCanBeSharedAcrossTasks|concurrentSchemaEvaluationsHaveExclusiveDynamicRegexAndAnnotationState|sharedRegexProgramHasExclusiveConcurrentSearchState|immutableIndexedObjectCanBeSharedAcrossTasks|qaSharedPayloadAndRootHaveIndependentConcurrentReadScopes|sha256InputCapacityAndConcurrentInvocationsRemainIndependent|qaStreaming|qaE2ESharedPlan' R0_TESTS = 'readerProbe|indexProbes|duplicateRecognitionPrecedesValueAndClosingSyntax' def snapshot(): @@ -68,6 +68,7 @@ def commands(lane, output): ['python3', '-B', 'Scripts/qualify_release_1_1.py', '--mode', 'ownership', '--output', str(output / 'ownership.json')], ['python3', '-B', 'Scripts/qualify_release_1_1.py', '--mode', 'reader', '--output', str(output / 'reader.json')], ['swift', 'run', '-c', 'release', '--package-path', 'Examples/Consumer', 'Consumer'], + ['python3', '-B', 'Scripts/qualify_distribution.py', '--output', str(output / 'distribution')], ['swift', 'test', '--package-path', 'Tools/MobileQualification'], ['swift', 'run', '-c', 'release', '--package-path', 'Tools/MobileQualification', 'MobileQualification'], ['swift', 'test', '--package-path', 'Tools/MCPReadiness'], @@ -200,7 +201,7 @@ def save(): if (coverage['status'] != 'passed' or coverage['root_counts'] != ROOT_COUNTS or coverage['parameterized_invocations'] != PARAMETER_INVOCATIONS or coverage['omitted_binary_probe']['status'] != 'rejected' - or set(coverage['modules']) != {name.removesuffix('Tests') for name in ROOT_COUNTS}): + or set(coverage['modules']) != set(DISCOVERY['coverage_modules'])): raise ValueError('Missing complete five-module coverage boundary') receipt['materialized_coverage'] = {'path':'export/coverage.json', 'sha256':hashlib.sha256((output / 'export/coverage.json').read_bytes()).hexdigest()} except (OSError,KeyError,ValueError,TypeError) as error: diff --git a/Scripts/coverage.py b/Scripts/coverage.py index 983eecf..42a7395 100644 --- a/Scripts/coverage.py +++ b/Scripts/coverage.py @@ -33,7 +33,7 @@ def executable_objects(bin_directory): def shared_objects(bin_directory): """Swift Build's Linux runners load instrumented product/test DSOs.""" result = {} - targets = list(ROOT_COUNTS) + [name.removesuffix('Tests') for name in ROOT_COUNTS] + targets = list(ROOT_COUNTS) + DISCOVERY['coverage_modules'] for target in targets: found = [bin_directory / name for name in [target + '.so', 'lib' + target + '.so'] if (bin_directory / name).is_file()] @@ -89,8 +89,7 @@ def totals(names): return result modules = {} - for test_target in policy['root_functions']: - module = test_target.removesuffix('Tests') + for module in policy.get('coverage_modules', [name.removesuffix('Tests') for name in policy['root_functions']]): names = {name for name in mapped if name.startswith('Sources/' + module + '/')} if not names or not totals(names)['regions']['count']: raise ValueError('Missing executable module coverage: ' + module) @@ -157,7 +156,7 @@ def run(command, name, binary=False): raise ValueError('Merged coverage profile missing') shutil.copy2(profile, output / 'default.profdata') receipt['profile_sha256'] = digest(profile) - # Preserve the actual five objects, not just paths into a discarded CI build. + # Preserve the actual instrumented objects, not just paths into a discarded CI build. # Together with the merged profile these allow independent LLVM replay. binary_output = output / 'objects' binary_output.mkdir(exist_ok=True) diff --git a/Scripts/mutation_acceptance.py b/Scripts/mutation_acceptance.py new file mode 100644 index 0000000..4225910 --- /dev/null +++ b/Scripts/mutation_acceptance.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Single-site native mutants in an explicitly supplied disposable Git worktree.""" +import argparse, collections, hashlib, json, os, pathlib, re, signal, subprocess, time +ROOT = pathlib.Path(__file__).resolve().parents[1] + +def sha(path): return hashlib.sha256(path.read_bytes()).hexdigest() +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--worktree', required=True); parser.add_argument('--output', required=True) + parser.add_argument('--only', nargs='*'); args = parser.parse_args() + worker = pathlib.Path(args.worktree).resolve(); output = (ROOT/args.output).resolve() + # Never patch the control checkout or an unrelated checkout. The caller creates + # this detached disposable worktree; no reset/clean affects another session. + assert worker != ROOT and worker.is_relative_to(ROOT/'Artifacts'), 'Worktree must be task-owned and disposable' + def git(*parts): return subprocess.check_output(['git','-C',str(worker),*parts], text=True).strip() + assert git('rev-parse','--show-toplevel') == str(worker) + assert git('rev-parse','--abbrev-ref','HEAD') == 'HEAD' + assert not git('status','--porcelain'), 'Worker must initially be clean' + pin = git('rev-parse','HEAD'); output.mkdir(parents=True,exist_ok=True) + spec = ROOT/'Documentation/Qualification/Release1.2.1/MutationPilot.json' + selected = [m for m in json.loads(spec.read_text())['mutants'] if not args.only or m['id'] in args.only] + assert selected and len({m['id'] for m in selected}) == len(selected) + receipt = dict(schema_version=1, status='running', source=pin, specification_sha256=sha(spec), selected=len(selected), baselines=[], mutants=[]) + def save(): + receipt['counts'] = dict(collections.Counter(m['outcome'] for m in receipt['mutants'])) + (output/'mutation.json').write_text(json.dumps(receipt,indent=2)+'\n') + def run(name, test_filter): + command = ['swift','test','--disable-xctest','--filter',test_filter] + log = output/(name+'.log'); start = time.monotonic(); timed_out=False + with log.open('w') as stream: + process = subprocess.Popen(command,cwd=worker,stdout=stream,stderr=subprocess.STDOUT,start_new_session=True) + try: code=process.wait(timeout=1800) + except subprocess.TimeoutExpired: + timed_out=True; os.killpg(process.pid,signal.SIGTERM) + try: process.wait(timeout=10) + except subprocess.TimeoutExpired: os.killpg(process.pid,signal.SIGKILL);process.wait() + code=process.returncode + text=log.read_text() + return dict(command=command, exit_code=code, seconds=time.monotonic()-start, timed_out=timed_out, + log=log.name, sha256=sha(log), function_counts=[int(n) for n in re.findall(r'Test run with (\d+) tests? in[^\n]*passed',text)]),text + # Every selected filter must first demonstrate its exact nonzero baseline. + for test_filter in sorted({m['filter'] for m in selected}): + record,text=run('baseline-'+test_filter,test_filter) + expected=[20] if test_filter == 'qaE2E' else [1] + assert record['exit_code']==0 and record['function_counts']==expected, 'Baseline failed: '+test_filter + receipt['baselines'].append(record); save() + for mutant in selected: + relative=pathlib.PurePosixPath(mutant['path']) + assert not relative.is_absolute() and '..' not in relative.parts and relative.parts[0]=='Sources' + source=worker/relative; original=source.read_bytes(); text=original.decode() + assert text.count(mutant['before']) == 1, 'Mutant site changed: '+mutant['id'] + modified=text.replace(mutant['before'],mutant['after'],1).encode() + source.write_bytes(modified) + try: + record,text=run(mutant['id'],mutant['filter']) + if record['timed_out']: outcome='timed_out' + elif 'error: Build failed' in text or 'error: fatalError' in text: outcome='unbuildable' + elif record['exit_code'] != 0 and ('recorded an issue' in text or re.search(r'Test run with \d+ tests? in[^\n]*failed',text)): outcome='killed' + elif record['exit_code']==0 and record['function_counts']==([20] if mutant['filter']=='qaE2E' else [1]): outcome='survived' + else: outcome='execution_error' + record.update(id=mutant['id'],path=mutant['path'],family=mutant['family'],outcome=outcome, + original_sha256=hashlib.sha256(original).hexdigest(),mutated_sha256=hashlib.sha256(modified).hexdigest()) + receipt['mutants'].append(record) + print(mutant['id']+' '+outcome,flush=True) + finally: source.write_bytes(original) + assert not git('status','--porcelain'), 'Restoration failed' + save() + receipt.update(status='completed', restored=True);save() + +if __name__=='__main__': main() diff --git a/Scripts/qualify_distribution.py b/Scripts/qualify_distribution.py new file mode 100644 index 0000000..b1a795f --- /dev/null +++ b/Scripts/qualify_distribution.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Public consumer plus installed-resource failures; touch only owned build output.""" +import argparse, hashlib, json, pathlib, subprocess, time +ROOT = pathlib.Path(__file__).resolve().parents[1] + +def main(): + p = argparse.ArgumentParser(); p.add_argument('--output', required=True); a = p.parse_args() + output = ROOT / a.output; output.mkdir(parents=True, exist_ok=True) + receipt = dict(status='running', commands=[], resource_probes=3) + def run(name, command, expected=None): + start = time.monotonic(); log = output / (name+'.log') + with log.open('w') as stream: + result = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT, timeout=1800) + text = log.read_text() + receipt['commands'].append(dict(command=command, exit_code=result.returncode, seconds=time.monotonic()-start, + log=log.name, sha256=hashlib.sha256(log.read_bytes()).hexdigest())) + assert result.returncode == 0 and (expected is None or expected in text), name + return text.strip() + package = 'Tools/DistributionAcceptance' + run('build', ['swift','build','-c','release','--package-path',package]) + directory = pathlib.Path(run('bin-path', ['swift','build','-c','release','--package-path',package,'--show-bin-path'])) + executable = directory / 'DistributionAcceptance' + candidates = list(directory.glob('Swift-JSON_JSONSchema.*')) + bundles = [q for q in candidates if q.is_dir() and q.suffix in ('.bundle','.resources')] + assert len(bundles) == 1, 'Missing/ambiguous installed schema bundle' + bundle = bundles[0]; files = sorted(q for q in bundle.rglob('*') if q.is_file()) + resource = bundle / 'Draft2020-12/schema.json'; original = resource.read_bytes() + receipt['bundle'] = dict(layout=bundle.name, files=[dict(path=q.relative_to(bundle).as_posix(),sha256=hashlib.sha256(q.read_bytes()).hexdigest()) for q in files]) + receipt['executable_sha256'] = hashlib.sha256(executable.read_bytes()).hexdigest() + run('intact', [str(executable),'intact'], 'DISTRIBUTION intact nine_offline_resources') + hidden = bundle.with_name(bundle.name+'.held-by-acceptance') + assert not hidden.exists() + bundle.rename(hidden) + try: run('missing', [str(executable),'unavailable'], 'DISTRIBUTION unavailable structured_metaSchema') + finally: hidden.rename(bundle) + # Still valid JSON and the same byte count: catches missing integrity checks. + assert original.count(b'"title"') == 1 + resource.write_bytes(original.replace(b'"title"',b'"titlf"',1)) + try: run('corrupt', [str(executable),'unavailable'], 'DISTRIBUTION unavailable structured_metaSchema') + finally: resource.write_bytes(original) + assert resource.read_bytes() == original + receipt.update(status='passed', restored=True) + (output/'distribution.json').write_text(json.dumps(receipt,indent=2)+'\n') + +if __name__ == '__main__': main() diff --git a/Scripts/test_qualification_harness.py b/Scripts/test_qualification_harness.py index 88244f5..a80afff 100644 --- a/Scripts/test_qualification_harness.py +++ b/Scripts/test_qualification_harness.py @@ -32,6 +32,15 @@ def test_complete_denominator_and_unavailable_counters(self): self.assertFalse(result['branch_counters_available']) self.assertEqual(set(result['modules']), {'JSON','JSONPatch'}) + def test_integration_bundle_does_not_create_a_library_denominator(self): + self.policy['root_functions']['JSONIntegrationTests'] = 19 + self.policy['coverage_modules'] = ['JSON', 'JSONPatch'] + result = validate_export(self.export, self.root, self.policy) + self.assertEqual(set(result['modules']), {'JSON', 'JSONPatch'}) + self.export['data'][0]['files'].pop() + with self.assertRaisesRegex(ValueError, 'Incomplete library export'): + validate_export(self.export, self.root, self.policy) + def test_omitted_module_is_not_a_higher_percentage(self): self.export['data'][0]['files'].pop() with self.assertRaisesRegex(ValueError, 'Incomplete library export'): @@ -108,7 +117,7 @@ def test_linux_runner_and_ambiguous_layout(self): (self.root/(name+'-test-runner')).touch() self.assertEqual(set(executable_objects(self.root)),set(ROOT_COUNTS)) self.assertEqual(shared_objects(self.root), {}) # Static-linked backend remains valid. - targets = set(ROOT_COUNTS) | {name.removesuffix('Tests') for name in ROOT_COUNTS} + targets = set(ROOT_COUNTS) | set(DISCOVERY['coverage_modules']) for target in targets: (self.root/(target+'.so')).touch() self.assertEqual(set(shared_objects(self.root)), targets) (self.root/'libJSON.so').touch() diff --git a/Tests/JSONIntegrationTests/PublicWorkflowTests.swift b/Tests/JSONIntegrationTests/PublicWorkflowTests.swift new file mode 100644 index 0000000..ce30a6a --- /dev/null +++ b/Tests/JSONIntegrationTests/PublicWorkflowTests.swift @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: Apache-2.0 +import Testing +import Foundation +import JSON +import JSONSchema +import JSONPatch +import JSONStreaming + +// Test-application state is deliberately outside the library. No private imports. +private let input = #"{"id":9007199254740993,"amount":1.2300,"label":"entry","optional":null,"é":1,"é":2}"# +private let golden = #"{"id":9007199254740993,"amount":1.2300,"label":"updated","é":1,"é":2,"ready":true}"# +private struct Entry: Codable, Equatable, Sendable { let id: Int64; let label: String } +private func plan() throws -> CompiledJSONSchema { + let raw = try JSONParser().parse(#"{"type":"object","properties":{"id":{"type":"integer","minimum":9007199254740993},"label":{"type":"string","pattern":"^[a-z]+$"}},"required":["id","label"]}"#) + return try JSONSchemaCompiler().compile(JSONSchemaDocument(raw)) +} +private func edited(_ value: JSON) throws -> JSON { + let patch = try JSONPatch(operations: [.replace(path: JSONPointer(parsing: "/label"), value: "updated")]) + guard case .document(let staged) = try patch.applying(to: value) else { throw AppError.removed } + return try JSONMergePatch(JSONParser().parse(#"{"optional":null,"ready":true}"#)).applying(to: staged) +} +private enum AppError: Error, Equatable { case removed, source, sink, terminated, accounting, invalid } +private actor Output { + var bytes: [UInt8] = [], calls = 0 + let rejectAt: Int?, cancelAt: Int? + init(rejectAt: Int? = nil, cancelAt: Int? = nil) { self.rejectAt = rejectAt; self.cancelAt = cancelAt } + func accept(_ chunk: JSONByteChunk) throws { + calls += 1 + if calls == rejectAt { throw AppError.sink } + bytes.append(contentsOf: chunk.bytes) + if calls == cancelAt { withUnsafeCurrentTask { $0?.cancel() } } + } + func snapshot() -> ([UInt8], Int) { (bytes, calls) } +} + +@Test func qaE2EExactWorkflowHasIndependentGoldenOutput() async throws { + let parsed = try JSONParser().parseWithReport(Array(input.utf8)) + let read = try parsed.result.readWithReport { root in + #expect(try root.presence(of: "missing") == .absent) + #expect(try root.presence(of: "optional") == .null) + #expect(try root["amount"].requiredNumber().originalLexeme == "1.2300") + #expect(try root["é"].requiredInteger(as: Int.self) == 1) + #expect(try root[JSONKey("e\u{301}")].requiredInteger(as: Int.self) == 2) + return try root.decode(Entry.self) + } + #expect(read.result == Entry(id: 9007199254740993, label: "entry")) + let checked = try JSONSchemaValidator().validateWithReport(parsed.result, against: plan()) + #expect(checked.result.isValid && checked.statistics[.aggregateRegexWorkUnits] > 0) + let result = try edited(parsed.result), sink = Output() + let written = try await JSONAsyncWriter(options: .init(numberStyle: .preserved), chunkSize: 7) + .writeWithReport(result) { try await sink.accept($0) } + let (bytes, calls) = await sink.snapshot() + #expect(bytes == Array(golden.utf8) && calls == (bytes.count + 6) / 7) + #expect(written[.outputBytes] == UInt64(golden.utf8.count)) + #expect(try JSONParser().parse(bytes) == result) + for stats in [parsed.statistics, read.statistics, checked.statistics, written] { + #expect(stats.accountingVersion == 1 && stats[.workUnits] > 0) + #expect(stats.peakLogicalRetainedBytes > 0) + } +} + +@Test func qaE2EVersionOneEnvelopePreservesExactInnerState() throws { + let value = try JSONParser().parse(input), payload = try JSONCodablePayload(preserving: value) + let envelope = try JSONValueEncoder().encode(payload) + #expect(envelope.objectValue?["format"] == "swift-json-value") + #expect(envelope.objectValue?["version"] == 1) + let opened = try JSONValueDecoder().decode(JSONCodablePayload.self, from: envelope).value() + #expect(try JSONWriter(options: .init(numberStyle: .preserved)).write(opened) == Data(input.utf8)) + #expect(opened.objectValue?.members.map { Array($0.key.string.utf8) } == value.objectValue?.members.map { Array($0.key.string.utf8) }) + #expect(try opened.presence(of: "missing") == .absent && opened.presence(of: "optional") == .null) + #expect(try JSONWriter(options: .init(numberStyle: .preserved)).write(edited(opened)) == Data(golden.utf8)) +} + +@Test func qaE2EUnknownEnvelopeVersionIsCorruption() throws { + let payload = try JSONCodablePayload(preserving: JSONParser().parse(input)) + let encoded = try JSONValueEncoder().encode(payload) + let changed = try JSONPatch(operations: [.replace(path: JSONPointer(parsing: "/version"), value: 2)]) + guard case .document(let invalid) = try changed.applying(to: encoded) else { throw AppError.removed } + do { _ = try JSONValueDecoder().decode(JSONCodablePayload.self, from: invalid); Issue.record("Accepted an unknown envelope version") } + catch DecodingError.dataCorrupted(let context) { #expect(context.codingPath.isEmpty) } +} + +@Test func qaE2ESharedPlanRecoversAfterIndependentFailures() async throws { + let shared = try plan() + let outcomes = try await withThrowingTaskGroup(of: Int.self) { group in + for index in 0..<24 { + group.addTask { + switch index % 3 { + case 0: + let result = try JSONSchemaValidator().validate(JSONParser().parse(input), against: shared) + #expect(result.isValid); return 0 + case 1: + let result = try JSONSchemaValidator().validate(JSONParser().parse(#"{"id":1,"label":"UPPER"}"#), against: shared) + #expect(!result.isValid && !result.issues.isEmpty); return 1 + default: + var limits = JSONLimits.balanced; limits[.evaluationWorkUnits] = 0 + do { _ = try JSONSchemaValidator(options: .init(limits: limits)).validate("entry", against: shared); Issue.record("Operational exhaustion became a verdict") } + catch let failure as JSONFailure { #expect(failure.category == .resource && failure.resource?.metric == .evaluationWorkUnits) } + return 2 + } + } + } + var values: [Int] = []; for try await value in group { values.append(value) }; return values + } + #expect(outcomes.filter { $0 == 0 }.count == 8 && outcomes.filter { $0 == 1 }.count == 8 && outcomes.filter { $0 == 2 }.count == 8) + #expect(try JSONSchemaValidator().validate(JSONParser().parse(input), against: shared).isValid) +} + +@Test func qaE2EOfflineReferenceRequiresExplicitRegistry() throws { + let resource = try JSONSchemaDocument(JSONParser().parse(#"{"type":"integer","minimum":7}"#), retrievalURI: JSONSchemaURI("urn:acceptance:item")) + let root = try JSONSchemaDocument(JSONParser().parse(#"{"$ref":"urn:acceptance:item"}"#)) + do { _ = try JSONSchemaCompiler().compile(root); Issue.record("Resolved an unsupplied resource") } + catch let failure as JSONSchemaCompilationFailure { #expect(failure.category == .missingResource) } + let registry = try JSONSchemaRegistry(documents: [resource]) + let shared = try JSONSchemaCompiler().compile(root, registry: registry) + #expect(try JSONSchemaValidator().validate(7, against: shared).isValid) + #expect(try !JSONSchemaValidator().validate(6, against: shared).isValid) +} + +// Carries only owned instruction buckets. Per-stage shapes and live capacities stay finite. +private struct Account { + static let buckets: [JSONResourceMetric] = [.workUnits, .numericWorkUnits, .compileWorkUnits, .evaluationWorkUnits, .regexCompileWorkUnits, .aggregateRegexWorkUnits, .patchWorkUnits, .totalWorkUnits] + var limits: JSONLimits + var terminal = false, reports: [JSONOperationStatistics] = [] + mutating func perform(_ operation: (JSONLimits) throws -> JSONOperationReport) throws -> T { + guard !terminal else { throw AppError.terminated } + do { + let report = try operation(limits) + guard report.statistics.accountingVersion == 1 else { throw AppError.accounting } + var next = limits + for metric in Self.buckets { + let (remaining, overflow) = next[metric].subtractingReportingOverflow(report.statistics[metric]) + guard !overflow else { throw AppError.accounting }; next[metric] = remaining + } + limits = next; reports.append(report.statistics); return report.result + } catch { terminal = true; throw error } + } +} +private func composition(_ account: inout Account, shared: CompiledJSONSchema) throws -> Data { + let value = try account.perform { try JSONParser(options: .init(limits: $0)).parseWithReport(Array(input.utf8)) } + let valid = try account.perform { try JSONSchemaValidator(options: .init(limits: $0)).validateWithReport(value, against: shared) } + guard valid.isValid else { throw AppError.invalid } + let patch = JSONMergePatch(try JSONParser().parse(#"{"optional":null,"ready":true}"#)) + let output = try account.perform { try patch.applyingWithReport(to: value, options: .init(limits: $0)) } + return try account.perform { try JSONWriter(options: .init(limits: $0, numberStyle: .preserved)).writeWithReport(output) } +} +@Test func qaE2ECumulativeWorkSucceedsAtNAndFailsAtNMinusOne() throws { + let shared = try plan(); var measure = Account(limits: .balanced) + let bytes = try composition(&measure, shared: shared) + let total = measure.reports.reduce(UInt64(0)) { $0 + $1[.workUnits] } + #expect(total > 0 && measure.reports.count == 4) + var exact = JSONLimits.balanced; exact[.workUnits] = total + var success = Account(limits: exact) + #expect(try composition(&success, shared: shared) == bytes) + #expect(success.limits[.workUnits] == 0 && !success.terminal) + #expect(success.limits[.evaluationWorkUnits] == exact[.evaluationWorkUnits] - measure.reports[1][.evaluationWorkUnits]) + #expect(success.limits[.patchWorkUnits] == exact[.patchWorkUnits] - measure.reports[2][.patchWorkUnits]) + exact[.workUnits] = total - 1; var failed = Account(limits: exact) + do { _ = try composition(&failed, shared: shared); Issue.record("N-1 succeeded") } + catch let error as JSONFailure { #expect(error.category == .resource && error.resource?.metric == .workUnits && error.resource?.limit == failed.limits[.workUnits]) } + #expect(failed.terminal && failed.reports.count == 3) + let retained = failed.limits[.workUnits] + do { _ = try failed.perform { try JSONParser(options: .init(limits: $0)).parseWithReport([48]) }; Issue.record("Restarted failed composition") } + catch { #expect(error as? AppError == .terminated) } + #expect(failed.limits[.workUnits] == retained && failed.reports.count == 3) +} + +@Test func qaE2EPatchRollbackRootAbsenceAndMergeNullRemainDistinct() throws { + let original = try JSONParser().parse(input) + let patch = try JSONPatch(operations: [.replace(path: JSONPointer(parsing: "/label"), value: "changed"), .test(path: JSONPointer(parsing: "/id"), value: 0)]) + do { _ = try patch.applying(to: original); Issue.record("Failed Patch published a result") } + catch let failure as JSONFailure { #expect(failure.category == .patch && failure.code == .failedTest) } + #expect(try JSONWriter(options: .init(numberStyle: .preserved)).write(original) == Data(input.utf8)) + let root = try JSONPointer(parsing: "") + #expect(try JSONPatch(operations: [.remove(path: root)]).applying(to: original) == .removed) + #expect(try JSONMergePatch(.null).applying(to: original) == .null) + #expect(try JSONPatch(operations: [.remove(path: root), .add(path: root, value: 7)]).applying(to: original) == .document(7)) +} + +@Test func qaE2EBuiltinRegistryHasAllPinnedOfflineIdentities() throws { + let registry = try JSONSchemaRegistry.draft2020_12() + let expected = ["schema", "meta/applicator", "meta/content", "meta/core", "meta/format-annotation", "meta/format-assertion", "meta/meta-data", "meta/unevaluated", "meta/validation"] + #expect(registry.documents.map { $0.retrievalURI?.string } == expected.map { "https://json-schema.org/draft/2020-12/" + $0 }) + #expect(registry.documents.count == 9 && registry.documents.allSatisfy { $0.raw.objectValue != nil }) +} + +@Test func qaE2EAwaitedSinkFailureNeverCertifiesAcceptedPrefix() async throws { + let sink = Output(rejectAt: 2) + var report: JSONOperationStatistics? + do { report = try await JSONAsyncWriter(chunkSize: 2).writeWithReport(.array([1,2,3])) { try await sink.accept($0) }; Issue.record("Sink refusal returned accounting") } + catch { #expect(error as? AppError == .sink) } + let (bytes, calls) = await sink.snapshot() + #expect(bytes == Array("[1".utf8) && calls == 2 && report == nil) +} +@Test func qaE2EAwaitedSinkCancellationKeepsOnlyAcceptedBytes() async { + let sink = Output(cancelAt: 2) + let error = await Task { () -> (any Error)? in + do { _ = try await JSONAsyncWriter(chunkSize: 2).writeWithReport(.array([1,2,3])) { try await sink.accept($0) }; return nil } + catch { return error } + }.value + let (bytes, calls) = await sink.snapshot() + #expect(error is CancellationError && bytes == Array("[1,2".utf8) && calls == 2) +} +@Test func qaE2EAnnotationsFromFailedBranchesDoNotAuthorizeProperties() throws { + let raw = try JSONParser().parse(#"{"anyOf":[{"properties":{"a":{"const":1}},"required":["a"]},{"properties":{"b":{"const":2}},"required":["b"]}],"unevaluatedProperties":false}"#) + let shared = try JSONSchemaCompiler().compile(JSONSchemaDocument(raw)) + let validator = JSONSchemaValidator(options: .init(collectAnnotations: true)) + #expect(try validator.validate(JSONParser().parse(#"{"a":1}"#), against: shared).isValid) + #expect(try validator.validate(JSONParser().parse(#"{"b":2}"#), against: shared).isValid) + let invalid = try validator.validate(JSONParser().parse(#"{"a":0,"b":2}"#), against: shared) + #expect(!invalid.isValid && invalid.issues.contains { $0.instanceLocation == (try? JSONPointer(parsing: "/a")) && $0.keywordLocation == (try? JSONPointer(parsing: "/unevaluatedProperties")) }) +} + +@Test func qaE2EExactScalarAndIntegerConstraintsRejectNearMatches() throws { + let shared = try plan() + #expect(try !JSONSchemaValidator().validate(JSONParser().parse(#"{"id":9007199254740993.5,"label":"entry"}"#), against: shared).isValid) + let original = try JSONParser().parse(#"{"text":"é"}"#) + let patch = try JSONPatch(operations: [.test(path: JSONPointer(parsing: "/text"), value: "é")]) + do { _ = try patch.applying(to: original); Issue.record("Canonical-equivalent scalars passed exact Patch test") } + catch let failure as JSONFailure { #expect(failure.category == .patch && failure.code == .failedTest) } +} diff --git a/Tests/JSONIntegrationTests/SeededReplayTests.swift b/Tests/JSONIntegrationTests/SeededReplayTests.swift new file mode 100644 index 0000000..6e5ba30 --- /dev/null +++ b/Tests/JSONIntegrationTests/SeededReplayTests.swift @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: Apache-2.0 +import Testing +import Foundation +import JSON +import JSONPatch + +private struct ReplayGenerator { + var state: UInt64 = 0x53574a313231 + mutating func next() -> UInt64 { + state = state &* 6364136223846793005 &+ 1442695040888963407; return state + } +} +// Independent positive-integer decimal oracle: factor powers of ten explicitly. +private func decimal(_ value: Int64) -> String { + if value == 0 { return "0" } + var coefficient = value, power = 0 + while coefficient % 10 == 0 { coefficient /= 10; power += 1 } + return String(coefficient) + (power == 0 ? "" : "e" + String(power)) +} +@Test func qaE2ESeededModelsMatchOrderedMemberAndIntegerOracles() throws { + var random = ReplayGenerator() + for index in 0..<128 { + let a = Int64(random.next() % 100_000), b = Int64(random.next() % 100_000) + let text = "{\"case\":\(index),\"é\":\(a),\"e\\u0301\":\(b),\"items\":[true,null,\(a)],\"amount\":\(a).00}" + let value = try JSONParser().parse(text), object = try #require(value.objectValue) + let keys = ["case", "é", "e\u{301}", "items", "amount"].map { Array($0.utf8) } + #expect(object.members.map { Array($0.key.string.utf8) } == keys) + #expect(object["case"] == .number(.init(integer: index))) + #expect(object["é"] == .number(.init(integer: a)) && object["e\u{301}"] == .number(.init(integer: b))) + #expect(object["items"] == .array([true, .null, .number(.init(integer: a))])) + #expect(object["amount"]?.numberValue == JSONNumber(integer: a)) + let n = decimal(Int64(index)), x = decimal(a), y = decimal(b) + let canonical = "{\"case\":\(n),\"é\":\(x),\"é\":\(y),\"items\":[true,null,\(x)],\"amount\":\(x)}" + #expect(try JSONWriter().write(value) == Data(canonical.utf8)) + let changed = try JSONMergePatch(JSONParser().parse(#"{"amount":null,"ok":true}"#)).applying(to: value) + let expected = "{\"case\":\(n),\"é\":\(x),\"é\":\(y),\"items\":[true,null,\(x)],\"ok\":true}" + #expect(try JSONWriter().write(changed) == Data(expected.utf8)) + } +} +@Test func qaE2EMinimizedMalformedReplaysHavePreciseFailures() throws { + // Each edit independently violates JSON grammar; no random 'must be invalid' assumption. + let cases: [(String, [UInt8], JSONFailure.Code)] = [ + ("trailing-comma", Array("[0,]".utf8), .invalidSyntax), + ("leading-zero", Array("[01]".utf8), .invalidNumber), + ("comment", Array("[0/*x*/]".utf8), .invalidSyntax), + ("raw-control", [34,10,34], .invalidSyntax), + ("invalid-utf8", [34,0xc0,0xaf,34], .invalidUTF8), + ("decoded-duplicate", Array(#"{"a":0,"\u0061":1}"#.utf8), .duplicateKey), + ("unpaired-surrogate", Array(#""\uD800""#.utf8), .invalidSyntax), + ("truncated-escape", Array(#""\u00"#.utf8), .unexpectedEOF), + ("second-root", Array("0 1".utf8), .trailingData) + ] + var generator = ReplayGenerator() + for index in 0..<128 { + let number = generator.next() % 99_999 + 1 + var bytes = Array("[\(number)]".utf8) + bytes.insert(48, at: 1) // A leading zero before a nonzero integer is always invalid. + do { _ = try JSONParser().parse(bytes); Issue.record("Accepted seeded leading-zero mutation \(index)") } + catch let failure as JSONFailure { + #expect(failure.category == .syntax && failure.code == .invalidNumber && failure.resource == nil) + } + } + for (id, bytes, code) in cases { + do { _ = try JSONParser().parse(bytes); Issue.record("Accepted minimized replay: \(id)") } + catch let failure as JSONFailure { + #expect(failure.category == .syntax && failure.code == code, "\(id)") + #expect(failure.resource == nil, "\(id)") + #expect((failure.location != nil) == (id != "leading-zero"), "\(id)") + } + } +} diff --git a/Tests/JSONIntegrationTests/StagedImportTests.swift b/Tests/JSONIntegrationTests/StagedImportTests.swift new file mode 100644 index 0000000..ed05876 --- /dev/null +++ b/Tests/JSONIntegrationTests/StagedImportTests.swift @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: Apache-2.0 +import Testing +import JSON +import JSONStreaming + +private enum ImportError: Error, Equatable { case source } +private actor ImportSource { + let texts: [String], failAt: Int?, cancelAt: Int? + var pulls = 0 + init(_ texts: [String], failAt: Int? = nil, cancelAt: Int? = nil) { + self.texts = texts; self.failAt = failAt; self.cancelAt = cancelAt + } + func next() throws -> JSONByteChunk? { + pulls += 1 + if pulls == failAt { throw ImportError.source } + if pulls == cancelAt { withUnsafeCurrentTask { $0?.cancel() } } + return pulls <= texts.count ? .init(Array(texts[pulls - 1].utf8)) : nil + } + func count() -> Int { pulls } +} +private struct Source: AsyncSequence, Sendable { + typealias Element = JSONByteChunk + let state: ImportSource + func makeAsyncIterator() -> Iterator { .init(state: state) } + struct Iterator: AsyncIteratorProtocol { + let state: ImportSource + mutating func next() async throws -> JSONByteChunk? { try await state.next() } + } +} +private struct ImportOutcome: Sendable { + var accepted: [JSON] = [], committed: [JSON] = [], completions = 0 + var error: (any Error)?, terminal = false, statistics: JSONOperationStatistics? +} +// Atomic publication belongs to this test application. Earlier accepted values +// remain visible here even when the enclosing document can never be committed. +private func arrayImport(_ state: ImportSource, limits: JSONLimits = .balanced) async -> ImportOutcome { + var result = ImportOutcome() + var iterator = JSONTopLevelArraySequence(Source(state: state), options: .init(limits: limits)).makeAsyncIterator() + do { + while let event = try await iterator.next() { + switch event { + case .element(let index, let value): + #expect(index == UInt64(result.accepted.count)); result.accepted.append(value) + case .documentCompleted(let summary): + result.completions += 1; result.statistics = summary.statistics + } + } + if result.completions == 1 { result.committed = result.accepted } + } catch { result.error = error } + do { result.terminal = try await iterator.next() == nil } catch { result.terminal = false } + return result +} +private func ndjsonImport(_ state: ImportSource, limits: JSONLimits = .balanced) async -> ImportOutcome { + var result = ImportOutcome() + var iterator = NDJSONValueSequence(Source(state: state), options: .init(limits: limits)).makeAsyncIterator() + do { + while let value = try await iterator.next() { result.accepted.append(value) } + // Only normal exhaustive iteration certifies the NDJSON input for this caller. + result.committed = result.accepted + } catch { result.error = error } + do { result.terminal = try await iterator.next() == nil } catch { result.terminal = false } + return result +} +@Test func qaE2ERootImportCommitsOnlyAfterCertifiedCompletion() async { + let source = ImportSource(["[1,", "2]", " \r\n"]) + let result = await arrayImport(source) + #expect(result.error == nil && result.terminal && result.completions == 1) + #expect(result.accepted == [1,2] && result.committed == [1,2]) + #expect(result.statistics?[.totalInputBytes] == 8 && result.statistics?[.totalRecords] == 2) + #expect(await source.count() == 4) +} +@Test func qaE2ERootImportMalformedTailDoesNotCommitAcceptedElements() async throws { + let source = ImportSource(["[1,", "2]", " false", "unused"]) + let result = await arrayImport(source) + let failure = try #require(result.error as? JSONFailure) + #expect(failure.category == .syntax && failure.code == .trailingData && failure.location?.byteOffset == 6) + #expect(result.accepted == [1,2] && result.committed.isEmpty && result.completions == 0 && result.statistics == nil && result.terminal) + #expect(await source.count() == 3) +} +@Test func qaE2ERootImportQuotaStopsBeforeCommitAndFurtherPulls() async throws { + let source = ImportSource(["[1,", "2]", "unused"]) + var limits = JSONLimits.balanced; limits[.totalRecords] = 1 + let result = await arrayImport(source, limits: limits) + let failure = try #require(result.error as? JSONFailure) + #expect(failure.category == .resource && failure.resource?.metric == .totalRecords) + #expect(failure.resource?.limit == 1 && failure.resource?.attemptedUsage == 2) + #expect(result.accepted == [1] && result.committed.isEmpty && result.completions == 0 && result.terminal) + #expect(await source.count() == 2) +} +@Test func qaE2ERootImportCancellationAfterProvisionalAcceptanceIsTerminal() async { + let source = ImportSource(["[1,", "2]", "unused"], cancelAt: 2) + let result = await Task { await arrayImport(source) }.value + #expect(result.error is CancellationError && result.accepted == [1] && result.committed.isEmpty) + #expect(result.completions == 0 && result.terminal && result.statistics == nil) + #expect(await source.count() == 2) +} +@Test func qaE2ENDJSONImportStrictFramingCommitsOnlyAtEOF() async { + let source = ImportSource(["1\r", "\n2", "\n"]) + let result = await ndjsonImport(source) + #expect(result.error == nil && result.accepted == [1,2] && result.committed == [1,2] && result.terminal) + #expect(await source.count() == 4) +} +@Test func qaE2ENDJSONImportFailuresKeepAcceptedPrefixWithoutCommit() async throws { + for fault in 0..<4 { + let source = ImportSource(fault == 0 ? ["1\n", "2 3\n", "unused"] : ["1\n", "2\n", "unused"], failAt: fault == 1 ? 2 : nil, cancelAt: fault == 2 ? 2 : nil) + var limits = JSONLimits.balanced; if fault == 3 { limits[.totalRecords] = 1 } + let result = await Task { await ndjsonImport(source, limits: limits) }.value + switch fault { + case 0: + let failure = try #require(result.error as? JSONFailure) + #expect(failure.category == .syntax && failure.code == .trailingData) + case 1: #expect(result.error as? ImportError == .source) + case 2: #expect(result.error is CancellationError) + default: + let failure = try #require(result.error as? JSONFailure) + #expect(failure.category == .resource && failure.resource?.metric == .totalRecords) + } + #expect(result.accepted == [1] && result.committed.isEmpty && result.terminal) + #expect(await source.count() == 2) + } +} diff --git a/Tools/DistributionAcceptance/Package.swift b/Tools/DistributionAcceptance/Package.swift new file mode 100644 index 0000000..6881b01 --- /dev/null +++ b/Tools/DistributionAcceptance/Package.swift @@ -0,0 +1,12 @@ +// swift-tools-version: 6.4 +// SPDX-License-Identifier: Apache-2.0 +import PackageDescription +let package = Package( + name: "DistributionAcceptance", platforms: [.macOS("27.0"), .iOS("27.0")], + dependencies: [.package(name: "Swift-JSON", path: "../..")], + targets: [.executableTarget(name: "DistributionAcceptance", dependencies: [ + .product(name: "JSON", package: "Swift-JSON"), + .product(name: "JSONSchema", package: "Swift-JSON"), + .product(name: "JSONStreaming", package: "Swift-JSON"), + .product(name: "JSONPatch", package: "Swift-JSON")])], + swiftLanguageModes: [.v6]) diff --git a/Tools/DistributionAcceptance/Sources/DistributionAcceptance/main.swift b/Tools/DistributionAcceptance/Sources/DistributionAcceptance/main.swift new file mode 100644 index 0000000..7c2a0ef --- /dev/null +++ b/Tools/DistributionAcceptance/Sources/DistributionAcceptance/main.swift @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +import JSONStreaming +import JSONPatch + +@main struct DistributionAcceptance { + static func main() throws { + switch CommandLine.arguments.dropFirst().first { + case "intact": + let registry = try JSONSchemaRegistry.draft2020_12() + guard registry.documents.count == 9 else { throw ProbeError.registry } + let plan = try JSONSchemaCompiler().compile(JSONSchemaDocument(true)) + guard try JSONSchemaValidator().validate(7, against: plan).isValid else { throw ProbeError.verdict } + print("DISTRIBUTION intact nine_offline_resources") + case "unavailable": + do { _ = try JSONSchemaRegistry.draft2020_12(); throw ProbeError.accepted } + catch let failure as JSONSchemaCompilationFailure { + guard failure.category == .metaSchema else { throw failure } + print("DISTRIBUTION unavailable structured_metaSchema") + } + default: throw ProbeError.arguments + } + } + enum ProbeError: Error { case registry, verdict, accepted, arguments } +} From 617c0577713c35bc25d0b0565c1dd5b787976405 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Wed, 14 Oct 2026 20:57:46 +0200 Subject: [PATCH 19/24] fix: report missing schema bundles and close mutation assertion gaps --- .../Qualification/TestDiscovery.json | 2 +- Scripts/ci.py | 2 +- Scripts/coverage.py | 5 ++- Scripts/qualify_distribution.py | 9 ++++- Scripts/test_qualification_harness.py | 7 +++- .../Document/JSONSchemaBuiltins.swift | 34 ++++++++++++++++++- .../PublicWorkflowTests.swift | 26 ++++++++++---- 7 files changed, 72 insertions(+), 13 deletions(-) diff --git a/Documentation/Qualification/TestDiscovery.json b/Documentation/Qualification/TestDiscovery.json index 7800fb0..d2b7ef2 100644 --- a/Documentation/Qualification/TestDiscovery.json +++ b/Documentation/Qualification/TestDiscovery.json @@ -22,7 +22,7 @@ "JSONSchemaTests": 1, "JSONRegexRuntimeTests": 1, "JSONStreamingTests": 16, - "JSONIntegrationTests": 1 + "JSONIntegrationTests": 10 }, "coverage_unmapped": { "Sources/JSON/Pointer/JSONLocationComponent.swift": "declaration-only enum", diff --git a/Scripts/ci.py b/Scripts/ci.py index 708b612..a99781f 100644 --- a/Scripts/ci.py +++ b/Scripts/ci.py @@ -7,7 +7,7 @@ from test_discovery import DISCOVERY, ROOT_COUNTS, PARAMETER_INVOCATIONS ROOT = pathlib.Path(__file__).resolve().parents[1] -SHARED_TESTS = 'independentCodecFacadesAndExactValuesCanBeSharedAcrossTasks|concurrentSchemaEvaluationsHaveExclusiveDynamicRegexAndAnnotationState|sharedRegexProgramHasExclusiveConcurrentSearchState|immutableIndexedObjectCanBeSharedAcrossTasks|qaSharedPayloadAndRootHaveIndependentConcurrentReadScopes|sha256InputCapacityAndConcurrentInvocationsRemainIndependent|qaStreaming|qaE2ESharedPlan' +SHARED_TESTS = 'independentCodecFacadesAndExactValuesCanBeSharedAcrossTasks|concurrentSchemaEvaluationsHaveExclusiveDynamicRegexAndAnnotationState|sharedRegexProgramHasExclusiveConcurrentSearchState|immutableIndexedObjectCanBeSharedAcrossTasks|qaSharedPayloadAndRootHaveIndependentConcurrentReadScopes|sha256InputCapacityAndConcurrentInvocationsRemainIndependent|qaStreaming|qaE2E(SharedPlan|RootImport|NDJSONImport|AwaitedSink|ExactWorkflow)' R0_TESTS = 'readerProbe|indexProbes|duplicateRecognitionPrecedesValueAndClosingSyntax' def snapshot(): diff --git a/Scripts/coverage.py b/Scripts/coverage.py index 42a7395..90e3ff1 100644 --- a/Scripts/coverage.py +++ b/Scripts/coverage.py @@ -88,8 +88,11 @@ def totals(names): percent=100*covered/count if count else None) return result + declared = policy.get('coverage_modules', [name.removesuffix('Tests') for name in policy['root_functions']]) + if set(declared) != {name.split('/')[1] for name in expected}: + raise ValueError('Coverage module policy differs from actual library source') modules = {} - for module in policy.get('coverage_modules', [name.removesuffix('Tests') for name in policy['root_functions']]): + for module in declared: names = {name for name in mapped if name.startswith('Sources/' + module + '/')} if not names or not totals(names)['regions']['count']: raise ValueError('Missing executable module coverage: ' + module) diff --git a/Scripts/qualify_distribution.py b/Scripts/qualify_distribution.py index b1a795f..01c7852 100644 --- a/Scripts/qualify_distribution.py +++ b/Scripts/qualify_distribution.py @@ -25,7 +25,14 @@ def run(name, command, expected=None): bundles = [q for q in candidates if q.is_dir() and q.suffix in ('.bundle','.resources')] assert len(bundles) == 1, 'Missing/ambiguous installed schema bundle' bundle = bundles[0]; files = sorted(q for q in bundle.rglob('*') if q.is_file()) - resource = bundle / 'Draft2020-12/schema.json'; original = resource.read_bytes() + base = bundle / 'Contents/Resources' if (bundle/'Contents').is_dir() else bundle + expected = {'schema.json','Provenance.json'} | {'meta/'+name+'.json' for name in + ['applicator','content','core','format-annotation','format-assertion','meta-data','unevaluated','validation']} + prefix = base.relative_to(bundle).as_posix() + installed = {(base/'Draft2020-12'/name).relative_to(bundle).as_posix() for name in expected} + if prefix != '.': installed.add('Contents/Info.plist') + assert {q.relative_to(bundle).as_posix() for q in files} == installed, 'Unexpected installed bundle contents' + resource = base / 'Draft2020-12/schema.json'; original = resource.read_bytes() receipt['bundle'] = dict(layout=bundle.name, files=[dict(path=q.relative_to(bundle).as_posix(),sha256=hashlib.sha256(q.read_bytes()).hexdigest()) for q in files]) receipt['executable_sha256'] = hashlib.sha256(executable.read_bytes()).hexdigest() run('intact', [str(executable),'intact'], 'DISTRIBUTION intact nine_offline_resources') diff --git a/Scripts/test_qualification_harness.py b/Scripts/test_qualification_harness.py index a80afff..aa6e6e5 100644 --- a/Scripts/test_qualification_harness.py +++ b/Scripts/test_qualification_harness.py @@ -33,7 +33,7 @@ def test_complete_denominator_and_unavailable_counters(self): self.assertEqual(set(result['modules']), {'JSON','JSONPatch'}) def test_integration_bundle_does_not_create_a_library_denominator(self): - self.policy['root_functions']['JSONIntegrationTests'] = 19 + self.policy['root_functions']['JSONIntegrationTests'] = 20 self.policy['coverage_modules'] = ['JSON', 'JSONPatch'] result = validate_export(self.export, self.root, self.policy) self.assertEqual(set(result['modules']), {'JSON', 'JSONPatch'}) @@ -41,6 +41,11 @@ def test_integration_bundle_does_not_create_a_library_denominator(self): with self.assertRaisesRegex(ValueError, 'Incomplete library export'): validate_export(self.export, self.root, self.policy) + def test_module_policy_cannot_hide_a_mapped_library(self): + self.policy['coverage_modules'] = ['JSON'] + with self.assertRaisesRegex(ValueError, 'module policy differs'): + validate_export(self.export, self.root, self.policy) + def test_omitted_module_is_not_a_higher_percentage(self): self.export['data'][0]['files'].pop() with self.assertRaisesRegex(ValueError, 'Incomplete library export'): diff --git a/Sources/JSONSchema/Document/JSONSchemaBuiltins.swift b/Sources/JSONSchema/Document/JSONSchemaBuiltins.swift index e15d763..b873cb4 100644 --- a/Sources/JSONSchema/Document/JSONSchemaBuiltins.swift +++ b/Sources/JSONSchema/Document/JSONSchemaBuiltins.swift @@ -37,7 +37,7 @@ extension JSONSchemaRegistry { let root = URL(fileURLWithPath: command).deletingLastPathComponent() .appendingPathComponent("Swift-JSON_JSONSchema.bundle", isDirectory: true) #else - guard let root = Bundle.module.resourceURL else { throw JSONSchemaCompilationFailure(.metaSchema) } + let root = try Self.builtinResourceRoot(meter: &meter) #endif for (path, identifier, size, expectedHash) in files { try meter.checkpoint(); try meter.charge(.schemaDocuments); try meter.resetValueBounds() @@ -77,6 +77,34 @@ extension JSONSchemaRegistry { } return documents } + #if !os(WASI) + // SwiftPM's synthesized Bundle.module traps when the whole bundle is absent. + // Resolve only standard installed locations and propagate a structured failure. + // This uses Foundation for locating bytes, never as JSON/schema authority. + private static func builtinResourceRoot(meter: inout JSONWorkMeter) throws -> URL { + var candidates: [URL] = [] + #if canImport(ObjectiveC) + let owner = Bundle(for: JSONSchemaBundleMarker.self) + if let resources = Bundle.main.resourceURL { candidates.append(resources) } + if let resources = owner.resourceURL { candidates.append(resources) } + candidates.append(owner.bundleURL.deletingLastPathComponent()) + #endif + candidates.append(Bundle.main.bundleURL) + if let command = CommandLine.arguments.first, command.hasPrefix("/") { + candidates.append(URL(fileURLWithPath: command).deletingLastPathComponent()) + } + for candidate in candidates { + try meter.work(); try meter.checkpoint() + for name in ["Swift-JSON_JSONSchema.bundle", "Swift-JSON_JSONSchema.resources"] { + try meter.work() + if let bundle = Bundle(url: candidate.appendingPathComponent(name)), + let root = bundle.resourceURL { return root } + } + } + throw JSONSchemaCompilationFailure(.metaSchema) + } + #endif + package static func verifyBuiltin(_ data: Data, size: UInt64, expectedHash: String, meter: inout JSONWorkMeter) throws { try meter.work() @@ -87,3 +115,7 @@ extension JSONSchemaRegistry { guard hash.utf8.elementsEqual(expectedHash.utf8) else { throw JSONSchemaCompilationFailure(.metaSchema) } } } + +#if canImport(ObjectiveC) +private final class JSONSchemaBundleMarker {} +#endif diff --git a/Tests/JSONIntegrationTests/PublicWorkflowTests.swift b/Tests/JSONIntegrationTests/PublicWorkflowTests.swift index ce30a6a..b6aa799 100644 --- a/Tests/JSONIntegrationTests/PublicWorkflowTests.swift +++ b/Tests/JSONIntegrationTests/PublicWorkflowTests.swift @@ -193,13 +193,17 @@ private func composition(_ account: inout Account, shared: CompiledJSONSchema) t #expect(bytes == Array("[1".utf8) && calls == 2 && report == nil) } @Test func qaE2EAwaitedSinkCancellationKeepsOnlyAcceptedBytes() async { - let sink = Output(cancelAt: 2) - let error = await Task { () -> (any Error)? in - do { _ = try await JSONAsyncWriter(chunkSize: 2).writeWithReport(.array([1,2,3])) { try await sink.accept($0) }; return nil } - catch { return error } - }.value - let (bytes, calls) = await sink.snapshot() - #expect(error is CancellationError && bytes == Array("[1,2".utf8) && calls == 2) + // Final full-chunk acceptance is a distinct cancellation fault point: there + // may be no later byte reservation to detect it on the writer's behalf. + for (value, size, cancelAt, prefix) in [(JSON.array([1,2,3]), 2, 2, "[1,2"), (JSON.number(.init(integer: 7)), 1, 1, "7")] { + let sink = Output(cancelAt: cancelAt) + let error = await Task { () -> (any Error)? in + do { _ = try await JSONAsyncWriter(chunkSize: size).writeWithReport(value) { try await sink.accept($0) }; return nil } + catch { return error } + }.value + let (bytes, calls) = await sink.snapshot() + #expect(error is CancellationError && bytes == Array(prefix.utf8) && calls == cancelAt) + } } @Test func qaE2EAnnotationsFromFailedBranchesDoNotAuthorizeProperties() throws { let raw = try JSONParser().parse(#"{"anyOf":[{"properties":{"a":{"const":1}},"required":["a"]},{"properties":{"b":{"const":2}},"required":["b"]}],"unevaluatedProperties":false}"#) @@ -209,6 +213,14 @@ private func composition(_ account: inout Account, shared: CompiledJSONSchema) t #expect(try validator.validate(JSONParser().parse(#"{"b":2}"#), against: shared).isValid) let invalid = try validator.validate(JSONParser().parse(#"{"a":0,"b":2}"#), against: shared) #expect(!invalid.isValid && invalid.issues.contains { $0.instanceLocation == (try? JSONPointer(parsing: "/a")) && $0.keywordLocation == (try? JSONPointer(parsing: "/unevaluatedProperties")) }) + // A failed branch can still have successfully evaluated children. Their + // annotations must not authorize a sibling branch's unevaluated properties. + let failedParent = try JSONParser().parse(#"{"anyOf":[{"properties":{"a":true},"required":["missing"]},{"properties":{"b":true},"required":["b"]}],"unevaluatedProperties":false}"#) + let failedPlan = try JSONSchemaCompiler().compile(JSONSchemaDocument(failedParent)) + let failedResult = try validator.validate(JSONParser().parse(#"{"a":1,"b":2}"#), against: failedPlan) + #expect(!failedResult.isValid) + #expect(failedResult.issues.contains { $0.instanceLocation == (try? JSONPointer(parsing: "/a")) && $0.keywordLocation == (try? JSONPointer(parsing: "/unevaluatedProperties")) }) + } @Test func qaE2EExactScalarAndIntegerConstraintsRejectNearMatches() throws { From 9210f2fc717da6744149b78d259cd11598eb2360 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Wed, 14 Oct 2026 21:09:04 +0200 Subject: [PATCH 20/24] test: retain exact acceptance cardinality and add regex mutants --- .../Release1.2.1/MutationPilot.json | 16 ++++++++++++++++ Scripts/ci.py | 2 +- Scripts/test_qualification_harness.py | 4 ++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/Documentation/Qualification/Release1.2.1/MutationPilot.json b/Documentation/Qualification/Release1.2.1/MutationPilot.json index a7f5316..cef8ffe 100644 --- a/Documentation/Qualification/Release1.2.1/MutationPilot.json +++ b/Documentation/Qualification/Release1.2.1/MutationPilot.json @@ -193,6 +193,22 @@ "after": "frame.outcome.valid = true\n if options.output", "family": "ordinary-invalidity", "filter": "qaE2E" + }, + { + "id": "M25", + "path": "Sources/JSONRegexRuntime/RegexVM.swift", + "before": "case .end: success = position == input.count;", + "after": "case .end: success = true;", + "family": "regex-end-anchor", + "filter": "deterministicIndependentRegexDifferentialCases" + }, + { + "id": "M26", + "path": "Sources/JSONRegexRuntime/RegexBudget.swift", + "before": "try meter.charge(.aggregateRegexWorkUnits, amount)", + "after": "/* dropped aggregate regex charge */", + "family": "regex-cumulative-budget", + "filter": "regexSearchLimitResetsButAggregateLimitNeverResets" } ] } diff --git a/Scripts/ci.py b/Scripts/ci.py index a99781f..7e76230 100644 --- a/Scripts/ci.py +++ b/Scripts/ci.py @@ -132,7 +132,7 @@ def check_result(command, code, log): if command[:2] == ['swift', 'run'] and 'Tools/MobileQualification' in command: passed = passed and all(line in log.splitlines() for line in ['PUBLIC_BEHAVIOR passed fixtures=8', 'STREAM_CANCELLATION passed', 'DISK_PERSISTENCE passed containers=3 rows=8', 'MOBILE_QUALIFICATION passed cases=3']) if 'Scripts/test_qualification_harness.py' in command: - passed = passed and 'HARNESS_TESTS passed cases=10' in log.splitlines() + passed = passed and 'HARNESS_TESTS passed cases=12' in log.splitlines() return passed, counts def main(): diff --git a/Scripts/test_qualification_harness.py b/Scripts/test_qualification_harness.py index aa6e6e5..9dd9225 100644 --- a/Scripts/test_qualification_harness.py +++ b/Scripts/test_qualification_harness.py @@ -94,6 +94,10 @@ def test_exact_discovery_rejects_missing_duplicate_and_changed_parameters(self): self.assertFalse(check_root(broken)) self.assertFalse(check_result(['swift','test'],0,broken)[0]) self.assertFalse(check_result(['swift','test'],1,valid)[0]) + harness = ['python3', '-B', 'Scripts/test_qualification_harness.py'] + self.assertTrue(check_result(harness, 0, 'HARNESS_TESTS passed cases=12')[0]) + for stale in ['HARNESS_TESTS passed cases=10', 'HARNESS_TESTS passed cases=11', '']: + self.assertFalse(check_result(harness, 0, stale)[0]) # Scoped portability lanes retain exact discovery and safety rather # than describing an exit-zero subset as full qualification. output = pathlib.Path(__file__).resolve().parents[1] / 'Artifacts/HarnessProbe' From e9aa82b759b17c518fea8c3d98c05c10e1bc57c7 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Wed, 14 Oct 2026 21:24:26 +0200 Subject: [PATCH 21/24] test: qualify integration coverage omission and portable bundle metadata --- Scripts/coverage.py | 12 +++++++++--- Scripts/qualify_distribution.py | 11 +++++++++-- Scripts/test_qualification_harness.py | 5 ++++- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/Scripts/coverage.py b/Scripts/coverage.py index 90e3ff1..1dc40c5 100644 --- a/Scripts/coverage.py +++ b/Scripts/coverage.py @@ -43,6 +43,12 @@ def shared_objects(bin_directory): result[target] = found[0] return result +def without_schema_objects(objects): + # Integration imports schema too, including the private regex implementation. + omitted = {'JSONSchemaTests/runner', 'JSONSchemaTests', 'JSONSchema', + 'JSONIntegrationTests/runner', 'JSONIntegrationTests'} + return {name:path for name,path in objects.items() if name not in omitted} + def validate_export(export, root=ROOT, policy=DISCOVERY): """Validate the executable source denominator before calculating percentages.""" expected = {p.relative_to(root).as_posix() for p in (root / 'Sources').rglob('*.swift')} @@ -193,9 +199,9 @@ def run(command, name, binary=False): receipt['default_export'] = dict(status='complete') except ValueError as error: receipt['default_export'] = dict(status='incomplete', error=str(error)) - # This actual exporter probe must omit the schema runner and its DSOs; + # This actual exporter probe must omit schema and integration object families; # regex code alone is also linked in schema tests and is not a useful omission. - omitted = [p for name,p in materialized_objects.items() if name not in ('JSONSchemaTests/runner', 'JSONSchemaTests', 'JSONSchema')] + omitted = list(without_schema_objects(materialized_objects).values()) negative_command = llvm + ['export', str(omitted[0]), '-instr-profile=' + str(profile)] for other in omitted[1:]: negative_command.extend(['-object', str(other)]) @@ -205,7 +211,7 @@ def run(command, name, binary=False): except ValueError as error: if 'Incomplete library export:' not in str(error): raise - receipt['omitted_binary_probe'] = dict(status='rejected', omitted='JSONSchemaTests runner and schema/test shared objects when present', reason=str(error)) + receipt['omitted_binary_probe'] = dict(status='rejected', omitted='JSONSchemaTests and JSONIntegrationTests runners, test DSOs and schema DSO when present', reason=str(error)) else: raise ValueError('Harness accepted export missing the schema executable') # Public summaries and normalized export contain no local source path. diff --git a/Scripts/qualify_distribution.py b/Scripts/qualify_distribution.py index 01c7852..87cfd3a 100644 --- a/Scripts/qualify_distribution.py +++ b/Scripts/qualify_distribution.py @@ -7,7 +7,9 @@ def main(): p = argparse.ArgumentParser(); p.add_argument('--output', required=True); a = p.parse_args() output = ROOT / a.output; output.mkdir(parents=True, exist_ok=True) - receipt = dict(status='running', commands=[], resource_probes=3) + receipt = dict(status='failed', commands=[], resource_probes=3) + def save(): (output/'distribution.json').write_text(json.dumps(receipt,indent=2)+'\n') + save() def run(name, command, expected=None): start = time.monotonic(); log = output / (name+'.log') with log.open('w') as stream: @@ -15,6 +17,7 @@ def run(name, command, expected=None): text = log.read_text() receipt['commands'].append(dict(command=command, exit_code=result.returncode, seconds=time.monotonic()-start, log=log.name, sha256=hashlib.sha256(log.read_bytes()).hexdigest())) + save() assert result.returncode == 0 and (expected is None or expected in text), name return text.strip() package = 'Tools/DistributionAcceptance' @@ -30,7 +33,11 @@ def run(name, command, expected=None): ['applicator','content','core','format-annotation','format-assertion','meta-data','unevaluated','validation']} prefix = base.relative_to(bundle).as_posix() installed = {(base/'Draft2020-12'/name).relative_to(bundle).as_posix() for name in expected} - if prefix != '.': installed.add('Contents/Info.plist') + metadata = 'Contents/Info.plist' if prefix != '.' else 'Info.plist' + if (bundle/metadata).is_file(): installed.add(metadata) + receipt['observed_layout'] = dict(resource_prefix=prefix, metadata_present=(bundle/metadata).is_file(), + files=[q.relative_to(bundle).as_posix() for q in files]) + save() assert {q.relative_to(bundle).as_posix() for q in files} == installed, 'Unexpected installed bundle contents' resource = base / 'Draft2020-12/schema.json'; original = resource.read_bytes() receipt['bundle'] = dict(layout=bundle.name, files=[dict(path=q.relative_to(bundle).as_posix(),sha256=hashlib.sha256(q.read_bytes()).hexdigest()) for q in files]) diff --git a/Scripts/test_qualification_harness.py b/Scripts/test_qualification_harness.py index 9dd9225..5b72150 100644 --- a/Scripts/test_qualification_harness.py +++ b/Scripts/test_qualification_harness.py @@ -6,7 +6,7 @@ import pathlib import tempfile import unittest -from coverage import executable_objects, shared_objects, validate_export +from coverage import executable_objects, shared_objects, validate_export, without_schema_objects from test_discovery import DISCOVERY, ROOT_COUNTS, PARAMETER_INVOCATIONS, check_root from ci import check_result, commands @@ -37,6 +37,9 @@ def test_integration_bundle_does_not_create_a_library_denominator(self): self.policy['coverage_modules'] = ['JSON', 'JSONPatch'] result = validate_export(self.export, self.root, self.policy) self.assertEqual(set(result['modules']), {'JSON', 'JSONPatch'}) + objects = {name:object() for name in ['JSONSchemaTests/runner', 'JSONIntegrationTests/runner', + 'JSONSchemaTests', 'JSONIntegrationTests', 'JSONSchema', 'JSONTests/runner']} + self.assertEqual(set(without_schema_objects(objects)), {'JSONTests/runner'}) self.export['data'][0]['files'].pop() with self.assertRaisesRegex(ValueError, 'Incomplete library export'): validate_export(self.export, self.root, self.policy) From 7b0c67fd115c0de171dda2b85b0dfa202a501856 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Wed, 14 Oct 2026 21:49:28 +0200 Subject: [PATCH 22/24] docs: close public acceptance and mutation qualification ledger --- Documentation/Qualification/README.md | 6 + .../Release1.2.1/AcceptanceMatrix.json | 23 +- .../Release1.2.1/AcceptanceWave.md | 76 +++ .../Release1.2.1/AcceptanceWaveOutcomes.json | 513 ++++++++++++++++++ .../Qualification/Release1.2.1/README.md | 7 + 5 files changed, 616 insertions(+), 9 deletions(-) create mode 100644 Documentation/Qualification/Release1.2.1/AcceptanceWave.md create mode 100644 Documentation/Qualification/Release1.2.1/AcceptanceWaveOutcomes.json diff --git a/Documentation/Qualification/README.md b/Documentation/Qualification/README.md index 8a551ba..77aaed8 100644 --- a/Documentation/Qualification/README.md +++ b/Documentation/Qualification/README.md @@ -108,3 +108,9 @@ The 1.2.1 [portability follow-up](Release1.2.1/Portability.md) has 245/173 roots WASI debug/release QA passed. Atomic native profiling reports 97.46% authored lines / 93.04% regions and rejects saturated LLVM counters; earlier percentages remain historical measurements, not directly comparable corrected coverage. + +The [public workflow/mutation acceptance wave](Release1.2.1/AcceptanceWave.md) +adds a sixth test-only target while preserving five implementation modules and four +products. Native, both Linux architectures, scoped WASI and clean source extraction +are qualified with exact discoveries and materialized evidence. Mutation survivors +led to real final-chunk cancellation and failed-branch annotation assertions. diff --git a/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json b/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json index 3e4f1a9..66a4b0b 100644 --- a/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json +++ b/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json @@ -3,7 +3,7 @@ "source_version": "1.2.0", "source_commit": "25fb04f747e583209b4c4511abf39e11ef5b4f80", "target_version": "1.2.1", - "status": "core_public_paths_qualified_quiet_diagnostic_calibration_blocked_broader_plan_open", + "status": "public_e2e_mutation_qualified_quiet_diagnostic_calibration_blocked", "rows": [ { "id": "HAR-01", @@ -452,7 +452,7 @@ "work_package": "T05", "module": "Public products", "topic": "Portable public workflow", - "status": "planned_extension", + "status": "qualified_native_linux_and_scoped_wasi", "existing_evidence": [ "Examples/Consumer ProductExamples", "20 rooted public consumer vectors", @@ -465,14 +465,15 @@ "regex through public schema", "exact scalar/number/presence preserved", "positive/invalid/operational separate" - ] + ], + "acceptance_wave_evidence": "AcceptanceWaveOutcomes.json" }, { "id": "E2E-02", "work_package": "T05", "module": "Public products", "topic": "Transactional import and error injection", - "status": "planned_extension", + "status": "qualified_native_linux_and_scoped_wasi", "existing_evidence": [ "provisionalEventsNeverCertifyMalformedTail", "failedStagePermanentlyTerminatesTheCallerAccount", @@ -485,14 +486,15 @@ "no fabricated failure report", "no retry with full allowance", "actual disk phases remain separate" - ] + ], + "acceptance_wave_evidence": "AcceptanceWaveOutcomes.json" }, { "id": "ASS-01", "work_package": "T06", "module": "All", "topic": "Seeded valid/invalid replay and mutants", - "status": "planned_extension", + "status": "qualified_native_linux_and_scoped_wasi", "existing_evidence": [ "qaGeneratedExactRoundTripsAcrossNativeAndForeignPayloads", "deterministicMalformedReplayPreservesStrictProgressAndRoundTrips" @@ -504,7 +506,8 @@ "survived/equivalent/unbuildable/timeout separate", "no blanket nonzero tests", "no conditional optimization selected" - ] + ], + "acceptance_wave_evidence": "AcceptanceWaveOutcomes.json" }, { "id": "QUAL-01", @@ -529,7 +532,8 @@ "second_wave_evidence": "Wave2Outcomes.json", "portability_evidence": "PortabilityOutcomes.json", "coverage_wave_evidence": "CoverageWaveOutcomes.json", - "core_closure_evidence": "CoreClosureOutcomes.json" + "core_closure_evidence": "CoreClosureOutcomes.json", + "acceptance_wave_evidence": "AcceptanceWaveOutcomes.json" } ], "proposed_targets": { @@ -554,5 +558,6 @@ "second_wave_evidence": "Wave2Outcomes.json", "portability_evidence": "PortabilityOutcomes.json", "coverage_wave_evidence": "CoverageWaveOutcomes.json", - "core_closure_evidence": "CoreClosureOutcomes.json" + "core_closure_evidence": "CoreClosureOutcomes.json", + "acceptance_wave_evidence": "AcceptanceWaveOutcomes.json" } diff --git a/Documentation/Qualification/Release1.2.1/AcceptanceWave.md b/Documentation/Qualification/Release1.2.1/AcceptanceWave.md new file mode 100644 index 0000000..b0d626e --- /dev/null +++ b/Documentation/Qualification/Release1.2.1/AcceptanceWave.md @@ -0,0 +1,76 @@ +# Public workflows and mutation acceptance + +Twenty new Swift Testing functions in JSONIntegrationTests use only the four +public products. They combine strict parsing, scoped native projection and +presence, offline schema plans, Patch/Merge Patch, exact Version 1 payloads, +awaited writing and strict reparsing. Independent golden bytes, ordered scalar +keys and integer oracles supplement round trips. Consumer-owned accounts carry +successful cumulative reports; failure/cancellation latches termination without +inventing failed-work accounting or retrying with a full allowance. + +Caller-owned staged root-array and NDJSON import tests keep provisional accepted +values separate from publication. Malformed tails, record exhaustion, source +failure and cancellation prevent commit and stop further pulls. Sink refusal and +cancellation preserve accepted bytes, including cancellation immediately after +accepting the final full output chunk. These tests add no library persistence or +transport semantics. + +The seed `0x53574a313231` produces 128 valid models and 128 guaranteed-invalid +leading-zero edits. [Nine minimized replays](AcceptanceReplays.json) retain exact +bytes, digests and diagnoses. Unicode keys remain scalar-distinct; normalized +number bytes have an independent powers-of-ten oracle. + +## Discovered defect and assertion gaps + +Removing an entire native resource bundle exposed a fatal error in SwiftPM's +generated Bundle.module accessor. The library now searches a finite set of +standard installed locations without trapping and throws structured metaSchema +failure. All nine resource integrity pins remain enforced. A separate public +consumer verifies intact loading, entire-bundle absence and same-size valid-JSON +corruption, restoring every byte afterward. Observed native resources use +Contents/Resources plus Contents/Info.plist; both Linux architectures use flat +Draft2020-12 plus Info.plist. Exact file membership is enforced. + +The [single-site pilot](MutationPilot.json) first killed 22 of 24 mutants. M17 +survived because later allocations masked the missing post-await cancellation +checkpoint; a one-byte final full chunk minimizes that gap. M20 survived because +the failed child itself had no successful evaluated locations; a failed parent +with successfully evaluated children exposes erroneous annotation merging. +Both regressions pass on normal code and kill their mutants. The complete repeat +kills all 24; two private regex anchor/aggregate-budget mutants bring the pilot +to 26 behavioral kills. Zero survivors, equivalents, unbuildable cases, timeouts +or execution errors remain. This bounded pilot does not prove exhaustive correctness. + +## Qualification and retained failures + +[Actual outcomes](AcceptanceWaveOutcomes.json) bind source, environments, commands, +counts, input digests and receipts. Native debug/release/full ASan each run 318 +functions and 173 parameterized invocations; targeted TSan runs 32 functions. +Strict memory safety, DocC, all 49 cap probes, public consumers and actual disk +teardown/reopen tests pass. The pinned independent suites retain 287/318 and +2300/2337 evaluated successes with 31 and 37 policy exclusions separately. + +Three Native CI jobs and six ARM64/x86_64 Linux jobs pass. All six coverage +executables and five implementation denominators remain required. The integration +binary also owns schema mappings, so the actual negative export omits both schema +and integration object families before asserting incomplete schema coverage. +Generated/unmapped source and unavailable branch/MC/DC counters remain explicit. +Apple-only lookup code produces a distinct native denominator; Linux exports agree +across architectures. All per-module native review targets remain met. + +The scoped WASI lane passes 44 commands: Node full debug/release roots and Node/ +WasmKit exact affected QA, with strict memory safety. No WASI sanitizer or separate +independent-suite refresh is inferred. A twice-identical source archive is extracted +without Git/build state, built in release mode, and passes all 20 public integration +functions plus the resource-failure consumer. Final documentation-only changes are +bound separately to the same tested code snapshot in the delivered evidence bundle. + +Compiler/type/assertion corrections, two mutation survivors, the initial native +bundle crash, stale Python harness count, and initial Linux omission/layout failures +remain in the evidence. The exact harness count is twelve; no check was reduced to +exit-zero or merely nonzero discovery. + +No sibling work, API/product/dependency, merge, tag or release is added. Existing +quiet-host diagnostic calibration remains blocked with its unchanged 10% trigger; +this wave adds no performance claim. Historical streaming disposition, deferred +physical iPad and unavailable WASI sanitizers remain recorded separately. diff --git a/Documentation/Qualification/Release1.2.1/AcceptanceWaveOutcomes.json b/Documentation/Qualification/Release1.2.1/AcceptanceWaveOutcomes.json new file mode 100644 index 0000000..0a7e866 --- /dev/null +++ b/Documentation/Qualification/Release1.2.1/AcceptanceWaveOutcomes.json @@ -0,0 +1,513 @@ +{ + "schema_version": 1, + "status": "acceptance_and_mutation_qualified_diagnostic_calibration_still_blocked", + "baseline_head": "c11d96ff8197362537e8353e1ad3199d4f95df3f", + "qualified_code_head": "e9aa82b759b17c518fea8c3d98c05c10e1bc57c7", + "new_test_target": "JSONIntegrationTests", + "new_functions": 20, + "root_functions": 318, + "parameterized_invocations": 173, + "qa_functions": 145, + "tsan_functions": 32, + "python_harness_tests": 12, + "public_products": [ + "JSON", + "JSONSchema", + "JSONStreaming", + "JSONPatch" + ], + "private_implementation": "JSONRegexRuntime", + "root_dependencies": [], + "replay": { + "seed": "0x53574a313231", + "valid_models": 128, + "invalid_leading_zero_mutations": 128, + "minimized_cases": 9, + "registry": "AcceptanceReplays.json", + "registry_sha256": "599c094ca6fa18be18b6b42f7a522a837a0d38b980bebe068692d30ef4690413" + }, + "mutation": { + "selected": 26, + "killed": 26, + "survived": 0, + "equivalent": 0, + "unbuildable": 0, + "timed_out": 0, + "execution_error": 0, + "initial_selected": 24, + "initial_killed": 22, + "initial_survived": [ + "M17", + "M20" + ], + "specification": "MutationPilot.json", + "specification_sha256": "bdb3dc1df8d8dc2820a60290192aa0bf2419f32be7b25e14268aea6998d56c57", + "policy": "Behavioral pilot, not exhaustive correctness or decision coverage" + }, + "defect": { + "source": "Sources/JSONSchema/Document/JSONSchemaBuiltins.swift", + "before": "Missing whole native resource bundle traps in SwiftPM generated Bundle.module", + "after": "Finite installed-location lookup throws structured metaSchema failure; integrity pins unchanged" + }, + "native": { + "commands": 17, + "preflight": { + "schema_version": 1, + "time_utc": "2026-10-14T19:09:13.944930+00:00", + "os": "Darwin", + "architecture": "arm64", + "status": "passed", + "commands": [ + { + "command": [ + "sw_vers", + "-productVersion" + ], + "exit_code": 0, + "stdout": "27.0", + "stderr": "" + }, + { + "command": [ + "xcrun", + "swift", + "--version" + ], + "exit_code": 0, + "stdout": "Apple Swift version 6.4 (swiftlang-6.4.0.34.1 clang-2100.3.34.1)\nTarget: arm64-apple-macosx27.0.0", + "stderr": "swift-driver version: 1.168.6" + }, + { + "command": [ + "xcrun", + "--sdk", + "macosx", + "--show-sdk-version" + ], + "exit_code": 0, + "stdout": "27.2", + "stderr": "" + }, + { + "command": [ + "xcode-select", + "-p" + ], + "exit_code": 0, + "stdout": "/Applications/Xcode-beta.app/Contents/Developer", + "stderr": "" + }, + { + "command": [ + "xcodebuild", + "-version" + ], + "exit_code": 0, + "stdout": "Xcode 27.2\nBuild version 27B5019j", + "stderr": "" + } + ], + "errors": [], + "swift_language_mode_required": "6", + "manifest_compiled": false, + "macos": "27.0", + "swift": "Apple Swift version 6.4 (swiftlang-6.4.0.34.1 clang-2100.3.34.1)", + "sdk": "27.2" + }, + "safety_commands": 3, + "docs_commands": 1, + "independent": { + "json_test_suite": { + "observed": 318, + "evaluated_pass": 287, + "policy_exclusions": 31 + }, + "schema_suite": { + "observed": 2337, + "evaluated_pass": 2300, + "policy_exclusions": 37 + }, + "malformed_replays": 4396 + }, + "cap_probes": 49, + "rooted_consumer": { + "cases": 20, + "valid": 8, + "invalid": 12, + "operational_refusals": 2 + }, + "disk_consumer": { + "containers": 3, + "rows": 8 + } + }, + "remote": { + "native_run": "https://github.com/ThorFuchs/Swift-JSON/actions/runs/38621245405", + "native_source": "9210f2fc717da6744149b78d259cd11598eb2360", + "native_jobs": 3, + "linux_run": "https://github.com/ThorFuchs/Swift-JSON/actions/runs/38623417029", + "linux_source": "e9aa82b759b17c518fea8c3d98c05c10e1bc57c7", + "linux_jobs": 6, + "initial_linux_run": "https://github.com/ThorFuchs/Swift-JSON/actions/runs/38621239738", + "receipt_identity_note": "Linux receipt git_commit is null; source identity verified through GitHub head SHA and every source input digest" + }, + "coverage": { + "native": { + "JSON": { + "mapped_files": 50, + "authored": { + "lines": { + "count": 3443, + "covered": 3381, + "percent": 98.19924484461225 + }, + "regions": { + "count": 3427, + "covered": 3316, + "percent": 96.76101546542165 + }, + "functions": { + "count": 595, + "covered": 589, + "percent": 98.99159663865547 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 0, + "covered": 0, + "percent": null + }, + "regions": { + "count": 0, + "covered": 0, + "percent": null + }, + "functions": { + "count": 0, + "covered": 0, + "percent": null + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + "JSONSchema": { + "mapped_files": 32, + "authored": { + "lines": { + "count": 2437, + "covered": 2408, + "percent": 98.81001231021747 + }, + "regions": { + "count": 3103, + "covered": 2966, + "percent": 95.5849178214631 + }, + "functions": { + "count": 385, + "covered": 376, + "percent": 97.66233766233766 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 0, + "covered": 0, + "percent": null + }, + "regions": { + "count": 0, + "covered": 0, + "percent": null + }, + "functions": { + "count": 0, + "covered": 0, + "percent": null + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + "JSONRegexRuntime": { + "mapped_files": 6, + "authored": { + "lines": { + "count": 659, + "covered": 650, + "percent": 98.63429438543247 + }, + "regions": { + "count": 695, + "covered": 668, + "percent": 96.11510791366906 + }, + "functions": { + "count": 88, + "covered": 86, + "percent": 97.72727272727273 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 30, + "covered": 30, + "percent": 100.0 + }, + "regions": { + "count": 44, + "covered": 44, + "percent": 100.0 + }, + "functions": { + "count": 3, + "covered": 3, + "percent": 100.0 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + "JSONStreaming": { + "mapped_files": 10, + "authored": { + "lines": { + "count": 232, + "covered": 229, + "percent": 98.70689655172414 + }, + "regions": { + "count": 269, + "covered": 259, + "percent": 96.28252788104089 + }, + "functions": { + "count": 60, + "covered": 59, + "percent": 98.33333333333333 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 0, + "covered": 0, + "percent": null + }, + "regions": { + "count": 0, + "covered": 0, + "percent": null + }, + "functions": { + "count": 0, + "covered": 0, + "percent": null + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + }, + "JSONPatch": { + "mapped_files": 4, + "authored": { + "lines": { + "count": 185, + "covered": 184, + "percent": 99.45945945945945 + }, + "regions": { + "count": 212, + "covered": 210, + "percent": 99.05660377358491 + }, + "functions": { + "count": 16, + "covered": 16, + "percent": 100.0 + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + }, + "generated": { + "lines": { + "count": 0, + "covered": 0, + "percent": null + }, + "regions": { + "count": 0, + "covered": 0, + "percent": null + }, + "functions": { + "count": 0, + "covered": 0, + "percent": null + }, + "branches": { + "count": 0, + "covered": 0, + "percent": null + }, + "mcdc": { + "count": 0, + "covered": 0, + "percent": null + } + } + } + }, + "all_five_denominators": true, + "six_root_binaries": true, + "omission": "Schema and integration runners/DSOs omitted together; actual incomplete-schema export rejected", + "branch_counters_available": false, + "mcdc_counters_available": false, + "platform_denominators": "Native Apple lookup branches differ from Linux; both Linux architectures agree" + }, + "wasi": { + "commands": 44, + "node_root_functions_per_configuration": 318, + "parameterized_invocations_per_configuration": 173, + "node_and_wasmkit_qa_per_configuration": 145, + "scope": "Node full roots; Node and WasmKit exact affected QA; strict-memory-safety. No separate WASI corpora refresh or sanitizer claim" + }, + "clean_code_archive": { + "sha256": "33117803bf485ecb8cbe62a75bebe74ad99f76d0f34f9cc34c31f6a8d5281e26", + "bytes": 1251634, + "entries": 861, + "byte_equal_rebuild": true, + "build_exit": 0, + "integration_test_exit": 0, + "integration_functions": 20, + "distribution_exit": 0 + }, + "receipts": { + "native": { + "path": "Artifacts/AcceptanceWave/Accepted-native/receipt.json", + "sha256": "ac280dd0fcac9976d7714b633848d8b003e92f58481decbac3a312fc66df54dd" + }, + "safety": { + "path": "Artifacts/AcceptanceWave/Final-safety/receipt.json", + "sha256": "a8982430b26a3fb14596fb4db64b4bb8426b60870444fa9ed859cc1b080f9127" + }, + "docs": { + "path": "Artifacts/AcceptanceWave/Final-docs/receipt.json", + "sha256": "229a3d34247a6ba64b9b684f82cb53e00d666245a68b978871a20b54db4cc06d" + }, + "coverage": { + "path": "Artifacts/AcceptanceWave/Accepted-coverage/receipt.json", + "sha256": "26540ecbe3a6db0267033979b3a9e9a79daaf1e30a49fbb54db937444bd6e4d6" + }, + "wasi": { + "path": "Artifacts/AcceptanceWave/Accepted-wasi/receipt.json", + "sha256": "11d47b389c2cbefd754fcc31412e6dbff8d09af3fd73a3101847bc987af8ab8e" + }, + "clean_code_rebuild": { + "path": "Artifacts/AcceptanceWave/CleanCodeRebuild/receipt.json", + "sha256": "f7ef0239180cceb62659ca1b5036165c8c1085567ea00ea7e5c9e6770da2e600" + }, + "initial_mutation": { + "path": "Artifacts/AcceptanceWave/MutationInitial/mutation.json", + "sha256": "dd9350be29584e0ee22179876ec1eab525529353c716d398b95cd7e709afc5fa" + }, + "repeated_mutation": { + "path": "Artifacts/AcceptanceWave/MutationFinal/mutation.json", + "sha256": "beb325d1189d7864d5c080dbf7df120eb14ec46bc16e00e74dbe5fb7b1aab2a3" + }, + "regex_mutation": { + "path": "Artifacts/AcceptanceWave/RegexMutation/mutation.json", + "sha256": "ca16fb2fe2485a3aa75b270eb0675a38acc3c7dd3d3f27840f7fc72b2f680742" + } + }, + "history_preserved": true, + "source_changes_since_safety": "Only harness/spec/documentation changed after 617c057; Sources, Package.swift and behavior tests are identical", + "remaining": [ + "Quiet-host diagnostic recalibration remains blocked; existing +11.983% long-run median exceeds unchanged 10% review trigger", + "Broader matrix rows retain their specific existing scope/dispositions; this closes T05/T06, not a claim of exhaustive tests", + "Physical iPad deferred; WASI sanitizers unavailable under existing policy" + ], + "no_merge_tag_release": true, + "sibling_1_3_untouched": true +} diff --git a/Documentation/Qualification/Release1.2.1/README.md b/Documentation/Qualification/Release1.2.1/README.md index c531c44..b48beb8 100644 --- a/Documentation/Qualification/Release1.2.1/README.md +++ b/Documentation/Qualification/Release1.2.1/README.md @@ -47,3 +47,10 @@ qualification is recorded separately from the earlier unchanged-source receipts. The [reachable core follow-up](CoreClosure.md) adds 22 public behavior functions and extends escaped schema-output/presence assertions. Unadjusted coverage meets the five-module review targets; quiet-host diagnostic recalibration stays blocked. + +The [public acceptance and mutation wave](AcceptanceWave.md) closes T05/T06: +20 public integration functions, 318/173 roots, 145 QA and 32 targeted TSan; +26 behavioral mutant kills after two minimized assertion repairs. It also repairs +missing native schema-bundle failure and qualifies actual installed resources. +[Current outcomes](AcceptanceWaveOutcomes.json) retain every earlier wave and +separate native, Linux, WASI, archive and outstanding calibration evidence. From 7907333dcf5f31c7999dc4681bba91e38169c030 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Wed, 14 Oct 2026 21:50:01 +0200 Subject: [PATCH 23/24] docs: distinguish native mutation pilot from portable replay tests --- Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json b/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json index 66a4b0b..936a25b 100644 --- a/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json +++ b/Documentation/Qualification/Release1.2.1/AcceptanceMatrix.json @@ -494,7 +494,7 @@ "work_package": "T06", "module": "All", "topic": "Seeded valid/invalid replay and mutants", - "status": "qualified_native_linux_and_scoped_wasi", + "status": "native_mutation_pilot_qualified_replays_portably_qualified", "existing_evidence": [ "qaGeneratedExactRoundTripsAcrossNativeAndForeignPayloads", "deterministicMalformedReplayPreservesStrictProgressAndRoundTrips" From f6ef9f7eda52864eaba8d399ff99107b66107dce Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Wed, 14 Oct 2026 22:18:49 +0200 Subject: [PATCH 24/24] docs: prepare 1.2.1 review and record loaded diagnostics --- CHANGELOG.md | 6 + Documentation/Performance/README.md | 6 + .../Qualification/Release1.2.1/README.md | 4 + .../Release1.2.1/ReviewReadiness.json | 215 ++++++++++++++++++ .../Release1.2.1/ReviewReadiness.md | 106 +++++++++ 5 files changed, 337 insertions(+) create mode 100644 Documentation/Qualification/Release1.2.1/ReviewReadiness.json create mode 100644 Documentation/Qualification/Release1.2.1/ReviewReadiness.md diff --git a/CHANGELOG.md b/CHANGELOG.md index a0bdebf..42cc101 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ terminal. Add 31 behavior and precise failure tests across all five modules, plus reproducible five-module Linux coverage exports on ARM64 and x86_64. +- Extend Codable, exact value, streaming, Patch, schema/regex and public workflow + assertions to 318 Swift Testing functions. Preserve precise errors, complete + five-module coverage and finite cumulative accounting; add seeded replays and + a bounded native mutation pilot. Report a wholly missing native schema bundle + as structured metaSchema failure while retaining all resource integrity pins. + - Add local full-language wasm32 WASI support, preserving all four products and the empty root dependency graph. Twelve cumulative UInt64 quotas can exceed Int.max without changing profiles; allocation capacities retain Int guards. diff --git a/Documentation/Performance/README.md b/Documentation/Performance/README.md index ac190a2..4f3e9f9 100644 --- a/Documentation/Performance/README.md +++ b/Documentation/Performance/README.md @@ -29,3 +29,9 @@ Logical retained bytes differ from process RSS. Darwin ru_maxrss is cumulative; allocator end observations are not isolated workload allocations. These results make no cross-machine throughput or OS-enforced memory guarantee. Projection and JSON text-sequence framing remain deferred. + +The proposed 1.2.1 diagnostic repair has a separate +[matched comparison and disposition](../Qualification/Release1.2.1/ReviewReadiness.md). +The earlier +11.9833% median remains unresolved; a subsequently authorized +loaded-host run measured −1.6871%, with competing activity recorded. It establishes +no intrinsic improvement and does not replace qualifying quiet-host evidence. diff --git a/Documentation/Qualification/Release1.2.1/README.md b/Documentation/Qualification/Release1.2.1/README.md index b48beb8..fb448c2 100644 --- a/Documentation/Qualification/Release1.2.1/README.md +++ b/Documentation/Qualification/Release1.2.1/README.md @@ -54,3 +54,7 @@ The [public acceptance and mutation wave](AcceptanceWave.md) closes T05/T06: missing native schema-bundle failure and qualifies actual installed resources. [Current outcomes](AcceptanceWaveOutcomes.json) retain every earlier wave and separate native, Linux, WASI, archive and outstanding calibration evidence. + +The [review closeout](ReviewReadiness.md) preserves those results and records the +third quiet-host readiness attempt. Review proceeds as a draft; diagnostic +recalibration remains blocked and is required before a release decision. diff --git a/Documentation/Qualification/Release1.2.1/ReviewReadiness.json b/Documentation/Qualification/Release1.2.1/ReviewReadiness.json new file mode 100644 index 0000000..bd04178 --- /dev/null +++ b/Documentation/Qualification/Release1.2.1/ReviewReadiness.json @@ -0,0 +1,215 @@ +{ + "schema_version": 1, + "status": "draft_review_ready_quiet_calibration_unqualified", + "date": "2026-10-14", + "baseline": "25fb04f747e583209b4c4511abf39e11ef5b4f80", + "implementation_head": "7907333dcf5f31c7999dc4681bba91e38169c030", + "changes": "Closeout documentation only; implementation/test/harness inputs unchanged", + "native": { + "os": "macOS 27.0", + "architecture": "arm64", + "swift": "6.4 swiftlang-6.4.0.34.1", + "xcode": "27.2 27B5019j", + "sdk": "27.2", + "preflight_exit": 0 + }, + "quiet_attempt": { + "exit_code": 2, + "status": "blocked", + "logical_cores": 10, + "samples": 5, + "idle_percent": [ + 14.84, + 30.72 + ], + "load1": [ + 17.560546875, + 19.29443359375 + ], + "timings_executed": false, + "criteria": "Every sample: >=80% CPU idle, load1 <= half logical cores, no Swift compiler/test or WasmKit process, thermal query with no warning. Five samples. No processes stopped or priorities changed." + }, + "loaded_measurement": { + "status": "measured_under_load", + "parent_source": "5fcc87bef006a5d7791f0dba049f3a43ac3faaec", + "current_runtime_source": "20476dace4626528aba36649df2686b2158c7f9f", + "parent_binary_sha256": "a6219b1a2b6f519c43ad2479b97e1a4d68e1f5eff96daecc13f51f029d674731", + "current_binary_sha256": "9700ff4a92a5fb13b90a320db4166b7d372ebd46bb97fcdb99fbd5fda75bbc86", + "parent_median_seconds": 0.000576132609375, + "current_median_seconds": 0.000566412515625, + "change_percent": -1.6871278576896631, + "paired_changes_percent": [ + -10.731164270025884, + 3.6820517761657845, + 0.37760520240457396, + 0.37873241241175304, + -2.181057318314905 + ], + "quiet_before": false, + "quiet_during": false, + "quiet_after": false, + "execution_policy": "User explicitly requested execution despite competing CPU load. Host gate remains unchanged; this run cannot close quiet-host qualification.", + "exit_code": 0, + "processes": 10, + "alternating_pairs": 5, + "warmups_per_process": 3, + "samples_per_process": 9, + "invocations_per_sample": 1024, + "issues_per_invocation": 32, + "diagnostic_bytes": 7726, + "evaluation_work_units": 23833, + "peak_logical_retained_bytes": 7152, + "all_counter_pairs_equal": true, + "host_observations": 50, + "host_idle_percent": [ + 2.61, + 58.56 + ], + "host_load1": [ + 21.97509765625, + 31.68603515625 + ], + "samples_discarded": 0, + "interpretation": "Wall timing under competing CPU load; logical retention differs from RSS and owned work from caller instructions" + }, + "review": { + "trigger_percent": 10, + "trigger_unchanged": true, + "original_long_median_percent": 11.98325099056705, + "disposition": "Retain the correctness fix. Loaded-host aggregate -1.6871% does not establish intrinsic improvement or resolve original +11.9833%. Quiet-host qualification remains outstanding before release decision.", + "historical_streaming_percent": 10.9932, + "historical_streaming_disposition_unchanged": true + }, + "reuse": [ + { + "name": "Accepted-native", + "path": "Artifacts/AcceptanceWave/Accepted-native/receipt.json", + "sha256": "ac280dd0fcac9976d7714b633848d8b003e92f58481decbac3a312fc66df54dd", + "source": "9210f2fc717da6744149b78d259cd11598eb2360", + "status": "passed", + "commands": 17, + "unchanged_compilation_test_consumer_files": 754, + "scripts_changed_after_receipt": [ + "Scripts/coverage.py", + "Scripts/qualify_distribution.py", + "Scripts/test_qualification_harness.py" + ] + }, + { + "name": "Final-safety", + "path": "Artifacts/AcceptanceWave/Final-safety/receipt.json", + "sha256": "a8982430b26a3fb14596fb4db64b4bb8426b60870444fa9ed859cc1b080f9127", + "source": "617c0577713c35bc25d0b0565c1dd5b787976405", + "status": "passed", + "commands": 3, + "unchanged_compilation_test_consumer_files": 754, + "scripts_changed_after_receipt": [ + "Scripts/ci.py", + "Scripts/coverage.py", + "Scripts/qualify_distribution.py", + "Scripts/test_qualification_harness.py" + ] + }, + { + "name": "FinalClosureDocs", + "path": "Artifacts/AcceptanceWave/FinalClosureDocs/receipt.json", + "sha256": "72ebc75aa748a1811aa56f92fd8e32beeaf69d4055b357c9b7ef87bb7525db7a", + "source": "7907333dcf5f31c7999dc4681bba91e38169c030", + "status": "passed", + "commands": 1, + "unchanged_compilation_test_consumer_files": 754, + "scripts_changed_after_receipt": [] + }, + { + "name": "Accepted-coverage", + "path": "Artifacts/AcceptanceWave/Accepted-coverage/receipt.json", + "sha256": "26540ecbe3a6db0267033979b3a9e9a79daaf1e30a49fbb54db937444bd6e4d6", + "source": "e9aa82b759b17c518fea8c3d98c05c10e1bc57c7", + "status": "passed", + "commands": 2, + "unchanged_compilation_test_consumer_files": 754, + "scripts_changed_after_receipt": [] + }, + { + "name": "Accepted-wasi", + "path": "Artifacts/AcceptanceWave/Accepted-wasi/receipt.json", + "sha256": "11d47b389c2cbefd754fcc31412e6dbff8d09af3fd73a3101847bc987af8ab8e", + "source": "e9aa82b759b17c518fea8c3d98c05c10e1bc57c7", + "status": "passed", + "commands": 2, + "unchanged_compilation_test_consumer_files": 754, + "scripts_changed_after_receipt": [] + } + ], + "fresh_checks": { + "harness_cases": 12, + "public_distribution_privacy": "passed", + "whitespace": "passed", + "required_pr_checks": [ + "Native / native", + "Native / safety", + "Native / docs" + ], + "current_pr_checks": "rerun on PR; status recorded separately" + }, + "receipts": { + "Preflight.json": { + "path": "Artifacts/DiagnosticReview/Preflight.json", + "sha256": "7e7c048814c0bc34e2a4eaa59bebd5453caf95421cb7cf381954e25af181d623" + }, + "Command.json": { + "path": "Artifacts/DiagnosticReview/Command.json", + "sha256": "014a38a64ccd52dce091fe7e68f6e0019910022ced39f74307b2c241739306b5" + }, + "HostBefore.json": { + "path": "Artifacts/DiagnosticReview/HostBefore.json", + "sha256": "e98e5675cef197649edfe9f6ddd92aa5452bc94b8c31ff1a5d6c9498dd3d23e5" + }, + "Recalibration.json": { + "path": "Artifacts/DiagnosticReview/Recalibration.json", + "sha256": "901737d36596fd967246b3a265ba8f0777edcb2edcf56935c5cb5603bc18db1c" + }, + "SourceBinding.json": { + "path": "Artifacts/DiagnosticReview/SourceBinding.json", + "sha256": "4077b71c4264ea318292b78608f207d7c663956e1adf3fd023f8c9271205caeb" + }, + "EvidenceReuse.json": { + "path": "Artifacts/DiagnosticReview/EvidenceReuse.json", + "sha256": "d5ac7b0f30a2d0217786d88f83690ece8a10fc62c36698621d8f3cd2f11a417f" + }, + "LoadedRun/Command.json": { + "path": "Artifacts/DiagnosticReview/LoadedRun/Command.json", + "sha256": "cda4b26aa9eeb1fb4e4fef767926f918c075d4f825f6a97bbb7f84ab7b35f9e2" + }, + "LoadedRun/Recalibration.json": { + "path": "Artifacts/DiagnosticReview/LoadedRun/Recalibration.json", + "sha256": "c7cd178cb015de0a34349e194e69d00cc6d90c4dcca0dae0c62a29d72afa6c35" + }, + "LoadedRun/HostBefore.json": { + "path": "Artifacts/DiagnosticReview/LoadedRun/HostBefore.json", + "sha256": "b0084e0e39fc708fbc0fdb2ac47fe5534293b18588de8a37e4c3f5c27746a1c3" + }, + "LoadedRun/HostDuring.json": { + "path": "Artifacts/DiagnosticReview/LoadedRun/HostDuring.json", + "sha256": "f25a002b1603b6d6f2fdf9f0aa6153073c1e00739d863590ec9245581582e8e6" + }, + "LoadedRun/HostAfter.json": { + "path": "Artifacts/DiagnosticReview/LoadedRun/HostAfter.json", + "sha256": "6d6d5d32b753a845fb9a894b33311dafb869317a517b12c461c2b9bb5c56150a" + }, + "LoadedRun/loaded_calibration.py": { + "path": "Artifacts/DiagnosticReview/LoadedRun/loaded_calibration.py", + "sha256": "e1563912a7805c1c1b935d18386d4acd672766c5ad52fab55561e43c90a44273" + }, + "Remote/Protection.json": { + "path": "Artifacts/DiagnosticReview/Remote/Protection.json", + "sha256": "c9e9942f2e720ab7989881844c0732e7a0ea918526c7cb86c912ff2b3e349a8b" + } + }, + "remaining": [ + "Qualifying quiet-host diagnostic comparison", + "Current PR checks and maintainer review before separate merge decision", + "Existing physical iPad deferral and unavailable WASI sanitizers" + ], + "merge_tag_publication_performed": false +} diff --git a/Documentation/Qualification/Release1.2.1/ReviewReadiness.md b/Documentation/Qualification/Release1.2.1/ReviewReadiness.md new file mode 100644 index 0000000..138b2c8 --- /dev/null +++ b/Documentation/Qualification/Release1.2.1/ReviewReadiness.md @@ -0,0 +1,106 @@ +# 1.2.1 review readiness + +This branch is based on published 1.2.0, `25fb04f747e583209b4c4511abf39e11ef5b4f80`. +The completed [acceptance wave](AcceptanceWave.md) remains unchanged. This +closeout adds review records; it changes no library, test or qualification-harness +code and incorporates no sibling 1.3.0 work. Merge, tagging and publication remain +separate maintainer decisions. + +## Proposed patch + +The branch adds 126 Swift Testing functions to the 192-function baseline: +318 root functions, 173 parameterized invocations, 145 affected QA functions and +32 targeted shared-state functions. Six test targets cover five implementation +modules. Twenty integration functions depend only on the four public products. + +The two production repairs are deliberately narrow: + +- Best-effort schema diagnostics can omit oversized optional output without + latching an owned resource failure. Ordinary invalidity remains a completed + result; required work, retention and annotation failures remain terminal. +- A completely missing native schema resource bundle produces structured + metaSchema failure instead of SwiftPM's generated Bundle.module fatal error. + Location search is finite; all nine integrity pins remain enforced. + +The public products, signatures, empty root dependency graph, Swift 6.4/language 6, +Apple 27 minima, Linux/WASI scopes, exact values/scalars, cumulative limits and +upstream notices remain unchanged. Typed Int128/UInt128 Codable remains explicitly +unsupported; exact JSONNumber projection is separately qualified. + +## Fresh quiet-host attempt — 2026-10-14 + +Native preflight passed on macOS 27.0 ARM64, Swift 6.4 +`swiftlang-6.4.0.34.1`, Xcode 27.2 `27B5019j`, SDK 27.2. +The third readiness attempt exited **2**. All five observations failed the +unchanged gate: CPU idle 14.84–30.72%, load1 17.56–19.29 on ten logical cores, +with competing Swift compiler/test processes. The gate requires every sample to +have at least 80% idle CPU, load1 at most five, no competing compiler/test/WasmKit +process, successful observations and no thermal warning. No other process was +stopped or reprioritized. **No new benchmark timings were taken.** + +Both frozen release benchmark binaries match their original SHA-256 receipts; +the parent/current authored workload sources are identical. All evaluation/runtime +sources match the qualified diagnostic repair, except the later resource locator, +which is outside the timed validation loop. Plan compilation occurs before timing. +The retained protocol uses 32 issues, three warmups, nine samples of 1024 validations +and five alternating parent/current pairs; it discards no timing sample. + +The previous longer diagnostic median remains **+11.9833%**, with pair changes +from −0.5332% to +73.2086%, above the unchanged **10% review trigger**. Retain the +correctness repair; attribution remains unresolved. A qualifying quiet-host +comparison is still needed before a 1.2.1 release decision. This branch can be +reviewed as a draft while that blocker remains. Historical streaming +10.9932% +and its existing reviewed disposition are unchanged. Equal logical retention and +owned work in the diagnostic pairs establish neither RSS nor caller-work caps. + +## Subsequent measurement under competing load + +The maintainer then explicitly requested running despite the busy host. The +unchanged frozen workload completed all ten processes, five alternating pairs, +and all nine samples per process without discarding any sample; the command +exited **0**. Median-of-run medians were 0.5761326 ms per validation for the parent +and 0.5664125 ms for the repaired evaluator: **−1.6871%**. Individual pair changes +were −10.7312%, +3.6821%, +0.3776%, +0.3787% and −2.1811%. + +Fifty before/during/after host observations recorded CPU idle 2.61–58.56% and +load1 21.98–31.69 on ten cores. All before/during/after quiet classifications +failed. Every pair retained 32 issues, 7726 diagnostic bytes, 23833 evaluation +work units and 7152 peak logical retained bytes. Process CPU/context-switch +observations are scheduling context only. + +Disposition: this loaded-host aggregate is below the unchanged 10% trigger, but +does not establish an intrinsic improvement or resolve the earlier +11.9833% +comparison. Preserve both observations and the correctness repair. The explicit +authorization to measure under load does not turn it into quiet-host evidence. +No threshold, quota, optimization or release gate was relaxed. + +## Qualification and review scope + +The passed [Native run](https://github.com/ThorFuchs/Swift-JSON/actions/runs/38621245405) +and [Linux run](https://github.com/ThorFuchs/Swift-JSON/actions/runs/38623417029) +remain bound to their actual historical heads. Fresh file comparisons confirm +754 library/test/consumer inputs are unchanged. Older Native receipts predate +some harness repairs; their reuse does **not** assert identical harness bytes. +Later native coverage, clean distribution/reproduction, Linux and WASI receipts +cover those repairs. The PR reruns current required Native checks and Linux lanes. + +Five-module native coverage meets the 98% line / 95% region review targets without +removing unused-helper lines. Both Linux architectures retain their own complete +exports. Branch/MC/DC counters remain unavailable. The native mutation pilot +killed 26 selected behavioral mutants; it is bounded assertion evidence, not +exhaustive assurance. Fixed-seed replay tests are portable; the mutation pilot +itself was executed natively. WASI scoped QA is not a new sanitizer/corpus claim. +Physical iPad qualification remains deferred under the existing policy. + +Review checks found no additional production/API change. Exact six-target +discovery, all five coverage denominators, actual exporter omission rejection, +49 finite cap probes, policy exclusions and invalid/error distinctions remain +explicit. Fresh Python harness tests (12 cases), public-distribution/privacy +checks and whitespace checks passed. Repository protection remains strict +`Native / native`, `Native / safety`, `Native / docs`, with resolved review threads +and no bypass; no settings were changed. + +[Machine-readable readiness and receipt hashes](ReviewReadiness.json) bind this +attempt to materialized local evidence. Earlier source archives and evidence +bundles remain immutable. Current PR checks must pass before a separate merge +decision; this record grants no merge or release approval.