diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 321e20d..a90a827 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,3 +60,6 @@ jobs: - run: /tmp/samsarix-spirals-smoke/bin/samsarix-spirals validate examples/hello.json - run: /tmp/samsarix-spirals-smoke/bin/samsarix-spirals run examples/hello.json --input examples/hello.input.json --compact - run: /tmp/samsarix-spirals-smoke/bin/samsarix-spirals test examples/release-policy.json examples/release-policy.suite.json --json --compact + - run: /tmp/samsarix-spirals-smoke/bin/samsarix-spirals test examples/release-policy.json examples/release-policy.suite.json --junit + - run: /tmp/samsarix-spirals-smoke/bin/samsarix-spirals schema workflow --compact + - run: /tmp/samsarix-spirals-smoke/bin/samsarix-spirals schema suite --compact diff --git a/CHANGELOG.md b/CHANGELOG.md index 6259523..c7b8aec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,9 +11,14 @@ use semantic versioning while the public API remains pre-1.0. contracts through the Python API and `samsarix-spirals test` command. - A release-policy example that demonstrates a practical CI approval gate. - Competitive positioning, flagship use cases, and measurable adoption gates. +- Bundled JSON Schema Draft 2020-12 documents with CLI and Python discovery APIs. +- Deterministic, value-redacted JUnit XML reports for native CI ingestion. ### Changed +- Exact suite-output comparison now distinguishes JSON booleans from numbers while + retaining JSON numeric equality between integer and decimal representations. + - Renamed the product, distribution, import package, CLI, and public base exception from Helix Spirals to Samsarix Spirals under Samsarix LLC ownership. - Replaced the inconsistent custom BSL text with the unmodified Mozilla Public License 2.0 diff --git a/MANIFEST.in b/MANIFEST.in index be1ee16..7951256 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,3 +4,4 @@ include NOTICE include SECURITY.md recursive-include docs *.md recursive-include examples *.json +recursive-include src/samsarix_spirals/schemas *.json diff --git a/README.md b/README.md index d70d4d0..4bfae2c 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ On macOS or Linux, use `.venv/bin/python` in place of `.venv\Scripts\python`. .venv\Scripts\samsarix-spirals validate examples/hello.json .venv\Scripts\samsarix-spirals run examples/hello.json --input examples/hello.input.json .venv\Scripts\samsarix-spirals test examples/release-policy.json examples/release-policy.suite.json +.venv\Scripts\samsarix-spirals schema workflow --compact ``` The output's `output` field is: @@ -76,11 +77,31 @@ reduces accidental disclosure of fixture data in CI logs. ```console samsarix-spirals test workflow.json workflow.suite.json samsarix-spirals test workflow.json workflow.suite.json --json --compact +samsarix-spirals test workflow.json workflow.suite.json --junit ``` See [`examples/release-policy.suite.json`](examples/release-policy.suite.json) for a release approval gate with both successful and rejected cases. +## JSON Schemas and CI reports + +Draft 2020-12 schemas for workflow and suite version `1` ship inside every wheel. Print +them without locating package files: + +```console +samsarix-spirals schema workflow +samsarix-spirals schema suite --compact +``` + +The schemas provide editor completion and structural validation. Runtime validation is +still authoritative for document byte/depth budgets, unique IDs and names, and semantic +template references. The schema `$id` values are stable identifiers; they do not promise +that a public schema host is deployed yet. + +For CI systems that ingest JUnit XML, use `--junit`. The deterministic report contains +suite and case names plus non-sensitive mismatch categories, but never fixture inputs, +expected outputs, or actual outputs. + ## Python API ```python @@ -91,6 +112,9 @@ result = run_workflow(workflow, {"name": "Ada"}) print(result.output) ``` +Bundled schemas and suite reports are also available through `get_schema` and +`suite_result_to_junit_xml` in the typed Python API. + The pre-1.0 API can change between minor releases. Workflow schema changes will use the top-level `schema_version` field and be documented in the changelog. diff --git a/ROADMAP.md b/ROADMAP.md index 9fcb49b..17b03e3 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -41,8 +41,8 @@ this boundary. - [x] Exact-output and expected-error assertions. - [x] Human and machine-readable reports with CI exit behavior. - [x] A realistic release-policy example. -- [ ] Publish JSON Schemas for workflows and suites. -- [ ] Emit JUnit XML for native CI test reporting. +- [x] Publish JSON Schemas for workflows and suites. +- [x] Emit JUnit XML for native CI test reporting. ### 0.3 — Useful deterministic shaping diff --git a/docs/WORKFLOW_FORMAT.md b/docs/WORKFLOW_FORMAT.md index 7dc49f9..b05e45a 100644 --- a/docs/WORKFLOW_FORMAT.md +++ b/docs/WORKFLOW_FORMAT.md @@ -163,3 +163,26 @@ Unknown fields are rejected. Suite files use the same 1 MiB, UTF-8, unique-key, finite-number, nesting, string, collection, and total-value limits as workflow and input documents. A suite runs all cases even after a mismatch. Human and JSON reports avoid echoing fixture values. + +### Published structural schemas + +The distribution bundles JSON Schema Draft 2020-12 documents for workflow and suite +version `1`. Retrieve the exact installed versions with: + +```console +samsarix-spirals schema workflow +samsarix-spirals schema suite +``` + +The schemas intentionally cover portable structural constraints. The runtime additionally +enforces requirements JSON Schema cannot express here, including the 1 MiB byte limit, +maximum nesting and total-value budgets, unique step IDs and case names, template syntax, +default existence, and prior-step reference ordering. Passing external schema validation +does not replace `samsarix-spirals validate` or `samsarix-spirals test`. + +### JUnit reports + +`samsarix-spirals test WORKFLOW SUITE --junit` emits deterministic JUnit XML on standard +output. It includes no timestamps, durations, fixture values, or workflow outputs. Invalid +XML 1.0 characters in user-provided suite, workflow, or case names are replaced with the +Unicode replacement character so the report always remains parseable. diff --git a/pyproject.toml b/pyproject.toml index 1152ab9..3a67d66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ dev = [ "build>=1.2,<2", "mypy>=1.11,<2", "pip-audit>=2.9,<3", + "jsonschema>=4.23,<5", "pytest>=8,<9", "pytest-cov>=5,<7", "ruff>=0.8,<1", @@ -58,7 +59,7 @@ package-dir = { "" = "src" } where = ["src"] [tool.setuptools.package-data] -samsarix_spirals = ["py.typed"] +samsarix_spirals = ["py.typed", "schemas/*.json"] [tool.pytest.ini_options] addopts = "--strict-config --strict-markers -ra --cov=samsarix_spirals --cov-branch --cov-report=term-missing --cov-fail-under=95" diff --git a/src/samsarix_spirals/__init__.py b/src/samsarix_spirals/__init__.py index 10fdb5c..aa43644 100644 --- a/src/samsarix_spirals/__init__.py +++ b/src/samsarix_spirals/__init__.py @@ -5,7 +5,8 @@ from .errors import SamsarixSpiralsError, WorkflowExecutionError, WorkflowValidationError from .model import Step, Workflow, load_workflow from .runner import RunResult, StepResult, run_workflow -from .suite import SuiteResult, WorkflowSuite, load_suite, run_suite +from .schema import get_schema +from .suite import SuiteResult, WorkflowSuite, load_suite, run_suite, suite_result_to_junit_xml __version__ = "0.1.0" @@ -19,8 +20,10 @@ "WorkflowExecutionError", "WorkflowSuite", "WorkflowValidationError", + "get_schema", "load_suite", "load_workflow", "run_suite", "run_workflow", + "suite_result_to_junit_xml", ] diff --git a/src/samsarix_spirals/cli.py b/src/samsarix_spirals/cli.py index 7e0da57..c6933fd 100644 --- a/src/samsarix_spirals/cli.py +++ b/src/samsarix_spirals/cli.py @@ -22,7 +22,8 @@ parse_json_object_bytes, ) from .runner import run_workflow -from .suite import load_suite, run_suite +from .schema import SCHEMA_NAMES, get_schema +from .suite import load_suite, run_suite, suite_result_to_junit_xml STARTER_WORKFLOW: dict[str, JsonValue] = { "schema_version": 1, @@ -86,9 +87,15 @@ def build_parser() -> argparse.ArgumentParser: test = commands.add_parser("test", help="run a workflow regression suite") test.add_argument("workflow", type=Path) test.add_argument("suite", type=Path) - test.add_argument("--json", action="store_true", help="emit a machine-readable report") + report = test.add_mutually_exclusive_group() + report.add_argument("--json", action="store_true", help="emit a machine-readable report") + report.add_argument("--junit", action="store_true", help="emit deterministic JUnit XML") test.add_argument("--compact", action="store_true", help="compact the JSON report") + schema = commands.add_parser("schema", help="print a bundled JSON Schema") + schema.add_argument("kind", choices=SCHEMA_NAMES) + schema.add_argument("--compact", action="store_true", help="emit compact JSON") + init = commands.add_parser("init", help="write a starter workflow without overwriting files") init.add_argument("path", type=Path) return parser @@ -103,6 +110,12 @@ def main(argv: Sequence[str] | None = None) -> int: return 0 if args.command == "init": return _init_workflow(args.path) + if args.command == "schema": + indent = None if args.compact else 2 + print( + json.dumps(get_schema(args.kind), ensure_ascii=False, indent=indent, sort_keys=True) + ) + return 0 if args.command == "run": workflow = load_workflow(args.workflow) input_data = _load_input(args.input) @@ -116,7 +129,9 @@ def main(argv: Sequence[str] | None = None) -> int: workflow = load_workflow(args.workflow) suite = load_suite(args.suite) suite_result = run_suite(workflow, suite) - if args.json: + if args.junit: + print(suite_result_to_junit_xml(suite_result, workflow=workflow.name)) + elif args.json: indent = None if args.compact else 2 print( json.dumps( diff --git a/src/samsarix_spirals/schema.py b/src/samsarix_spirals/schema.py new file mode 100644 index 0000000..0993658 --- /dev/null +++ b/src/samsarix_spirals/schema.py @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright (c) 2026 Samsarix LLC +"""Bundled JSON Schema discovery.""" + +from __future__ import annotations + +import json +from importlib.resources import files +from typing import Literal, cast + +from .model import JsonValue + +SchemaName = Literal["suite", "workflow"] +SCHEMA_NAMES: tuple[SchemaName, ...] = ("suite", "workflow") +_SCHEMA_FILES: dict[SchemaName, str] = { + "suite": "suite-v1.schema.json", + "workflow": "workflow-v1.schema.json", +} + + +def get_schema(name: SchemaName) -> dict[str, JsonValue]: + """Return a detached bundled Draft 2020-12 schema.""" + resource = files("samsarix_spirals").joinpath("schemas", _SCHEMA_FILES[name]) + document = json.loads(resource.read_text(encoding="utf-8")) + if not isinstance(document, dict): # pragma: no cover - packaged resource invariant + raise RuntimeError(f"bundled {name} schema root is not an object") + return cast(dict[str, JsonValue], document) diff --git a/src/samsarix_spirals/schemas/suite-v1.schema.json b/src/samsarix_spirals/schemas/suite-v1.schema.json new file mode 100644 index 0000000..1755db6 --- /dev/null +++ b/src/samsarix_spirals/schemas/suite-v1.schema.json @@ -0,0 +1,115 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.samsarix.com/spirals/suite-v1.schema.json", + "title": "Samsarix Spirals regression suite version 1", + "description": "Structural schema for deterministic workflow regression suites.", + "$comment": "Runtime validation remains authoritative for byte, depth, total-value, unique-case-name, and execution constraints.", + "type": "object", + "additionalProperties": false, + "required": ["suite_version", "name", "cases"], + "properties": { + "suite_version": { + "const": 1 + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 100, + "pattern": "\\S" + }, + "cases": { + "type": "array", + "minItems": 1, + "maxItems": 1000, + "items": { + "$ref": "#/$defs/case" + } + } + }, + "$defs": { + "jsonValue": { + "oneOf": [ + { + "type": ["string", "number", "boolean", "null"], + "maxLength": 100000 + }, + { + "type": "array", + "maxItems": 10000, + "items": { + "$ref": "#/$defs/jsonValue" + } + }, + { + "type": "object", + "maxProperties": 10000, + "additionalProperties": { + "$ref": "#/$defs/jsonValue" + } + } + ] + }, + "case": { + "type": "object", + "additionalProperties": false, + "required": ["name", "expect"], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 200, + "pattern": "\\S" + }, + "input": { + "type": "object", + "maxProperties": 10000, + "additionalProperties": { + "$ref": "#/$defs/jsonValue" + } + }, + "expect": { + "oneOf": [ + { + "$ref": "#/$defs/outputExpectation" + }, + { + "$ref": "#/$defs/errorExpectation" + } + ] + } + } + }, + "outputExpectation": { + "type": "object", + "additionalProperties": false, + "required": ["output"], + "properties": { + "output": { + "$ref": "#/$defs/jsonValue" + } + } + }, + "errorExpectation": { + "type": "object", + "additionalProperties": false, + "required": ["error"], + "properties": { + "error": { + "type": "object", + "additionalProperties": false, + "properties": { + "step_id": { + "type": "string", + "minLength": 1 + }, + "message_contains": { + "type": "string", + "minLength": 1, + "maxLength": 100000 + } + } + } + } + } + } +} diff --git a/src/samsarix_spirals/schemas/workflow-v1.schema.json b/src/samsarix_spirals/schemas/workflow-v1.schema.json new file mode 100644 index 0000000..d41d08d --- /dev/null +++ b/src/samsarix_spirals/schemas/workflow-v1.schema.json @@ -0,0 +1,166 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://schemas.samsarix.com/spirals/workflow-v1.schema.json", + "title": "Samsarix Spirals workflow version 1", + "description": "Structural schema for deterministic Samsarix Spirals workflows.", + "$comment": "Runtime validation remains authoritative for byte, depth, total-value, template-reference, and unique-step-ID constraints.", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "name", "steps"], + "properties": { + "schema_version": { + "const": 1 + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 100, + "pattern": "\\S" + }, + "description": { + "type": ["string", "null"], + "maxLength": 500 + }, + "defaults": { + "type": "object", + "maxProperties": 10000, + "additionalProperties": { + "$ref": "#/$defs/jsonValue" + } + }, + "steps": { + "type": "array", + "minItems": 1, + "maxItems": 1000, + "items": { + "$ref": "#/$defs/step" + } + }, + "output": { + "$ref": "#/$defs/jsonValue" + } + }, + "$defs": { + "jsonValue": { + "oneOf": [ + { + "type": ["string", "number", "boolean", "null"], + "maxLength": 100000 + }, + { + "type": "array", + "maxItems": 10000, + "items": { + "$ref": "#/$defs/jsonValue" + } + }, + { + "type": "object", + "maxProperties": 10000, + "additionalProperties": { + "$ref": "#/$defs/jsonValue" + } + } + ] + }, + "step": { + "oneOf": [ + { + "$ref": "#/$defs/setStep" + }, + { + "$ref": "#/$defs/assertStep" + } + ] + }, + "setStep": { + "type": "object", + "additionalProperties": false, + "required": ["id", "uses", "with"], + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9_-]{0,63}$" + }, + "uses": { + "const": "set" + }, + "with": { + "type": "object", + "maxProperties": 10000, + "additionalProperties": { + "$ref": "#/$defs/jsonValue" + } + } + } + }, + "assertStep": { + "type": "object", + "additionalProperties": false, + "required": ["id", "uses", "with"], + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9_-]{0,63}$" + }, + "uses": { + "const": "assert" + }, + "with": { + "type": "object", + "additionalProperties": false, + "required": ["value", "operator"], + "properties": { + "value": { + "$ref": "#/$defs/jsonValue" + }, + "operator": { + "enum": [ + "contains", + "equals", + "falsy", + "greater_or_equal", + "greater_than", + "less_or_equal", + "less_than", + "not_empty", + "not_equals", + "truthy" + ] + }, + "expected": { + "$ref": "#/$defs/jsonValue" + }, + "message": { + "type": "string", + "maxLength": 100000 + } + }, + "allOf": [ + { + "if": { + "properties": { + "operator": { + "enum": [ + "contains", + "equals", + "greater_or_equal", + "greater_than", + "less_or_equal", + "less_than", + "not_equals" + ] + } + }, + "required": ["operator"] + }, + "then": { + "required": ["expected"] + } + } + ] + } + } + } + } +} diff --git a/src/samsarix_spirals/suite.py b/src/samsarix_spirals/suite.py index f5252ac..6bdb9af 100644 --- a/src/samsarix_spirals/suite.py +++ b/src/samsarix_spirals/suite.py @@ -9,6 +9,9 @@ from pathlib import Path from typing import cast +# ElementTree only constructs reports; production code never parses XML input. +from xml.etree import ElementTree # nosec B405 + from .errors import WorkflowExecutionError, WorkflowValidationError from .model import JsonValue, Workflow, load_json_object, validate_input_object, validate_json_value from .runner import run_workflow @@ -253,11 +256,77 @@ def _run_case(workflow: Workflow, case: SuiteCase) -> CaseResult: return CaseResult( case.name, False, "workflow completed but an execution error was expected" ) - if result.output != case.expected_output: + if not _json_equal(result.output, case.expected_output): return CaseResult(case.name, False, "workflow output did not equal expected output") return CaseResult(case.name, True) +def suite_result_to_junit_xml(result: SuiteResult, *, workflow: str) -> str: + """Serialize a non-sensitive, deterministic JUnit XML report.""" + root = ElementTree.Element( + "testsuite", + { + "name": _xml_safe(result.suite), + "tests": str(len(result.cases)), + "failures": str(result.failed), + "errors": "0", + "skipped": "0", + "time": "0", + }, + ) + classname = f"samsarix_spirals.{_xml_safe(workflow)}" + for case in result.cases: + test_case = ElementTree.SubElement( + root, + "testcase", + {"name": _xml_safe(case.name), "classname": classname, "time": "0"}, + ) + if not case.passed: + detail = _xml_safe(case.detail or "workflow contract failed") + failure = ElementTree.SubElement( + test_case, + "failure", + {"message": detail, "type": "SamsarixContractFailure"}, + ) + failure.text = detail + return ElementTree.tostring(root, encoding="unicode", xml_declaration=True) + + +def _json_equal(actual: JsonValue, expected: JsonValue) -> bool: + if isinstance(actual, bool) or isinstance(expected, bool): + return type(actual) is type(expected) and actual == expected + if isinstance(actual, (int, float)) and isinstance(expected, (int, float)): + return actual == expected + if type(actual) is not type(expected): + return False + if isinstance(actual, list) and isinstance(expected, list): + return len(actual) == len(expected) and all( + _json_equal(actual_item, expected_item) + for actual_item, expected_item in zip(actual, expected, strict=True) + ) + if isinstance(actual, dict) and isinstance(expected, dict): + return actual.keys() == expected.keys() and all( + _json_equal(actual[key], expected[key]) for key in actual + ) + return actual == expected + + +def _xml_safe(value: str) -> str: + return "".join( + character if _is_xml_character(ord(character)) else "\N{REPLACEMENT CHARACTER}" + for character in value + ) + + +def _is_xml_character(codepoint: int) -> bool: + return ( + codepoint in {0x9, 0xA, 0xD} + or 0x20 <= codepoint <= 0xD7FF + or 0xE000 <= codepoint <= 0xFFFD + or 0x10000 <= codepoint <= 0x10FFFF + ) + + def _reject_unknown_keys( value: Mapping[str, object], allowed: set[str], path: str, issues: list[str] ) -> None: diff --git a/tests/test_cli.py b/tests/test_cli.py index dd31e46..774e22c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -5,6 +5,7 @@ import io import json import sys +from xml.etree import ElementTree import pytest @@ -158,3 +159,44 @@ def test_workflow_test_command_returns_one_for_contract_failure(tmp_path, capsys captured = capsys.readouterr() assert "FAIL mismatch" in captured.out assert "1 failed" in captured.out + + +def test_schema_command_emits_bundled_schema(capsys) -> None: + assert main(["schema", "workflow", "--compact"]) == 0 + schema = json.loads(capsys.readouterr().out) + assert schema["$schema"] == "https://json-schema.org/draft/2020-12/schema" + assert schema["properties"]["schema_version"] == {"const": 1} + + +def test_workflow_test_command_emits_junit(tmp_path, capsys) -> None: + workflow = tmp_path / "workflow.json" + suite = tmp_path / "suite.json" + write_workflow(workflow) + suite.write_text( + json.dumps( + { + "suite_version": 1, + "name": "greetings", + "cases": [ + { + "name": "greets Ada", + "input": {"name": "Ada"}, + "expect": {"output": {"message": "Hello Ada"}}, + } + ], + } + ), + encoding="utf-8", + ) + + assert main(["test", str(workflow), str(suite), "--junit"]) == 0 + report = capsys.readouterr().out + root = ElementTree.fromstring(report) # noqa: S314 - parses locally generated XML + assert root.attrib["tests"] == "1" + assert root.attrib["failures"] == "0" + + +def test_test_report_formats_are_mutually_exclusive() -> None: + with pytest.raises(SystemExit) as error: + main(["test", "workflow.json", "suite.json", "--json", "--junit"]) + assert error.value.code == 2 diff --git a/tests/test_public_api.py b/tests/test_public_api.py index ef91937..385d0b7 100644 --- a/tests/test_public_api.py +++ b/tests/test_public_api.py @@ -17,8 +17,10 @@ def test_public_api_is_small_and_versioned() -> None: "WorkflowExecutionError", "WorkflowValidationError", "WorkflowSuite", + "get_schema", "load_suite", "load_workflow", "run_workflow", "run_suite", + "suite_result_to_junit_xml", } diff --git a/tests/test_schema.py b/tests/test_schema.py new file mode 100644 index 0000000..306a41a --- /dev/null +++ b/tests/test_schema.py @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright (c) 2026 Samsarix LLC +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from jsonschema import Draft202012Validator, ValidationError + +from samsarix_spirals import Workflow, WorkflowSuite, get_schema + +EXAMPLES = Path(__file__).parents[1] / "examples" + + +def test_bundled_schemas_are_valid_draft_2020_12_and_detached() -> None: + for name in ("workflow", "suite"): + schema = get_schema(name) # type: ignore[arg-type] + Draft202012Validator.check_schema(schema) + assert schema["$schema"] == "https://json-schema.org/draft/2020-12/schema" + + first = get_schema("workflow") + first["title"] = "changed" + assert get_schema("workflow")["title"] == "Samsarix Spirals workflow version 1" + + +@pytest.mark.parametrize("path", sorted(EXAMPLES.glob("*.json")), ids=lambda path: path.name) +def test_examples_match_their_published_schema(path: Path) -> None: + if path.name.endswith(".input.json"): + pytest.skip("input fixtures do not use a published document schema") + document = json.loads(path.read_text(encoding="utf-8")) + if path.name.endswith(".suite.json"): + Draft202012Validator(get_schema("suite")).validate(document) + WorkflowSuite.from_dict(document) + else: + Draft202012Validator(get_schema("workflow")).validate(document) + Workflow.from_dict(document) + + +def test_workflow_schema_requires_expected_for_comparison_operator() -> None: + document = { + "schema_version": 1, + "name": "invalid", + "steps": [ + { + "id": "comparison", + "uses": "assert", + "with": {"value": 1, "operator": "equals"}, + } + ], + } + + errors = list(Draft202012Validator(get_schema("workflow")).iter_errors(document)) + + assert errors + assert any("'expected' is a required property" in message for message in _messages(errors[0])) + + +def _messages(error: ValidationError) -> list[str]: + return [error.message, *(message for child in error.context for message in _messages(child))] diff --git a/tests/test_suite.py b/tests/test_suite.py index 172e507..8ec8a6b 100644 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +from xml.etree import ElementTree import pytest @@ -12,6 +13,7 @@ WorkflowValidationError, load_suite, run_suite, + suite_result_to_junit_xml, ) @@ -126,6 +128,96 @@ def test_reports_expectation_mismatches_without_disclosing_values() -> None: assert "Ada" not in json.dumps(result.to_dict()) +def test_exact_output_preserves_json_types_and_nested_values() -> None: + workflow = Workflow.from_dict( + { + "schema_version": 1, + "name": "json-types", + "steps": [ + { + "id": "value", + "uses": "set", + "with": {"boolean": True, "number": 1, "nested": [{"value": 2}]}, + } + ], + "output": "{{ steps.value }}", + } + ) + suite = WorkflowSuite.from_dict( + { + "suite_version": 1, + "name": "json equality", + "cases": [ + { + "name": "numbers use JSON numeric equality", + "expect": { + "output": {"boolean": True, "number": 1.0, "nested": [{"value": 2.0}]} + }, + }, + { + "name": "booleans are not numbers", + "expect": {"output": {"boolean": 1, "number": 1, "nested": [{"value": 2}]}}, + }, + ], + } + ) + + result = run_suite(workflow, suite) + + assert result.passed == 1 + assert result.failed == 1 + assert result.cases[0].passed + assert not result.cases[1].passed + + +def test_junit_report_is_deterministic_non_sensitive_and_xml_safe() -> None: + workflow = make_workflow() + suite = WorkflowSuite.from_dict( + { + "suite_version": 1, + "name": "contract \u0001", + "cases": [ + { + "name": "passes & stays private", + "input": {"name": "Ada"}, + "expect": {"output": {"name": "Ada"}}, + }, + { + "name": "fails ", + "input": {"name": "Grace"}, + "expect": {"output": {"name": "wrong"}}, + }, + ], + } + ) + result = run_suite(workflow, suite) + + report = suite_result_to_junit_xml(result, workflow="contract<&") + + assert report == suite_result_to_junit_xml(result, workflow="contract<&") + assert "Ada" not in report + assert "Grace" not in report + root = ElementTree.fromstring(report) # noqa: S314 - parses locally generated XML + assert root.tag == "testsuite" + assert root.attrib == { + "name": "contract \N{REPLACEMENT CHARACTER}", + "tests": "2", + "failures": "1", + "errors": "0", + "skipped": "0", + "time": "0", + } + cases = root.findall("testcase") + assert [case.attrib["name"] for case in cases] == [ + "passes & stays private", + "fails ", + ] + failure = cases[1].find("failure") + assert failure is not None + assert failure.attrib["type"] == "SamsarixContractFailure" + assert failure.text == "workflow output did not equal expected output" + + @pytest.mark.parametrize( "document,message", [