From 7ab90c2e054ebe89d13a25243ffb4029f5eb541f Mon Sep 17 00:00:00 2001 From: Deathcharge Date: Sat, 8 Aug 2026 04:20:59 -0400 Subject: [PATCH] Add workflow contract regression suites --- .github/workflows/ci.yml | 1 + CHANGELOG.md | 7 + README.md | 39 ++++- ROADMAP.md | 96 ++++++++--- docs/COMPETITIVE_POSITIONING.md | 58 +++++++ docs/WORKFLOW_FORMAT.md | 41 +++++ examples/release-policy.json | 49 ++++++ examples/release-policy.suite.json | 50 ++++++ pyproject.toml | 4 +- src/samsarix_spirals/__init__.py | 5 + src/samsarix_spirals/cli.py | 34 +++- src/samsarix_spirals/model.py | 9 + src/samsarix_spirals/suite.py | 266 +++++++++++++++++++++++++++++ tests/test_cli.py | 51 ++++++ tests/test_public_api.py | 4 + tests/test_suite.py | 196 +++++++++++++++++++++ 16 files changed, 869 insertions(+), 41 deletions(-) create mode 100644 docs/COMPETITIVE_POSITIONING.md create mode 100644 examples/release-policy.json create mode 100644 examples/release-policy.suite.json create mode 100644 src/samsarix_spirals/suite.py create mode 100644 tests/test_suite.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 374bf7c..321e20d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,3 +59,4 @@ jobs: - run: /tmp/samsarix-spirals-smoke/bin/python -m pip install --no-deps dist/*.whl - run: /tmp/samsarix-spirals-smoke/bin/samsarix-spirals validate examples/hello.json - run: /tmp/samsarix-spirals-smoke/bin/samsarix-spirals run examples/hello.json --input examples/hello.input.json --compact + - run: /tmp/samsarix-spirals-smoke/bin/samsarix-spirals test examples/release-policy.json examples/release-policy.suite.json --json --compact diff --git a/CHANGELOG.md b/CHANGELOG.md index aeaa563..6259523 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ use semantic versioning while the public API remains pre-1.0. ## [Unreleased] +### Added + +- Versioned, bounded workflow regression suites with exact-output and expected-error + contracts through the Python API and `samsarix-spirals test` command. +- A release-policy example that demonstrates a practical CI approval gate. +- Competitive positioning, flagship use cases, and measurable adoption gates. + ### Changed - Renamed the product, distribution, import package, CLI, and public base exception from diff --git a/README.md b/README.md index 85bb46b..d70d4d0 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,9 @@ # Samsarix Spirals -Samsarix Spirals is a small, deterministic runner for JSON workflows that you can review, -test, and keep beside your code. It is aimed at local validation and data shaping—not -hosted automation, third-party integrations, or durable distributed orchestration. +Samsarix Spirals is a deterministic contract runner for JSON workflows that you can +review, regression-test, and keep beside your code. It is aimed at release gates, +agent-output contracts, configuration checks, and local data shaping—not hosted +automation or durable distributed orchestration. Version `0.1.0` is a source release candidate. The package is not currently published on PyPI, so install it from a checkout or a locally built wheel. @@ -11,13 +12,15 @@ on PyPI, so install it from a checkout or a locally built wheel. - Validates a versioned JSON workflow before execution. - Runs `set` and `assert` steps in a fixed order. +- Runs checked-in suites that prove expected outputs and expected failures. - Renders values from `input`, `defaults`, and completed `steps`. - Emits deterministic JSON with no timestamps, random IDs, or hidden state. - Performs no network requests, subprocess execution, credential storage, or imports from another Samsarix repository. -Samsarix Spirals is not a Zapier, n8n, Temporal, Prefect, or Dagster replacement. It has no -UI, server, scheduler, retries, connectors, parallelism, or persistence in this release. +Samsarix Spirals is not a Zapier, n8n, Temporal, Prefect, Dagster, Dagger, or CUE +replacement. Its advantage is a deliberately small, hermetic contract surface: no UI, +server, scheduler, connectors, code execution, network access, or persistence. ## Install from a checkout @@ -25,7 +28,7 @@ Python 3.11 or newer is required. ```console git clone https://github.com/Deathcharge/samsarix-spirals.git -cd helix-spirals +cd samsarix-spirals python -m venv .venv .venv\Scripts\python -m pip install -e . ``` @@ -37,6 +40,7 @@ On macOS or Linux, use `.venv/bin/python` in place of `.venv\Scripts\python`. ```console .venv\Scripts\samsarix-spirals validate examples/hello.json .venv\Scripts\samsarix-spirals run examples/hello.json --input examples/hello.input.json +.venv\Scripts\samsarix-spirals test examples/release-policy.json examples/release-policy.suite.json ``` The output's `output` field is: @@ -62,6 +66,21 @@ echo {"name":"Ada"} | .venv\Scripts\samsarix-spirals run examples/hello.json --i PowerShell users should prefer `'{"name":"Ada"}' | ...` so quoting is preserved. +## Regression suites + +A suite stores named inputs beside exact expected outputs or expected execution errors. +The `test` command runs every case, reports all mismatches, and exits `1` if the contract +has changed. Reports describe the mismatch without echoing input or output values, which +reduces accidental disclosure of fixture data in CI logs. + +```console +samsarix-spirals test workflow.json workflow.suite.json +samsarix-spirals test workflow.json workflow.suite.json --json --compact +``` + +See [`examples/release-policy.suite.json`](examples/release-policy.suite.json) for a +release approval gate with both successful and rejected cases. + ## Python API ```python @@ -87,6 +106,7 @@ output, and later steps do not run. ## Documentation - [Workflow format](docs/WORKFLOW_FORMAT.md) +- [Competitive position and use cases](docs/COMPETITIVE_POSITIONING.md) - [Productization record](docs/PRODUCTIZATION.md) - [Release process](docs/RELEASING.md) - [Security policy](SECURITY.md) @@ -94,14 +114,15 @@ output, and later steps do not run. ## Project status and release boundary -The 0.1 core journey—install, validate, run, inspect output—has local automated coverage. +The core journey—install, validate, run, regression-test, inspect output—has local +automated coverage. Before publishing, a maintainer still needs to observe the GitHub Actions matrix on the target commit, confirm the distribution name is still available, create the tag, and publish through an owned package index account. Those external steps are intentionally not claimed as complete here. -The GitHub repository currently retains its legacy `helix-spirals` slug. The product, -Python distribution, import package, and console command use the Samsarix name. +The repository, product, Python distribution, import package, and console command now use +the Samsarix identity. ## License diff --git a/ROADMAP.md b/ROADMAP.md index 8c9d43c..9fcb49b 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,43 +1,83 @@ # Samsarix Spirals roadmap -This roadmap separates four gates: merge, release, publication, and flagship adoption. Passing one does not imply the next. +This roadmap separates code completion, release, publication, and real adoption. Passing +one gate does not imply the next. -## Product boundary +## Product position -Portfolio role: **experiment or learning project**. Keep this as an evidence-producing experiment or reference. Promotion to a supported product requires a real consumer and a measured advantage over the simpler alternative. -Planned repository identity: `Deathcharge/samsarix-spirals` (ready-reference). +Samsarix Spirals is a **hermetic JSON workflow contract runner** for developers, platform +teams, and AI-agent builders. It should make small data-shaping and policy flows easier to +review and safer to execute than an ad hoc script, while remaining dramatically smaller +than a general orchestrator. -Current disposition: Merge as a labeled reference or experiment; do not imply production support. +The product earns its place only when all of these remain true: -## Stabilize the productized default +- a workflow and its regression suite fit naturally beside application code; +- identical workflow and input JSON values produce identical results; +- execution needs no daemon, container runtime, credentials, network, subprocess, or + dynamically loaded code; +- failure is bounded, attributable to a step, and useful in CI; +- the workflow is clearer to its owners than the equivalent bespoke script. -- Keep the default branch buildable from a clean checkout and preserve exact-head CI evidence. -- Keep Samsarix LLC branding, package identity, license metadata, and compatibility aliases internally consistent. -- Preserve the pre-productization default under a rollback ref before merging; do not delete legacy history. -- Review priority: Prove one real schema-v1 consumer. -- Review priority: otherwise tag a reviewed reference snapshot and freeze feature investment. +See [the competitive position](docs/COMPETITIVE_POSITIONING.md) for the evidence behind +this boundary. -## Release candidate +## Flagship use cases -- Define a falsifiable evaluation against a simpler baseline. -- Publish fixtures, limits, and reproducible results without overstating conclusions. -- Tag and freeze a useful reference if the experiment does not earn adoption. +1. **Release manifest gates** — validate approval and required metadata, then emit a + normalized manifest for a later publishing job. +2. **AI-output contracts** — turn untrusted structured model output into a bounded, + regression-tested deterministic value before another system consumes it. +3. **Repository policy fixtures** — keep configuration and metadata expectations in + reviewable JSON with positive and negative cases. +4. **Portable data-shaping checks** — produce the same small JSON artifact on a laptop, + pre-commit hook, and CI runner without provider-specific syntax. -Current hardening backlog: +## Milestones -- Only `set` and `assert`; many users can express the same job directly in tests or a short script. -- A new workflow schema creates another compatibility contract in an already crowded orchestration portfolio. -- No consumer, published JSON Schema, external adoption, release, or migration path from the removed prototype. -- The deletion-heavy PR needs careful legal/history review despite the superior direction. -- Repository slug remains Helix-named while distribution/import/CLI use Samsarix. +### 0.2 — Contract suites -## Samsarix adoption +- [x] Versioned suite files with named inputs. +- [x] Exact-output and expected-error assertions. +- [x] Human and machine-readable reports with CI exit behavior. +- [x] A realistic release-policy example. +- [ ] Publish JSON Schemas for workflows and suites. +- [ ] Emit JUnit XML for native CI test reporting. -- Define a public API, event, schema, artifact, or deployment contract before connecting to Samsarix Unified. -- Add a consumer-owned contract fixture covering authentication, privacy, limits, errors, and version compatibility. -- Make one implementation canonical; remove or freeze duplicate behavior only after parity and rollback are proven. -- Record an owner, support level, compatibility window, and measurable adoption signal. +### 0.3 — Useful deterministic shaping -## Completion evidence +- Add a small, orthogonal operation set for object merge, key selection, list mapping, + filtering, and string normalization without arbitrary expressions. +- Add an `explain` command that shows dependencies and referenced input paths without + executing the workflow. +- Define compatibility and deprecation rules for every schema-visible operation. +- Prove agent-output and repository-policy examples with adversarial fixtures. -A milestone is complete only when its exact commit, commands and results, artifact digest, consumer or deployment, and rollback path are recorded in a pull request or release record. README claims must not exceed that evidence. +### 0.4 — Repository adoption + +- Ship a pinned GitHub Action and documented pre-commit integration. +- Add stable SARIF or annotation output for step-scoped failures. +- Publish signed distributions and an SBOM through an owned package-index account. +- Measure startup time, maximum-memory behavior, and fixture-suite throughput. + +### 1.0 — Supported contract + +- Freeze schema version 1 and its compatibility window. +- Publish a support policy and migration fixtures for schema version changes. +- Complete an independent security review of all parsing and amplification limits. +- Provide a rollback-tested release and incident process. + +## Adoption gates + +Do not describe Samsarix Spirals as production-ready until evidence shows: + +- at least three independently owned repositories run it in CI; +- at least one flagship consumer has 30 consecutive days of successful contract checks; +- maintainers record 100 real workflow-suite executions with no nondeterministic result; +- at least one consumer demonstrates that a reviewed workflow is clearer or safer than + its prior script using a written before/after evaluation; +- release artifacts, checksums, exact-head CI, support ownership, and rollback steps are + recorded for the published version. + +If those gates are not met, keep the project an honest alpha rather than expanding into +connectors, scheduling, hosted execution, or another general orchestration platform. diff --git a/docs/COMPETITIVE_POSITIONING.md b/docs/COMPETITIVE_POSITIONING.md new file mode 100644 index 0000000..f0b489a --- /dev/null +++ b/docs/COMPETITIVE_POSITIONING.md @@ -0,0 +1,58 @@ +# Competitive position and use cases + +Research reviewed on 2026-08-08 supports a narrow product boundary for Samsarix Spirals. +The project should complement mature orchestrators and configuration languages, not +imitate their broadest capabilities. + +## What adjacent products already do well + +| Product | Established strength | Why Samsarix should not clone it | +| --- | --- | --- | +| [Temporal](https://docs.temporal.io/) | Durable application execution that resumes after infrastructure failures. | Competing requires a service, persistence model, worker lifecycle, and operational control plane. | +| [Prefect](https://docs.prefect.io/v3/concepts/tasks) | Observable Python tasks with retries, caching, concurrency, timeouts, and state. | Adding these features would erase the hermetic runner's small trust and deployment surface. | +| [Dagster](https://docs.dagster.io/) | Asset-oriented data orchestration with lineage, observability, and testability. | Samsarix has neither an asset catalog nor a data-platform control plane. | +| [Dagger](https://docs.dagger.io/) | Local-first, repeatable CI pipelines using containers, typed SDKs, caching, and traces. | Samsarix should stay useful where Docker and executable pipeline code are unnecessary or undesirable. | +| [CUE](https://cuelang.org/docs/concept/how-cue-enables-configuration/) | Expressive configuration constraints, validation, unification, and generation. | Recreating a constraint language would introduce far more semantic complexity than a reviewable step model needs. | +| [GitHub Actions](https://docs.github.com/en/actions/concepts/workflows-and-actions/workflows) | Event-triggered jobs and reusable actions on managed or self-hosted runners. | Samsarix can be one portable contract-checking step inside CI rather than another CI service. | + +## Differentiated promise + +Samsarix should optimize for a sentence a reviewer can verify: + +> This checked-in JSON turns these bounded inputs into this exact output—or this expected +> step failure—without executing code or contacting another system. + +That promise makes the product useful in security-sensitive and agentic development +loops where an orchestration platform is excessive but a shell or Python script creates +an unnecessarily broad execution surface. + +## Initial user journeys + +### Release-policy owner + +The owner checks in a workflow and positive/negative cases. Developers run the suite +locally; CI runs the same command before a separate, credentialed publish job. Samsarix +never receives publishing credentials and emits only normalized JSON. + +### AI application developer + +The developer treats structured model output as untrusted input. A workflow asserts +required policy decisions and shapes the accepted fields. Regression cases lock in known +good and known bad responses before the output reaches a side-effecting tool. + +### Platform repository maintainer + +The maintainer defines small configuration or metadata policies once, stores adversarial +fixtures beside them, and receives deterministic step-scoped failures across laptops and +CI providers. + +## Product guardrails + +- Do not add arbitrary Python, shell, JavaScript, dynamic imports, or expression `eval`. +- Keep network, filesystem writes, environment access, time, randomness, and credentials + outside the workflow runtime. +- Add new operations only when they are deterministic, bounded, composable, and supported + by a flagship fixture suite. +- Prefer machine-readable artifacts and CI integration over a hosted dashboard until + adoption proves that a control plane is necessary. +- Compare every new feature with a short script and remove it when the script is clearer. diff --git a/docs/WORKFLOW_FORMAT.md b/docs/WORKFLOW_FORMAT.md index 081ea85..7dc49f9 100644 --- a/docs/WORKFLOW_FORMAT.md +++ b/docs/WORKFLOW_FORMAT.md @@ -122,3 +122,44 @@ environment variables, and does not add time, randomness, or identifiers to resu Given the same workflow, input, and run limit, its JSON value result is the same. Object key order is not semantic. The CLI sorts keys when serializing its result. + +## Regression suite format + +The `test` command accepts a workflow and a separate suite document. Suite version `1` +has this shape: + +```json +{ + "suite_version": 1, + "name": "release contract", + "cases": [ + { + "name": "approved release", + "input": {"approved": true}, + "expect": {"output": {"publish": true}} + }, + { + "name": "unapproved release", + "input": {"approved": false}, + "expect": { + "error": { + "step_id": "require_approval", + "message_contains": "approval is required" + } + } + } + ] +} +``` + +A suite contains between 1 and 1,000 uniquely named cases. Each case has an optional +`input` object and exactly one expectation: + +- `output` compares the workflow's final JSON value using exact JSON equality; +- `error` expects execution to fail and can constrain `step_id`, `message_contains`, + both, or neither. + +Unknown fields are rejected. Suite files use the same 1 MiB, UTF-8, unique-key, +finite-number, nesting, string, collection, and total-value limits as workflow and input +documents. A suite runs all cases even after a mismatch. Human and JSON reports avoid +echoing fixture values. diff --git a/examples/release-policy.json b/examples/release-policy.json new file mode 100644 index 0000000..b7722fc --- /dev/null +++ b/examples/release-policy.json @@ -0,0 +1,49 @@ +{ + "schema_version": 1, + "name": "release-policy", + "description": "Gate and normalize release metadata before a CI publish job.", + "defaults": { + "channel": "stable" + }, + "steps": [ + { + "id": "require_version", + "uses": "assert", + "with": { + "value": "{{ input.version }}", + "operator": "not_empty", + "message": "version is required" + } + }, + { + "id": "require_commit", + "uses": "assert", + "with": { + "value": "{{ input.commit }}", + "operator": "not_empty", + "message": "commit is required" + } + }, + { + "id": "require_approval", + "uses": "assert", + "with": { + "value": "{{ input.approved }}", + "operator": "equals", + "expected": true, + "message": "release approval is required" + } + }, + { + "id": "manifest", + "uses": "set", + "with": { + "version": "{{ input.version }}", + "commit": "{{ input.commit }}", + "channel": "{{ defaults.channel }}", + "publish": true + } + } + ], + "output": "{{ steps.manifest }}" +} diff --git a/examples/release-policy.suite.json b/examples/release-policy.suite.json new file mode 100644 index 0000000..cf63976 --- /dev/null +++ b/examples/release-policy.suite.json @@ -0,0 +1,50 @@ +{ + "suite_version": 1, + "name": "release policy contract", + "cases": [ + { + "name": "approved stable release", + "input": { + "version": "1.4.0", + "commit": "abc123", + "approved": true + }, + "expect": { + "output": { + "version": "1.4.0", + "commit": "abc123", + "channel": "stable", + "publish": true + } + } + }, + { + "name": "unapproved release is rejected", + "input": { + "version": "1.4.0", + "commit": "abc123", + "approved": false + }, + "expect": { + "error": { + "step_id": "require_approval", + "message_contains": "release approval is required" + } + } + }, + { + "name": "missing version is rejected", + "input": { + "version": "", + "commit": "abc123", + "approved": true + }, + "expect": { + "error": { + "step_id": "require_version", + "message_contains": "version is required" + } + } + } + ] +} diff --git a/pyproject.toml b/pyproject.toml index 4b109ae..1152ab9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "samsarix-spirals" version = "0.1.0" -description = "A deterministic local runner for small, checked-in JSON workflows" +description = "Deterministic JSON workflow contracts for local development and CI" readme = "README.md" requires-python = ">=3.11" license = "MPL-2.0" @@ -17,7 +17,7 @@ authors = [ maintainers = [ { name = "Samsarix LLC", email = "support@samsarix.com" }, ] -keywords = ["automation", "cli", "json", "workflow"] +keywords = ["automation", "ci", "cli", "contract-testing", "json", "workflow"] classifiers = [ "Development Status :: 3 - Alpha", "Environment :: Console", diff --git a/src/samsarix_spirals/__init__.py b/src/samsarix_spirals/__init__.py index 9d438ae..10fdb5c 100644 --- a/src/samsarix_spirals/__init__.py +++ b/src/samsarix_spirals/__init__.py @@ -5,6 +5,7 @@ from .errors import SamsarixSpiralsError, WorkflowExecutionError, WorkflowValidationError from .model import Step, Workflow, load_workflow from .runner import RunResult, StepResult, run_workflow +from .suite import SuiteResult, WorkflowSuite, load_suite, run_suite __version__ = "0.1.0" @@ -13,9 +14,13 @@ "SamsarixSpiralsError", "Step", "StepResult", + "SuiteResult", "Workflow", "WorkflowExecutionError", + "WorkflowSuite", "WorkflowValidationError", + "load_suite", "load_workflow", + "run_suite", "run_workflow", ] diff --git a/src/samsarix_spirals/cli.py b/src/samsarix_spirals/cli.py index fd89218..7e0da57 100644 --- a/src/samsarix_spirals/cli.py +++ b/src/samsarix_spirals/cli.py @@ -22,6 +22,7 @@ parse_json_object_bytes, ) from .runner import run_workflow +from .suite import load_suite, run_suite STARTER_WORKFLOW: dict[str, JsonValue] = { "schema_version": 1, @@ -82,6 +83,12 @@ def build_parser() -> argparse.ArgumentParser: ) run.add_argument("--compact", action="store_true", help="emit compact JSON") + test = commands.add_parser("test", help="run a workflow regression suite") + test.add_argument("workflow", type=Path) + test.add_argument("suite", type=Path) + test.add_argument("--json", action="store_true", help="emit a machine-readable report") + test.add_argument("--compact", action="store_true", help="compact the JSON report") + init = commands.add_parser("init", help="write a starter workflow without overwriting files") init.add_argument("path", type=Path) return parser @@ -99,10 +106,33 @@ def main(argv: Sequence[str] | None = None) -> int: if args.command == "run": workflow = load_workflow(args.workflow) input_data = _load_input(args.input) - result = run_workflow(workflow, input_data, max_steps=args.max_steps) + run_result = run_workflow(workflow, input_data, max_steps=args.max_steps) indent = None if args.compact else 2 - print(json.dumps(result.to_dict(), ensure_ascii=False, indent=indent, sort_keys=True)) + print( + json.dumps(run_result.to_dict(), ensure_ascii=False, indent=indent, sort_keys=True) + ) return 0 + if args.command == "test": + workflow = load_workflow(args.workflow) + suite = load_suite(args.suite) + suite_result = run_suite(workflow, suite) + if args.json: + indent = None if args.compact else 2 + print( + json.dumps( + suite_result.to_dict(), ensure_ascii=False, indent=indent, sort_keys=True + ) + ) + else: + for case in suite_result.cases: + status = "PASS" if case.passed else "FAIL" + detail = f": {case.detail}" if case.detail else "" + print(f"{status} {case.name}{detail}") + print( + f"suite {suite_result.suite}: " + f"{suite_result.passed} passed, {suite_result.failed} failed" + ) + return 0 if suite_result.successful else 1 except WorkflowExecutionError as error: print(f"execution error: {error}", file=sys.stderr) return 1 diff --git a/src/samsarix_spirals/model.py b/src/samsarix_spirals/model.py index a6d33bc..c3d3181 100644 --- a/src/samsarix_spirals/model.py +++ b/src/samsarix_spirals/model.py @@ -235,6 +235,15 @@ def validate_input_object(document: Mapping[str, object]) -> dict[str, JsonValue return cast(dict[str, JsonValue], copy.deepcopy(dict(document))) +def validate_json_value(value: object) -> JsonValue: + """Validate and detach an API-provided JSON value.""" + issues: list[str] = [] + _validate_json_value(value, "$", issues) + if issues: + raise WorkflowValidationError(issues) + return cast(JsonValue, copy.deepcopy(value)) + + def _read_bounded(path: Path) -> bytes: try: with path.open("rb") as handle: diff --git a/src/samsarix_spirals/suite.py b/src/samsarix_spirals/suite.py new file mode 100644 index 0000000..f5252ac --- /dev/null +++ b/src/samsarix_spirals/suite.py @@ -0,0 +1,266 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright (c) 2026 Samsarix LLC +"""Bounded regression suites for deterministic workflow contracts.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import cast + +from .errors import WorkflowExecutionError, WorkflowValidationError +from .model import JsonValue, Workflow, load_json_object, validate_input_object, validate_json_value +from .runner import run_workflow + +SUITE_VERSION = 1 +MAX_SUITE_CASES = 1_000 +MAX_CASE_NAME_LENGTH = 200 + + +@dataclass(frozen=True, slots=True) +class ErrorExpectation: + """Expected execution-error details for a regression case.""" + + step_id: str | None = None + message_contains: str | None = None + + +@dataclass(frozen=True, slots=True) +class SuiteCase: + """One named input and its expected workflow outcome.""" + + name: str + input: dict[str, JsonValue] + expected_output: JsonValue + expected_error: ErrorExpectation | None + expects_output: bool + + +@dataclass(frozen=True, slots=True) +class WorkflowSuite: + """A validated version 1 workflow regression suite.""" + + suite_version: int + name: str + cases: tuple[SuiteCase, ...] + + @classmethod + def from_dict(cls, document: Mapping[str, object]) -> WorkflowSuite: + """Validate and detach a regression-suite mapping.""" + issues: list[str] = [] + _reject_unknown_keys(document, {"suite_version", "name", "cases"}, "$", issues) + + suite_version = document.get("suite_version") + if suite_version != SUITE_VERSION or isinstance(suite_version, bool): + issues.append(f"$.suite_version must be the integer {SUITE_VERSION}") + + name = document.get("name") + if not isinstance(name, str) or not name.strip(): + issues.append("$.name must be a non-empty string") + elif len(name) > 100: + issues.append("$.name must contain at most 100 characters") + + raw_cases = document.get("cases") + if not isinstance(raw_cases, Sequence) or isinstance(raw_cases, (str, bytes, bytearray)): + issues.append("$.cases must be an array") + raw_cases = [] + elif not raw_cases: + issues.append("$.cases must contain at least one case") + elif len(raw_cases) > MAX_SUITE_CASES: + issues.append(f"$.cases must contain at most {MAX_SUITE_CASES} cases") + raw_cases = [] + + cases: list[SuiteCase] = [] + seen_names: set[str] = set() + for index, raw_case in enumerate(raw_cases): + path = f"$.cases[{index}]" + if not isinstance(raw_case, Mapping): + issues.append(f"{path} must be an object") + continue + _reject_unknown_keys(raw_case, {"name", "input", "expect"}, path, issues) + + case_name = raw_case.get("name") + if not isinstance(case_name, str) or not case_name.strip(): + issues.append(f"{path}.name must be a non-empty string") + normalized_name = f"invalid-{index}" + else: + normalized_name = case_name.strip() + if len(normalized_name) > MAX_CASE_NAME_LENGTH: + issues.append( + f"{path}.name must contain at most {MAX_CASE_NAME_LENGTH} characters" + ) + if normalized_name in seen_names: + issues.append(f"{path}.name duplicates case name {normalized_name!r}") + seen_names.add(normalized_name) + + raw_input = raw_case.get("input", {}) + if not isinstance(raw_input, Mapping): + issues.append(f"{path}.input must be an object") + case_input: dict[str, JsonValue] = {} + else: + try: + case_input = validate_input_object(raw_input) + except WorkflowValidationError as error: + issues.extend(f"{path}.input: {issue}" for issue in error.issues) + case_input = {} + + raw_expect = raw_case.get("expect") + if not isinstance(raw_expect, Mapping): + issues.append(f"{path}.expect must be an object") + raw_expect = {} + _reject_unknown_keys(raw_expect, {"output", "error"}, f"{path}.expect", issues) + expects_output = "output" in raw_expect + expects_error = "error" in raw_expect + if expects_output == expects_error: + issues.append(f"{path}.expect must contain exactly one of 'output' or 'error'") + + expected_output: JsonValue = None + if expects_output: + try: + expected_output = validate_json_value(raw_expect.get("output")) + except WorkflowValidationError as error: + issues.extend(f"{path}.expect.output: {issue}" for issue in error.issues) + + expected_error: ErrorExpectation | None = None + if expects_error: + raw_error = raw_expect.get("error") + if not isinstance(raw_error, Mapping): + issues.append(f"{path}.expect.error must be an object") + else: + _reject_unknown_keys( + raw_error, + {"step_id", "message_contains"}, + f"{path}.expect.error", + issues, + ) + step_id = raw_error.get("step_id") + message_contains = raw_error.get("message_contains") + if step_id is not None and (not isinstance(step_id, str) or not step_id): + issues.append(f"{path}.expect.error.step_id must be a non-empty string") + if message_contains is not None and ( + not isinstance(message_contains, str) or not message_contains + ): + issues.append( + f"{path}.expect.error.message_contains must be a non-empty string" + ) + expected_error = ErrorExpectation( + step_id=step_id if isinstance(step_id, str) else None, + message_contains=( + message_contains if isinstance(message_contains, str) else None + ), + ) + + cases.append( + SuiteCase( + name=normalized_name, + input=case_input, + expected_output=expected_output, + expected_error=expected_error, + expects_output=expects_output, + ) + ) + + if issues: + raise WorkflowValidationError(issues) + return cls( + suite_version=SUITE_VERSION, + name=cast(str, name).strip(), + cases=tuple(cases), + ) + + +@dataclass(frozen=True, slots=True) +class CaseResult: + """The non-sensitive result of one regression case.""" + + name: str + passed: bool + detail: str | None = None + + def to_dict(self) -> dict[str, JsonValue]: + result: dict[str, JsonValue] = {"name": self.name, "passed": self.passed} + if self.detail is not None: + result["detail"] = self.detail + return result + + +@dataclass(frozen=True, slots=True) +class SuiteResult: + """Aggregate result of a complete regression suite.""" + + suite: str + cases: tuple[CaseResult, ...] + + @property + def passed(self) -> int: + return sum(case.passed for case in self.cases) + + @property + def failed(self) -> int: + return len(self.cases) - self.passed + + @property + def successful(self) -> bool: + return self.failed == 0 + + def to_dict(self) -> dict[str, JsonValue]: + return { + "suite": self.suite, + "successful": self.successful, + "passed": self.passed, + "failed": self.failed, + "cases": [case.to_dict() for case in self.cases], + } + + +def load_suite(path: str | Path) -> WorkflowSuite: + """Load a bounded regression suite from disk.""" + source = Path(path) + document = load_json_object(source) + try: + return WorkflowSuite.from_dict(document) + except WorkflowValidationError as error: + raise WorkflowValidationError(f"{source}: {issue}" for issue in error.issues) from error + + +def run_suite(workflow: Workflow, suite: WorkflowSuite) -> SuiteResult: + """Run every suite case without stopping at the first failed expectation.""" + return SuiteResult( + suite=suite.name, + cases=tuple(_run_case(workflow, case) for case in suite.cases), + ) + + +def _run_case(workflow: Workflow, case: SuiteCase) -> CaseResult: + try: + result = run_workflow(workflow, case.input) + except WorkflowExecutionError as error: + if case.expects_output: + return CaseResult(case.name, False, "workflow failed but output was expected") + expectation = case.expected_error + if expectation is None: # pragma: no cover - validated suite invariant + return CaseResult(case.name, False, "invalid error expectation") + if expectation.step_id is not None and error.step_id != expectation.step_id: + return CaseResult(case.name, False, "execution failed at an unexpected step") + if expectation.message_contains is not None and expectation.message_contains not in str( + error + ): + return CaseResult(case.name, False, "execution error did not contain expected text") + return CaseResult(case.name, True) + + if not case.expects_output: + return CaseResult( + case.name, False, "workflow completed but an execution error was expected" + ) + if result.output != case.expected_output: + return CaseResult(case.name, False, "workflow output did not equal expected output") + return CaseResult(case.name, True) + + +def _reject_unknown_keys( + value: Mapping[str, object], allowed: set[str], path: str, issues: list[str] +) -> None: + for key in value: + if key not in allowed: + issues.append(f"{path} contains unknown field {key!r}") diff --git a/tests/test_cli.py b/tests/test_cli.py index ab61986..dd31e46 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -107,3 +107,54 @@ def test_version_and_required_command(capsys) -> None: with pytest.raises(SystemExit) as missing: main([]) assert missing.value.code == 2 + + +def test_workflow_test_command_reports_human_and_json_results(tmp_path, capsys) -> None: + workflow = tmp_path / "workflow.json" + suite = tmp_path / "suite.json" + write_workflow(workflow) + suite.write_text( + json.dumps( + { + "suite_version": 1, + "name": "greetings", + "cases": [ + { + "name": "greets Ada", + "input": {"name": "Ada"}, + "expect": {"output": {"message": "Hello Ada"}}, + } + ], + } + ), + encoding="utf-8", + ) + + assert main(["test", str(workflow), str(suite)]) == 0 + assert capsys.readouterr().out == "PASS greets Ada\nsuite greetings: 1 passed, 0 failed\n" + + assert main(["test", str(workflow), str(suite), "--json", "--compact"]) == 0 + result = json.loads(capsys.readouterr().out) + assert result["successful"] is True + assert result["passed"] == 1 + + +def test_workflow_test_command_returns_one_for_contract_failure(tmp_path, capsys) -> None: + workflow = tmp_path / "workflow.json" + suite = tmp_path / "suite.json" + write_workflow(workflow) + suite.write_text( + json.dumps( + { + "suite_version": 1, + "name": "broken", + "cases": [{"name": "mismatch", "input": {"name": "Ada"}, "expect": {"output": {}}}], + } + ), + encoding="utf-8", + ) + + assert main(["test", str(workflow), str(suite)]) == 1 + captured = capsys.readouterr() + assert "FAIL mismatch" in captured.out + assert "1 failed" in captured.out diff --git a/tests/test_public_api.py b/tests/test_public_api.py index 4d71490..ef91937 100644 --- a/tests/test_public_api.py +++ b/tests/test_public_api.py @@ -12,9 +12,13 @@ def test_public_api_is_small_and_versioned() -> None: "RunResult", "Step", "StepResult", + "SuiteResult", "Workflow", "WorkflowExecutionError", "WorkflowValidationError", + "WorkflowSuite", + "load_suite", "load_workflow", "run_workflow", + "run_suite", } diff --git a/tests/test_suite.py b/tests/test_suite.py new file mode 100644 index 0000000..172e507 --- /dev/null +++ b/tests/test_suite.py @@ -0,0 +1,196 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright (c) 2026 Samsarix LLC +from __future__ import annotations + +import json + +import pytest + +from samsarix_spirals import ( + Workflow, + WorkflowSuite, + WorkflowValidationError, + load_suite, + run_suite, +) + + +def make_workflow() -> Workflow: + return Workflow.from_dict( + { + "schema_version": 1, + "name": "contract", + "steps": [ + { + "id": "required", + "uses": "assert", + "with": { + "value": "{{ input.name }}", + "operator": "not_empty", + "message": "name is required", + }, + }, + {"id": "shape", "uses": "set", "with": {"name": "{{ input.name }}"}}, + ], + "output": "{{ steps.shape }}", + } + ) + + +def make_suite() -> WorkflowSuite: + return WorkflowSuite.from_dict( + { + "suite_version": 1, + "name": "contract cases", + "cases": [ + { + "name": "shapes a name", + "input": {"name": "Ada"}, + "expect": {"output": {"name": "Ada"}}, + }, + { + "name": "rejects an empty name", + "input": {"name": ""}, + "expect": { + "error": {"step_id": "required", "message_contains": "name is required"} + }, + }, + ], + } + ) + + +def test_runs_output_and_error_contracts() -> None: + suite = make_suite() + result = run_suite(make_workflow(), suite) + + assert result.successful + assert result.passed == 2 + assert result.failed == 0 + assert result.to_dict() == { + "suite": "contract cases", + "successful": True, + "passed": 2, + "failed": 0, + "cases": [ + {"name": "shapes a name", "passed": True}, + {"name": "rejects an empty name", "passed": True}, + ], + } + + +def test_reports_expectation_mismatches_without_disclosing_values() -> None: + workflow = make_workflow() + suite = WorkflowSuite.from_dict( + { + "suite_version": 1, + "name": "mismatches", + "cases": [ + {"name": "wrong output", "input": {"name": "Ada"}, "expect": {"output": {}}}, + { + "name": "unexpected error", + "input": {"name": ""}, + "expect": {"output": {"name": ""}}, + }, + { + "name": "expected error", + "input": {"name": "Ada"}, + "expect": {"error": {}}, + }, + { + "name": "wrong step", + "input": {"name": ""}, + "expect": {"error": {"step_id": "other"}}, + }, + { + "name": "wrong message", + "input": {"name": ""}, + "expect": {"error": {"message_contains": "different"}}, + }, + ], + } + ) + + result = run_suite(workflow, suite) + + assert result.failed == 5 + assert not result.successful + details = [case.detail for case in result.cases] + assert details == [ + "workflow output did not equal expected output", + "workflow failed but output was expected", + "workflow completed but an execution error was expected", + "execution failed at an unexpected step", + "execution error did not contain expected text", + ] + assert "Ada" not in json.dumps(result.to_dict()) + + +@pytest.mark.parametrize( + "document,message", + [ + ({}, "suite_version"), + ({"suite_version": True, "name": "x", "cases": []}, "integer 1"), + ({"suite_version": 1, "name": "", "cases": []}, "non-empty string"), + ({"suite_version": 1, "name": "x", "cases": "no"}, "must be an array"), + ( + { + "suite_version": 1, + "name": "x", + "cases": [{"name": "one", "expect": {"output": 1, "error": {}}}], + }, + "exactly one", + ), + ( + { + "suite_version": 1, + "name": "x", + "cases": [ + {"name": "same", "expect": {"output": 1}}, + {"name": "same", "expect": {"output": 1}}, + ], + }, + "duplicates case name", + ), + ( + { + "suite_version": 1, + "name": "x", + "cases": [{"name": "one", "input": [], "expect": {"output": 1}}], + }, + "input must be an object", + ), + ( + { + "suite_version": 1, + "name": "x", + "cases": [{"name": "one", "expect": {"error": "bad"}}], + }, + "error must be an object", + ), + ( + { + "suite_version": 1, + "name": "x", + "cases": [ + { + "name": "one", + "expect": {"error": {"step_id": "", "message_contains": ""}}, + } + ], + }, + "non-empty string", + ), + ], +) +def test_rejects_invalid_suites(document: dict[str, object], message: str) -> None: + with pytest.raises(WorkflowValidationError, match=message): + WorkflowSuite.from_dict(document) + + +def test_load_suite_adds_source_path(tmp_path) -> None: + path = tmp_path / "invalid.suite.json" + path.write_text('{"suite_version":1,"name":"bad","cases":[]}', encoding="utf-8") + + with pytest.raises(WorkflowValidationError, match=r"invalid\.suite\.json"): + load_suite(path)