diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a90a827..9a23a07 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,5 +61,6 @@ jobs: - run: /tmp/samsarix-spirals-smoke/bin/samsarix-spirals run examples/hello.json --input examples/hello.input.json --compact - run: /tmp/samsarix-spirals-smoke/bin/samsarix-spirals test examples/release-policy.json examples/release-policy.suite.json --json --compact - run: /tmp/samsarix-spirals-smoke/bin/samsarix-spirals test examples/release-policy.json examples/release-policy.suite.json --junit + - run: /tmp/samsarix-spirals-smoke/bin/samsarix-spirals test examples/agent-tool-result.json examples/agent-tool-result.suite.json --json --compact - run: /tmp/samsarix-spirals-smoke/bin/samsarix-spirals schema workflow --compact - run: /tmp/samsarix-spirals-smoke/bin/samsarix-spirals schema suite --compact diff --git a/CHANGELOG.md b/CHANGELOG.md index c7b8aec..99e6051 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,10 @@ use semantic versioning while the public API remains pre-1.0. - Competitive positioning, flagship use cases, and measurable adoption gates. - Bundled JSON Schema Draft 2020-12 documents with CLI and Python discovery APIs. - Deterministic, value-redacted JUnit XML reports for native CI ingestion. +- Bounded `merge` and `pick` operations for shallow object composition and explicit + top-level output allowlists. +- An agent tool-result example with approval, required-field, and adversarial extra-field + regression cases. ### Changed diff --git a/README.md b/README.md index 4bfae2c..ad6b46f 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ on PyPI, so install it from a checkout or a locally built wheel. ## What it does - Validates a versioned JSON workflow before execution. -- Runs `set` and `assert` steps in a fixed order. +- Runs `set`, `assert`, `merge`, and `pick` steps in a fixed order. - Runs checked-in suites that prove expected outputs and expected failures. - Renders values from `input`, `defaults`, and completed `steps`. - Emits deterministic JSON with no timestamps, random IDs, or hidden state. @@ -41,6 +41,7 @@ On macOS or Linux, use `.venv/bin/python` in place of `.venv\Scripts\python`. .venv\Scripts\samsarix-spirals validate examples/hello.json .venv\Scripts\samsarix-spirals run examples/hello.json --input examples/hello.input.json .venv\Scripts\samsarix-spirals test examples/release-policy.json examples/release-policy.suite.json +.venv\Scripts\samsarix-spirals test examples/agent-tool-result.json examples/agent-tool-result.suite.json .venv\Scripts\samsarix-spirals schema workflow --compact ``` @@ -82,6 +83,10 @@ samsarix-spirals test workflow.json workflow.suite.json --junit See [`examples/release-policy.suite.json`](examples/release-policy.suite.json) for a release approval gate with both successful and rejected cases. +[`examples/agent-tool-result.suite.json`](examples/agent-tool-result.suite.json) proves +that an approved agent result is enriched, restricted to an explicit key allowlist, and +rejected when required output is absent. Extra reasoning and credential-shaped fields +never reach the workflow output. ## JSON Schemas and CI reports diff --git a/ROADMAP.md b/ROADMAP.md index 17b03e3..1bcad58 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -46,12 +46,16 @@ this boundary. ### 0.3 — Useful deterministic shaping -- Add a small, orthogonal operation set for object merge, key selection, list mapping, - filtering, and string normalization without arbitrary expressions. +- [x] Add bounded shallow object merge and explicit top-level key selection without + arbitrary expressions. +- [ ] Add bounded list mapping, filtering, and string normalization without arbitrary + expressions. - Add an `explain` command that shows dependencies and referenced input paths without executing the workflow. - Define compatibility and deprecation rules for every schema-visible operation. -- Prove agent-output and repository-policy examples with adversarial fixtures. +- [x] Prove an agent-output contract with adversarial extra-field, approval, and + required-field fixtures. +- [ ] Prove a repository-policy example with adversarial fixtures. ### 0.4 — Repository adoption diff --git a/docs/WORKFLOW_FORMAT.md b/docs/WORKFLOW_FORMAT.md index b05e45a..8e4491d 100644 --- a/docs/WORKFLOW_FORMAT.md +++ b/docs/WORKFLOW_FORMAT.md @@ -44,7 +44,7 @@ at most 1 MiB, at most 20 levels deep, and contain only finite JSON numbers. Each step has exactly three fields: - `id`: starts with a letter, then contains up to 63 letters, digits, `_`, or `-`. -- `uses`: one of `set` or `assert`. +- `uses`: one of `set`, `assert`, `merge`, or `pick`. - `with`: an object containing operation arguments. IDs are unique and case-sensitive. A step can reference only earlier steps. Execution is @@ -86,6 +86,50 @@ Ordered operands must both be numbers or both strings. Booleans are not treated numbers. A successful assertion exposes `{"passed": true, "value": ...}`. A failed assertion stops the workflow and makes the CLI exit `1`. +### `merge` + +`merge` requires `objects`, which must render to an array of objects. It creates a new +object by applying those objects from left to right. When a key occurs more than once, +the value in the later object wins. + +```json +{ + "id": "enriched", + "uses": "merge", + "with": { + "objects": [ + {"source": "agent", "reviewed": true}, + "{{ input.result }}" + ] + } +} +``` + +The operation is shallow: nested objects are replaced, not recursively merged. Inputs +are copied, so later processing cannot mutate workflow defaults or prior step outputs. +The normal collection and total-value budgets apply to the result. + +### `pick` + +`pick` requires `object` and `keys`. It returns a new object containing only the named +top-level keys, in the order given by `keys`. + +```json +{ + "id": "public_result", + "uses": "pick", + "with": { + "object": "{{ steps.enriched }}", + "keys": ["ticket_id", "summary", "source"] + } +} +``` + +`keys` must render to an array of unique strings. By default, every named key is required +and a missing key fails the step. Set `required` to `false` to omit missing keys instead. +`pick` is an allowlist rather than a redaction list: every field that may leave the +workflow boundary must be named explicitly. + ## Templates Templates use `{{ reference.path }}`. Available roots are: diff --git a/examples/agent-tool-result.json b/examples/agent-tool-result.json new file mode 100644 index 0000000..25548d5 --- /dev/null +++ b/examples/agent-tool-result.json @@ -0,0 +1,48 @@ +{ + "schema_version": 1, + "name": "agent-tool-result", + "description": "Gate an untrusted agent result and emit only explicitly allowed fields.", + "defaults": { + "policy_metadata": { + "source": "agent", + "reviewed": true + } + }, + "steps": [ + { + "id": "require_approval", + "uses": "assert", + "with": { + "value": "{{ input.approved }}", + "operator": "equals", + "expected": true, + "message": "agent result requires approval" + } + }, + { + "id": "enriched", + "uses": "merge", + "with": { + "objects": [ + "{{ defaults.policy_metadata }}", + "{{ input.result }}" + ] + } + }, + { + "id": "allowlist", + "uses": "pick", + "with": { + "object": "{{ steps.enriched }}", + "keys": [ + "ticket_id", + "summary", + "priority", + "source", + "reviewed" + ] + } + } + ], + "output": "{{ steps.allowlist }}" +} diff --git a/examples/agent-tool-result.suite.json b/examples/agent-tool-result.suite.json new file mode 100644 index 0000000..c752f41 --- /dev/null +++ b/examples/agent-tool-result.suite.json @@ -0,0 +1,61 @@ +{ + "suite_version": 1, + "name": "agent tool-result contract", + "cases": [ + { + "name": "approved result is enriched and allowlisted", + "input": { + "approved": true, + "result": { + "ticket_id": "INC-42", + "summary": "Investigate latency regression", + "priority": "high", + "internal_reasoning": "must never leave the contract boundary", + "credential": "must never leave the contract boundary" + } + }, + "expect": { + "output": { + "ticket_id": "INC-42", + "summary": "Investigate latency regression", + "priority": "high", + "source": "agent", + "reviewed": true + } + } + }, + { + "name": "unapproved result is rejected", + "input": { + "approved": false, + "result": { + "ticket_id": "INC-42", + "summary": "Investigate latency regression", + "priority": "high" + } + }, + "expect": { + "error": { + "step_id": "require_approval", + "message_contains": "requires approval" + } + } + }, + { + "name": "missing required output field is rejected", + "input": { + "approved": true, + "result": { + "ticket_id": "INC-42", + "summary": "Investigate latency regression" + } + }, + "expect": { + "error": { + "step_id": "allowlist", + "message_contains": "missing required key 'priority'" + } + } + } + ] +} diff --git a/src/samsarix_spirals/model.py b/src/samsarix_spirals/model.py index c3d3181..1b556e1 100644 --- a/src/samsarix_spirals/model.py +++ b/src/samsarix_spirals/model.py @@ -29,7 +29,7 @@ STEP_ID_PATTERN = re.compile(r"^[A-Za-z][A-Za-z0-9_-]{0,63}$") TEMPLATE_PATTERN = re.compile(r"{{\s*([A-Za-z][A-Za-z0-9_-]*(?:\.[A-Za-z0-9_-]+)*)\s*}}") -SUPPORTED_OPERATIONS = frozenset({"assert", "set"}) +SUPPORTED_OPERATIONS = frozenset({"assert", "merge", "pick", "set"}) ASSERT_OPERATORS = frozenset( { "contains", @@ -147,6 +147,10 @@ def from_dict(cls, document: Mapping[str, object]) -> Workflow: if uses == "assert": _validate_assert(arguments, path, issues) + elif uses == "merge": + _validate_merge(arguments, path, issues) + elif uses == "pick": + _validate_pick(arguments, path, issues) _validate_templates(arguments, f"{path}.with", seen_ids - {step_id}, defaults, issues) steps.append(Step(id=step_id, uses=uses, arguments=arguments)) @@ -396,6 +400,54 @@ def _validate_assert(arguments: dict[str, JsonValue], path: str, issues: list[st issues.append(f"{path}.with.message must be a string") +def _validate_merge(arguments: dict[str, JsonValue], path: str, issues: list[str]) -> None: + _reject_unknown_keys(arguments, {"objects"}, f"{path}.with", issues) + if "objects" not in arguments: + issues.append(f"{path}.with.objects is required for merge") + return + objects = arguments["objects"] + if isinstance(objects, str) and TEMPLATE_PATTERN.fullmatch(objects): + return + if not isinstance(objects, list): + issues.append(f"{path}.with.objects must be an array or exact template") + return + for index, value in enumerate(objects): + if isinstance(value, dict): + continue + if isinstance(value, str) and TEMPLATE_PATTERN.fullmatch(value): + continue + issues.append(f"{path}.with.objects[{index}] must be an object or exact template") + + +def _validate_pick(arguments: dict[str, JsonValue], path: str, issues: list[str]) -> None: + _reject_unknown_keys(arguments, {"keys", "object", "required"}, f"{path}.with", issues) + if "object" not in arguments: + issues.append(f"{path}.with.object is required for pick") + else: + value = arguments["object"] + if not isinstance(value, dict) and not ( + isinstance(value, str) and TEMPLATE_PATTERN.fullmatch(value) + ): + issues.append(f"{path}.with.object must be an object or exact template") + if "keys" not in arguments: + issues.append(f"{path}.with.keys is required for pick") + else: + keys = arguments["keys"] + if isinstance(keys, str) and TEMPLATE_PATTERN.fullmatch(keys): + pass + elif not isinstance(keys, list): + issues.append(f"{path}.with.keys must be an array or exact template") + else: + for index, key in enumerate(keys): + if not isinstance(key, str): + issues.append(f"{path}.with.keys[{index}] must be a string") + required = arguments.get("required") + if required is not None and not isinstance(required, (bool, str)): + issues.append(f"{path}.with.required must be a boolean or template") + elif isinstance(required, str) and not TEMPLATE_PATTERN.fullmatch(required): + issues.append(f"{path}.with.required must be a boolean or exact template") + + def _validate_templates( value: object, path: str, diff --git a/src/samsarix_spirals/runner.py b/src/samsarix_spirals/runner.py index 7188836..b6e0090 100644 --- a/src/samsarix_spirals/runner.py +++ b/src/samsarix_spirals/runner.py @@ -108,6 +108,10 @@ def run_workflow( def _execute_step(step: Step, arguments: dict[str, JsonValue]) -> JsonValue: if step.uses == "set": return copy.deepcopy(arguments) + if step.uses == "merge": + return _merge_objects(arguments, step_id=step.id) + if step.uses == "pick": + return _pick_keys(arguments, step_id=step.id) if step.uses == "assert": value = arguments.get("value") expected = arguments.get("expected") @@ -134,6 +138,55 @@ def _execute_step(step: Step, arguments: dict[str, JsonValue]) -> JsonValue: ) +def _merge_objects(arguments: dict[str, JsonValue], *, step_id: str) -> JsonValue: + objects = arguments.get("objects") + if not isinstance(objects, list): + raise WorkflowExecutionError("merge objects must render to an array", step_id=step_id) + merged: dict[str, JsonValue] = {} + for index, value in enumerate(objects): + if not isinstance(value, dict): + raise WorkflowExecutionError( + f"merge objects[{index}] must render to an object", step_id=step_id + ) + merged.update(copy.deepcopy(value)) + if len(merged) > MAX_COLLECTION_ITEMS: # pragma: no cover - render budget is tighter + raise WorkflowExecutionError( + f"merged object exceeds the {MAX_COLLECTION_ITEMS}-key limit", step_id=step_id + ) + return merged + + +def _pick_keys(arguments: dict[str, JsonValue], *, step_id: str) -> JsonValue: + value = arguments.get("object") + keys = arguments.get("keys") + required = arguments.get("required", True) + if not isinstance(value, dict): + raise WorkflowExecutionError("pick object must render to an object", step_id=step_id) + if not isinstance(keys, list): + raise WorkflowExecutionError("pick keys must render to an array", step_id=step_id) + if not isinstance(required, bool): + raise WorkflowExecutionError("pick required must render to a boolean", step_id=step_id) + + selected: dict[str, JsonValue] = {} + seen: set[str] = set() + for index, key in enumerate(keys): + if not isinstance(key, str): + raise WorkflowExecutionError( + f"pick keys[{index}] must render to a string", step_id=step_id + ) + if key in seen: + raise WorkflowExecutionError(f"pick key {key!r} is duplicated", step_id=step_id) + seen.add(key) + if key not in value: + if required: + raise WorkflowExecutionError( + f"pick object is missing required key {key!r}", step_id=step_id + ) + continue + selected[key] = copy.deepcopy(value[key]) + return selected + + def _evaluate_assertion(value: JsonValue, operator: str, expected: JsonValue) -> bool: if operator == "equals": return value == expected diff --git a/src/samsarix_spirals/schemas/workflow-v1.schema.json b/src/samsarix_spirals/schemas/workflow-v1.schema.json index d41d08d..427355f 100644 --- a/src/samsarix_spirals/schemas/workflow-v1.schema.json +++ b/src/samsarix_spirals/schemas/workflow-v1.schema.json @@ -63,6 +63,11 @@ } ] }, + "exactTemplate": { + "type": "string", + "maxLength": 100000, + "pattern": "^\\{\\{\\s*[A-Za-z][A-Za-z0-9_-]*(?:\\.[A-Za-z0-9_-]+)*\\s*\\}\\}$" + }, "step": { "oneOf": [ { @@ -70,6 +75,12 @@ }, { "$ref": "#/$defs/assertStep" + }, + { + "$ref": "#/$defs/mergeStep" + }, + { + "$ref": "#/$defs/pickStep" } ] }, @@ -94,6 +105,112 @@ } } }, + "mergeStep": { + "type": "object", + "additionalProperties": false, + "required": ["id", "uses", "with"], + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9_-]{0,63}$" + }, + "uses": { + "const": "merge" + }, + "with": { + "type": "object", + "additionalProperties": false, + "required": ["objects"], + "properties": { + "objects": { + "oneOf": [ + { + "$ref": "#/$defs/exactTemplate" + }, + { + "type": "array", + "maxItems": 10000, + "items": { + "oneOf": [ + { + "type": "object", + "maxProperties": 10000, + "additionalProperties": { + "$ref": "#/$defs/jsonValue" + } + }, + { + "$ref": "#/$defs/exactTemplate" + } + ] + } + } + ] + } + } + } + } + }, + "pickStep": { + "type": "object", + "additionalProperties": false, + "required": ["id", "uses", "with"], + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9_-]{0,63}$" + }, + "uses": { + "const": "pick" + }, + "with": { + "type": "object", + "additionalProperties": false, + "required": ["object", "keys"], + "properties": { + "object": { + "oneOf": [ + { + "type": "object", + "maxProperties": 10000, + "additionalProperties": { + "$ref": "#/$defs/jsonValue" + } + }, + { + "$ref": "#/$defs/exactTemplate" + } + ] + }, + "keys": { + "oneOf": [ + { + "type": "array", + "maxItems": 10000, + "items": { + "type": "string", + "maxLength": 100000 + } + }, + { + "$ref": "#/$defs/exactTemplate" + } + ] + }, + "required": { + "oneOf": [ + { + "type": "boolean" + }, + { + "$ref": "#/$defs/exactTemplate" + } + ] + } + } + } + } + }, "assertStep": { "type": "object", "additionalProperties": false, diff --git a/tests/test_model.py b/tests/test_model.py index f0bbf65..ef6f5ed 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -106,6 +106,48 @@ def test_rejects_invalid_top_level_fields(change: dict[str, object], expected: s }, "message must be a string", ), + ({"id": "good", "uses": "merge", "with": {}}, "objects is required"), + ( + {"id": "good", "uses": "merge", "with": {"objects": {}}}, + "objects must be an array or exact template", + ), + ( + {"id": "good", "uses": "merge", "with": {"objects": [{}, 1]}}, + r"objects\[1\] must be an object or exact template", + ), + ( + {"id": "good", "uses": "merge", "with": {"objects": [], "extra": True}}, + "unknown field", + ), + ({"id": "good", "uses": "pick", "with": {}}, "object is required"), + ( + {"id": "good", "uses": "pick", "with": {"object": [], "keys": []}}, + "object must be an object or exact template", + ), + ( + {"id": "good", "uses": "pick", "with": {"object": {}, "keys": {}}}, + "keys must be an array or exact template", + ), + ( + {"id": "good", "uses": "pick", "with": {"object": {}, "keys": [1]}}, + r"keys\[0\] must be a string", + ), + ( + { + "id": "good", + "uses": "pick", + "with": {"object": {}, "keys": [], "required": 1}, + }, + "required must be a boolean or template", + ), + ( + { + "id": "good", + "uses": "pick", + "with": {"object": {}, "keys": [], "required": "yes"}, + }, + "required must be a boolean or exact template", + ), ], ) def test_rejects_invalid_steps(step: object, expected: str) -> None: diff --git a/tests/test_runner.py b/tests/test_runner.py index 573665b..872415b 100644 --- a/tests/test_runner.py +++ b/tests/test_runner.py @@ -202,6 +202,132 @@ def test_default_output_is_last_step_and_result_is_detached() -> None: assert first.output == {"items": [1]} +def test_merge_is_left_to_right_and_does_not_mutate_input() -> None: + workflow = workflow_with_steps( + [ + { + "id": "combined", + "uses": "merge", + "with": { + "objects": [ + "{{ input.base }}", + {"status": "reviewed", "owner": "{{ input.owner }}"}, + "{{ input.override }}", + ] + }, + } + ] + ) + input_data = { + "base": {"id": 7, "status": "draft", "nested": {"value": 1}}, + "owner": "Ada", + "override": {"status": "approved"}, + } + + result = run_workflow(workflow, input_data) + + assert result.output == { + "id": 7, + "status": "approved", + "nested": {"value": 1}, + "owner": "Ada", + } + assert input_data["base"] == {"id": 7, "status": "draft", "nested": {"value": 1}} + + +def test_pick_allowlists_keys_and_can_skip_missing_optional_keys() -> None: + required = workflow_with_steps( + [ + { + "id": "public", + "uses": "pick", + "with": { + "object": "{{ input.payload }}", + "keys": ["id", "summary"], + }, + } + ] + ) + optional = workflow_with_steps( + [ + { + "id": "public", + "uses": "pick", + "with": { + "object": "{{ input.payload }}", + "keys": ["id", "missing"], + "required": False, + }, + } + ] + ) + payload = {"id": 7, "summary": "safe", "secret": "do not expose"} + + assert run_workflow(required, {"payload": payload}).output == {"id": 7, "summary": "safe"} + assert run_workflow(optional, {"payload": payload}).output == {"id": 7} + + +@pytest.mark.parametrize( + "uses,arguments,message", + [ + ("merge", {"objects": "{{ input.objects }}"}, "must render to an array"), + ( + "merge", + {"objects": "{{ input.objects_with_scalar }}"}, + r"objects\[1\] must render to an object", + ), + ( + "pick", + {"object": "{{ input.not_object }}", "keys": []}, + "object must render to an object", + ), + ("pick", {"object": {}, "keys": "{{ input.not_keys }}"}, "keys must render to an array"), + ( + "pick", + {"object": {"id": 1}, "keys": "{{ input.non_string_keys }}"}, + r"keys\[0\] must render to a string", + ), + ("pick", {"object": {"id": 1}, "keys": ["id", "id"]}, "is duplicated"), + ("pick", {"object": {"id": 1}, "keys": ["missing"]}, "missing required key"), + ], +) +def test_object_shaping_runtime_errors( + uses: str, arguments: dict[str, object], message: str +) -> None: + workflow = workflow_with_steps([{"id": "shape", "uses": uses, "with": arguments}]) + + with pytest.raises(WorkflowExecutionError, match=message): + run_workflow( + workflow, + { + "objects": {}, + "objects_with_scalar": [{}, 1], + "not_object": [], + "not_keys": {}, + "non_string_keys": [1], + }, + ) + + +def test_pick_required_template_must_render_to_boolean() -> None: + workflow = workflow_with_steps( + [ + { + "id": "shape", + "uses": "pick", + "with": { + "object": {}, + "keys": [], + "required": "{{ input.required }}", + }, + } + ] + ) + + with pytest.raises(WorkflowExecutionError, match="must render to a boolean"): + run_workflow(workflow, {"required": "yes"}) + + def test_run_limits_and_input_validation() -> None: workflow = workflow_with_steps( [