From b18528ea6dbf98f813cdb583134afbc33002ce6a Mon Sep 17 00:00:00 2001 From: Bruno Vicco Date: Sun, 6 Sep 2026 01:52:25 -0300 Subject: [PATCH 1/4] test: add cross-framework executor failure conformance --- ...t_governed_action_framework_conformance.py | 75 ++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_governed_action_framework_conformance.py b/tests/integration/test_governed_action_framework_conformance.py index cefd46a..15bee5a 100644 --- a/tests/integration/test_governed_action_framework_conformance.py +++ b/tests/integration/test_governed_action_framework_conformance.py @@ -31,6 +31,8 @@ ) from agentic_lab.application.action_runtime import ( ActionExecutionEvidence, + ActionExecutor, + GovernedActionExecutionError, GovernedActionRuntime, ) @@ -39,6 +41,7 @@ _APPROVED_AT = datetime(2026, 9, 5, 20, 0, tzinfo=UTC) _NOW = _APPROVED_AT + timedelta(minutes=5) _EXPIRES_AT = _APPROVED_AT + timedelta(minutes=15) +_RAW_EXECUTOR_ERROR = "synthetic cross-framework executor failure" _RULES: dict[ActionAuthorizationRuleKey, AuthorizationOutcome] = { ( @@ -82,6 +85,18 @@ class _Scenario: revoke_approval: bool = False +class _FailingExecutor: + """Count one mutable invocation before raising a deterministic failure.""" + + def __init__(self) -> None: + self.calls = 0 + + def execute(self, proposed_action: ProposedAction) -> None: + """Fail after the application has crossed the executor boundary.""" + self.calls += 1 + raise RuntimeError(_RAW_EXECUTOR_ERROR) + + def _action( *, resource: str = _FINDING_RESOURCE, @@ -210,7 +225,7 @@ def now(self) -> datetime: def _runtime( - executor: InMemoryFindingAcknowledgementExecutor, + executor: ActionExecutor, scenario: _Scenario, ) -> GovernedActionRuntime: approvals: tuple[HumanApprovalEvidence, ...] = () @@ -312,6 +327,30 @@ def _assert_expected_behavior( assert executor.is_acknowledged(_FINDING_RESOURCE) is scenario.expected_execution +def _assert_failure_behavior( + error: GovernedActionExecutionError, + executor: _FailingExecutor, + scenario: _Scenario, +) -> None: + evidence = error.evidence + + assert executor.calls == 1 + assert evidence.proposed_action == scenario.proposed_action + assert evidence.context == scenario.context + assert evidence.authorization.outcome == "allow" + assert evidence.authorization.reason == "explicit_allow" + assert evidence.approval_status == "not_applicable" + assert evidence.human_approval is None + assert evidence.approver_authorization is None + assert evidence.execution_attempted is True + assert evidence.failure_reason == "executor_error" + assert evidence.external_side_effect_state == "unknown" + assert _RAW_EXECUTOR_ERROR not in evidence.model_dump_json() + assert _RAW_EXECUTOR_ERROR not in str(error) + assert isinstance(error.__cause__, RuntimeError) + assert str(error.__cause__) == _RAW_EXECUTOR_ERROR + + @pytest.mark.parametrize( ("framework_name", "runner"), _FRAMEWORK_RUNNERS, @@ -341,3 +380,37 @@ def test_framework_governed_actions_match_application_baseline( assert framework_evidence == baseline_evidence, ( f"{framework_name} diverged from the application baseline for {scenario.name}" ) + + +@pytest.mark.parametrize( + ("framework_name", "runner"), + _FRAMEWORK_RUNNERS, + ids=[name for name, _ in _FRAMEWORK_RUNNERS], +) +def test_framework_executor_failures_match_application_baseline( + framework_name: str, + runner: FrameworkRunner, +) -> None: + """Require every framework to preserve application-owned executor failure provenance.""" + scenario = _SCENARIOS[0] + + baseline_executor = _FailingExecutor() + with pytest.raises(GovernedActionExecutionError) as baseline_caught: + _runtime(baseline_executor, scenario).execute( + scenario.proposed_action, + scenario.context, + ) + _assert_failure_behavior(baseline_caught.value, baseline_executor, scenario) + + framework_executor = _FailingExecutor() + with pytest.raises(GovernedActionExecutionError) as framework_caught: + runner( + _runtime(framework_executor, scenario), + scenario.context, + scenario.proposed_action, + ) + _assert_failure_behavior(framework_caught.value, framework_executor, scenario) + + assert framework_caught.value.evidence == baseline_caught.value.evidence, ( + f"{framework_name} diverged from the application failure baseline" + ) From 3f9375637b6d233623657ce637dbb5e78f0b1ff1 Mon Sep 17 00:00:00 2001 From: Bruno Vicco Date: Sun, 6 Sep 2026 01:54:10 -0300 Subject: [PATCH 2/4] chore: add guarded phase54 docs sync --- .github/workflows/phase54-doc-sync.yml | 104 +++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 .github/workflows/phase54-doc-sync.yml diff --git a/.github/workflows/phase54-doc-sync.yml b/.github/workflows/phase54-doc-sync.yml new file mode 100644 index 0000000..a1b122c --- /dev/null +++ b/.github/workflows/phase54-doc-sync.yml @@ -0,0 +1,104 @@ +name: phase54-doc-sync + +on: + push: + branches: + - test/phase-54-cross-framework-executor-failure-conformance + paths: + - .github/workflows/phase54-doc-sync.yml + +permissions: + contents: write + +jobs: + sync-docs: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - name: Update only Phase 54 documentation + shell: bash + run: | + python - <<'PY' + from pathlib import Path + + def replace_once(text: str, old: str, new: str, label: str) -> str: + count = text.count(old) + if count != 1: + raise SystemExit(f"{label}: expected exactly one match, found {count}") + return text.replace(old, new, 1) + + changelog_path = Path("CHANGELOG.md") + changelog = changelog_path.read_text() + evidence_anchor = "- Agno failure regression coverage proves a mutable executor is attempted exactly once and callers receive the original governed failure evidence with exact context/action/authorization, `execution_attempted=true`, `failure_reason=executor_error`, and `external_side_effect_state=unknown`; raw executor text remains only in the local exception cause." + changelog = replace_once( + changelog, + evidence_anchor, + evidence_anchor + "\n- Cross-framework governed-failure conformance now proves direct runtime, LangGraph, CrewAI, LlamaIndex, and Agno preserve baseline-equivalent `ActionExecutionFailureEvidence` for an authorized executor failure, with exactly one attempt and raw executor text confined to the local exception cause.", + "CHANGELOG evidence", + ) + changelog_path.write_text(changelog) + + security_path = Path("docs/security/GOVERNED_AGENT_ACTIONS.md") + security = security_path.read_text() + matrix_anchor = "| resource escalation | fail-closed `deny` | none |" + security = replace_once( + security, + matrix_anchor, + matrix_anchor + "\n| authorized executor failure | `GovernedActionExecutionError` with baseline-equivalent `ActionExecutionFailureEvidence` | external side-effect state remains `unknown`; exactly one attempt |", + "security conformance matrix", + ) + compare_block = ( + "- the complete `ActionExecutionEvidence` object;\n" + "- observable in-memory state;\n" + "- successful execution count." + ) + compare_replacement = ( + "- the complete `ActionExecutionEvidence` object for successful and pre-executor paths;\n" + "- the complete `ActionExecutionFailureEvidence` object for the authorized executor-failure path;\n" + "- observable in-memory state;\n" + "- successful execution or executor-attempt count, as applicable." + ) + security = replace_once( + security, + compare_block, + compare_replacement, + "security conformance comparisons", + ) + source_paragraph = "The identity-source mismatch scenario uses the same allowed caller and action scope but changes only the trusted context from `trusted_composition` to `api_key`. Because no API-key-specific rule exists in that fixture, the direct runtime and all four framework adapters must return `deny / no_matching_rule` with zero side effects." + failure_paragraph = ( + source_paragraph + + "\n\nThe executor-failure scenario uses the exact allowed action and a deterministic executor that raises only after invocation. The direct application runtime establishes the failure baseline, and every framework must surface `GovernedActionExecutionError` with equal structured evidence, exactly one executor attempt, and the raw executor message absent from both structured evidence and the governed error text. The raw exception remains only as the local Python cause, while `external_side_effect_state` remains `unknown`." + ) + security = replace_once( + security, + source_paragraph, + failure_paragraph, + "security failure conformance paragraph", + ) + security_path.write_text(security) + PY + - name: Guard documentation scope + shell: bash + run: | + git diff --check + mapfile -t changed < <(git diff --name-only | sort) + expected=( + CHANGELOG.md + docs/security/GOVERNED_AGENT_ACTIONS.md + ) + mapfile -t expected_sorted < <(printf '%s\n' "${expected[@]}" | sort) + if [[ "${changed[*]}" != "${expected_sorted[*]}" ]]; then + printf 'Unexpected changed files:\n%s\n' "${changed[*]}" + exit 1 + fi + - name: Commit documentation only + shell: bash + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add CHANGELOG.md docs/security/GOVERNED_AGENT_ACTIONS.md + git diff --cached --check + git commit -m "docs: document cross-framework failure conformance" + git push From 7b373d8320ff4ee455ce59978c72a14468b43da8 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 6 Sep 2026 04:54:19 +0000 Subject: [PATCH 3/4] docs: document cross-framework failure conformance --- CHANGELOG.md | 1 + docs/security/GOVERNED_AGENT_ACTIONS.md | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a53f51b..04ebd8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ - Governed trusted-composition MCP compatibility and real STDIO checks prove the same protocol-error boundary for action-level executor failure: safe `ActionExecutionFailureEvidence` remains in error `data`, the controlled failing resource shows zero observed fixture mutation without rewriting `external_side_effect_state=unknown`, and a subsequent valid mutation succeeds. - The MCP protocol guard removes this failure class from the normal model-correctable Tool-result channel; it does not prove that every host will avoid programmatic retry and does not provide idempotency, rollback, compensation, or distributed transaction semantics. - Agno failure regression coverage proves a mutable executor is attempted exactly once and callers receive the original governed failure evidence with exact context/action/authorization, `execution_attempted=true`, `failure_reason=executor_error`, and `external_side_effect_state=unknown`; raw executor text remains only in the local exception cause. +- Cross-framework governed-failure conformance now proves direct runtime, LangGraph, CrewAI, LlamaIndex, and Agno preserve baseline-equivalent `ActionExecutionFailureEvidence` for an authorized executor failure, with exactly one attempt and raw executor text confined to the local exception cause. ## 1.3.0 - 2026-09-05 diff --git a/docs/security/GOVERNED_AGENT_ACTIONS.md b/docs/security/GOVERNED_AGENT_ACTIONS.md index 5ee233f..f984926 100644 --- a/docs/security/GOVERNED_AGENT_ACTIONS.md +++ b/docs/security/GOVERNED_AGENT_ACTIONS.md @@ -483,15 +483,19 @@ The controlled matrix covers: | caller mismatch | fail-closed `deny` | none | | identity-source mismatch | fail-closed `deny` | none | | resource escalation | fail-closed `deny` | none | +| authorized executor failure | `GovernedActionExecutionError` with baseline-equivalent `ActionExecutionFailureEvidence` | external side-effect state remains `unknown`; exactly one attempt | For every framework, the suite compares: -- the complete `ActionExecutionEvidence` object; +- the complete `ActionExecutionEvidence` object for successful and pre-executor paths; +- the complete `ActionExecutionFailureEvidence` object for the authorized executor-failure path; - observable in-memory state; -- successful execution count. +- successful execution or executor-attempt count, as applicable. The identity-source mismatch scenario uses the same allowed caller and action scope but changes only the trusted context from `trusted_composition` to `api_key`. Because no API-key-specific rule exists in that fixture, the direct runtime and all four framework adapters must return `deny / no_matching_rule` with zero side effects. +The executor-failure scenario uses the exact allowed action and a deterministic executor that raises only after invocation. The direct application runtime establishes the failure baseline, and every framework must surface `GovernedActionExecutionError` with equal structured evidence, exactly one executor attempt, and the raw executor message absent from both structured evidence and the governed error text. The raw exception remains only as the local Python cause, while `external_side_effect_state` remains `unknown`. + This is **provider-free cross-framework conformance evidence**. It does not prove production security, provider behavior, remote identity propagation, or statistical performance. Relevant test: From d6a22ba4dd6a425a7014e3e370470b93db6d5524 Mon Sep 17 00:00:00 2001 From: Bruno Vicco Date: Sun, 6 Sep 2026 01:54:45 -0300 Subject: [PATCH 4/4] chore: remove temporary phase54 docs workflow --- .github/workflows/phase54-doc-sync.yml | 104 ------------------------- 1 file changed, 104 deletions(-) delete mode 100644 .github/workflows/phase54-doc-sync.yml diff --git a/.github/workflows/phase54-doc-sync.yml b/.github/workflows/phase54-doc-sync.yml deleted file mode 100644 index a1b122c..0000000 --- a/.github/workflows/phase54-doc-sync.yml +++ /dev/null @@ -1,104 +0,0 @@ -name: phase54-doc-sync - -on: - push: - branches: - - test/phase-54-cross-framework-executor-failure-conformance - paths: - - .github/workflows/phase54-doc-sync.yml - -permissions: - contents: write - -jobs: - sync-docs: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - name: Check out repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - - name: Update only Phase 54 documentation - shell: bash - run: | - python - <<'PY' - from pathlib import Path - - def replace_once(text: str, old: str, new: str, label: str) -> str: - count = text.count(old) - if count != 1: - raise SystemExit(f"{label}: expected exactly one match, found {count}") - return text.replace(old, new, 1) - - changelog_path = Path("CHANGELOG.md") - changelog = changelog_path.read_text() - evidence_anchor = "- Agno failure regression coverage proves a mutable executor is attempted exactly once and callers receive the original governed failure evidence with exact context/action/authorization, `execution_attempted=true`, `failure_reason=executor_error`, and `external_side_effect_state=unknown`; raw executor text remains only in the local exception cause." - changelog = replace_once( - changelog, - evidence_anchor, - evidence_anchor + "\n- Cross-framework governed-failure conformance now proves direct runtime, LangGraph, CrewAI, LlamaIndex, and Agno preserve baseline-equivalent `ActionExecutionFailureEvidence` for an authorized executor failure, with exactly one attempt and raw executor text confined to the local exception cause.", - "CHANGELOG evidence", - ) - changelog_path.write_text(changelog) - - security_path = Path("docs/security/GOVERNED_AGENT_ACTIONS.md") - security = security_path.read_text() - matrix_anchor = "| resource escalation | fail-closed `deny` | none |" - security = replace_once( - security, - matrix_anchor, - matrix_anchor + "\n| authorized executor failure | `GovernedActionExecutionError` with baseline-equivalent `ActionExecutionFailureEvidence` | external side-effect state remains `unknown`; exactly one attempt |", - "security conformance matrix", - ) - compare_block = ( - "- the complete `ActionExecutionEvidence` object;\n" - "- observable in-memory state;\n" - "- successful execution count." - ) - compare_replacement = ( - "- the complete `ActionExecutionEvidence` object for successful and pre-executor paths;\n" - "- the complete `ActionExecutionFailureEvidence` object for the authorized executor-failure path;\n" - "- observable in-memory state;\n" - "- successful execution or executor-attempt count, as applicable." - ) - security = replace_once( - security, - compare_block, - compare_replacement, - "security conformance comparisons", - ) - source_paragraph = "The identity-source mismatch scenario uses the same allowed caller and action scope but changes only the trusted context from `trusted_composition` to `api_key`. Because no API-key-specific rule exists in that fixture, the direct runtime and all four framework adapters must return `deny / no_matching_rule` with zero side effects." - failure_paragraph = ( - source_paragraph - + "\n\nThe executor-failure scenario uses the exact allowed action and a deterministic executor that raises only after invocation. The direct application runtime establishes the failure baseline, and every framework must surface `GovernedActionExecutionError` with equal structured evidence, exactly one executor attempt, and the raw executor message absent from both structured evidence and the governed error text. The raw exception remains only as the local Python cause, while `external_side_effect_state` remains `unknown`." - ) - security = replace_once( - security, - source_paragraph, - failure_paragraph, - "security failure conformance paragraph", - ) - security_path.write_text(security) - PY - - name: Guard documentation scope - shell: bash - run: | - git diff --check - mapfile -t changed < <(git diff --name-only | sort) - expected=( - CHANGELOG.md - docs/security/GOVERNED_AGENT_ACTIONS.md - ) - mapfile -t expected_sorted < <(printf '%s\n' "${expected[@]}" | sort) - if [[ "${changed[*]}" != "${expected_sorted[*]}" ]]; then - printf 'Unexpected changed files:\n%s\n' "${changed[*]}" - exit 1 - fi - - name: Commit documentation only - shell: bash - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add CHANGELOG.md docs/security/GOVERNED_AGENT_ACTIONS.md - git diff --cached --check - git commit -m "docs: document cross-framework failure conformance" - git push