Skip to content

Commit e516331

Browse files
fix(sessions): reject resuming a run whose accepted terminal output was not persisted (#4698)
1 parent 79d07ab commit e516331

6 files changed

Lines changed: 348 additions & 2 deletions

File tree

src/agents/result.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,9 @@ def _populate_state_from_result(
150150
state._pending_input = copy.deepcopy(source_state._pending_input)
151151
state._pending_session_write = copy.deepcopy(source_state._pending_session_write)
152152
state._current_step = source_state._current_step
153+
# A streamed result exists before its terminal append does, so a checkpoint taken from a
154+
# failed stream has to keep the fail-closed marker or it would look resumable.
155+
state._terminal_unrecoverable = source_state._terminal_unrecoverable
153156
else:
154157
state._generated_prompt_cache_key = getattr(result, "_generated_prompt_cache_key", None)
155158
state._pending_input = copy.deepcopy(getattr(result, "_pending_input_for_state", []))

src/agents/run.py

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@
7070
finalize_conversation_tracking,
7171
get_unsent_tool_call_ids_for_interrupted_state,
7272
input_guardrails_triggered,
73+
reject_unrecoverable_terminal_state,
7374
resolve_processed_response,
7475
resolve_resumed_context,
7576
resolve_trace_settings,
@@ -635,6 +636,7 @@ async def _run_impl(
635636
)
636637
context = context_wrapper.context
637638

639+
reject_unrecoverable_terminal_state(run_state)
638640
await resume_pending_session_write(run_state, session, wrapper=context_wrapper)
639641
max_turns = run_state._max_turns
640642
else:
@@ -1367,6 +1369,11 @@ def _mark_response_hooks_started() -> None:
13671369
current_agent,
13681370
run_config,
13691371
)
1372+
# The output, its guardrails, and its terminal hooks are all
1373+
# complete, so from here until the turn is persisted this run owns
1374+
# a result no resume can reproduce.
1375+
if run_state is not None:
1376+
run_state._terminal_unrecoverable = True
13701377
await save_final_turn_items_after_guardrails(
13711378
session=session,
13721379
run_state=run_state,
@@ -1377,6 +1384,10 @@ def _mark_response_hooks_started() -> None:
13771384
store=store_setting,
13781385
wrapper=context_wrapper,
13791386
)
1387+
# The append and any post-append maintenance both succeeded,
1388+
# so the turn is durable and the state is open again.
1389+
if run_state is not None:
1390+
run_state._terminal_unrecoverable = False
13801391
current_step = getattr(run_state, "_current_step", None)
13811392
approvals_from_state = approvals_from_step(current_step)
13821393
result = RunResult(
@@ -1550,7 +1561,16 @@ async def _save_max_turns_handler_output(
15501561
include_in_history=include_in_history,
15511562
)
15521563
if include_in_history and not handler_output_recorded:
1564+
# Only reachable once the handler output cleared its guardrails and
1565+
# ran its end hooks, so this append carries an accepted result like
1566+
# any other terminal one. The callback itself stays unmarked because
1567+
# finalize_max_turns_handler_output() also drives it from its
1568+
# guardrail-error path, where no output was ever accepted.
1569+
if run_state is not None:
1570+
run_state._terminal_unrecoverable = True
15531571
await _save_max_turns_handler_output([synthesized_item])
1572+
if run_state is not None:
1573+
run_state._terminal_unrecoverable = False
15541574
current_step = getattr(run_state, "_current_step", None)
15551575
approvals_from_state = approvals_from_step(current_step)
15561576
result = RunResult(
@@ -1994,6 +2014,8 @@ async def _save_max_turns_handler_output(
19942014
current_agent,
19952015
run_config,
19962016
)
2017+
if run_state is not None:
2018+
run_state._terminal_unrecoverable = True
19972019
await save_final_turn_items_after_guardrails(
19982020
session=session,
19992021
run_state=run_state,
@@ -2004,6 +2026,8 @@ async def _save_max_turns_handler_output(
20042026
store=store_setting,
20052027
wrapper=context_wrapper,
20062028
)
2029+
if run_state is not None:
2030+
run_state._terminal_unrecoverable = False
20072031

20082032
# Ensure starting_input is not None and not RunState
20092033
final_output_result_input: str | list[TResponseInputItem] = (

src/agents/run_internal/agent_runner_helpers.py

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
"build_interruption_result",
5454
"build_resumed_stream_debug_extra",
5555
"describe_run_state_step",
56+
"reject_unrecoverable_terminal_state",
5657
"ensure_context_wrapper",
5758
"finalize_conversation_tracking",
5859
"get_unsent_tool_call_ids_for_interrupted_state",
@@ -491,6 +492,22 @@ def build_interruption_result(
491492
return result
492493

493494

495+
def reject_unrecoverable_terminal_state(run_state: RunState | None) -> None:
496+
"""Fail closed when a previous run already produced a final output that cannot be reproduced.
497+
498+
The marker is set once that output, its guardrails, and its terminal hooks have completed,
499+
and is cleared only once the turn is fully persisted. In between, the run owns a result no
500+
resume can settle, so resuming would repeat the model call and the lifecycle hooks for an
501+
output the caller already received. Raised before any Session, sandbox, model, tool,
502+
guardrail, or hook work so the rejection has no side effects of its own.
503+
"""
504+
if run_state is not None and run_state._terminal_unrecoverable:
505+
raise UserError(
506+
"This RunState already produced a final output whose Session write did not "
507+
"complete, so it cannot be resumed. Start a new run instead."
508+
)
509+
510+
494511
def append_model_response_if_new(
495512
model_responses: list[ModelResponse],
496513
response: ModelResponse,

src/agents/run_internal/run_loop.py

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@
102102
apply_resumed_conversation_settings,
103103
attach_usage_to_span,
104104
get_unsent_tool_call_ids_for_interrupted_state,
105+
reject_unrecoverable_terminal_state,
105106
snapshot_usage,
106107
usage_delta,
107108
validate_output_guardrails_with_server_managed_conversation,
@@ -628,6 +629,10 @@ async def _finalize_streamed_final_output(
628629
# Saved as one ordered batch so the session mirrors the model response. Doing it in two
629630
# halves would both reorder the turn and, because the first save advances the turn's
630631
# persisted-item count, make the second one a no-op.
632+
# The output, its guardrails, and its terminal hooks are all complete, so from here until
633+
# the turn is persisted this run owns a result no resume can reproduce.
634+
if streamed_result._state is not None:
635+
streamed_result._state._terminal_unrecoverable = True
631636
if on_persisted_after_guardrails is None:
632637
await save_items(final_turn_items, response_id, store_setting)
633638
else:
@@ -640,6 +645,10 @@ async def _finalize_streamed_final_output(
640645
streamed_result.is_complete = True
641646
streamed_result._event_queue.put_nowait(QueueCompleteSentinel())
642647
return
648+
# The append and any post-append maintenance both succeeded, so the turn is durable and the
649+
# state is open again.
650+
if streamed_result._state is not None:
651+
streamed_result._state._terminal_unrecoverable = False
643652

644653
streamed_result.final_output = output
645654
if on_persisted_after_guardrails is not None:
@@ -923,6 +932,7 @@ async def start_streaming(
923932
streamed_result._reasoning_item_id_policy = resolved_reasoning_item_id_policy
924933

925934
if is_resumed_state and run_state is not None:
935+
reject_unrecoverable_terminal_state(run_state)
926936
await resume_pending_session_write(run_state, session, wrapper=context_wrapper)
927937
streamed_result._current_turn_persisted_item_count = (
928938
run_state._current_turn_persisted_item_count

src/agents/run_state.py

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,7 @@ def _default_run_state_validation_error(
227227
),
228228
"1.17": (
229229
"Persists Docker container labels and current-response generated-item ownership across "
230-
"resume flows, including pending resumed Session writes."
230+
"resume flows, including pending resumed Session writes and terminal-unrecoverable runs."
231231
),
232232
}
233233
SUPPORTED_SCHEMA_VERSIONS = frozenset(SCHEMA_VERSION_SUMMARIES)
@@ -878,6 +878,13 @@ class RunState(Generic[TContext, TAgent]):
878878
_session_write_in_progress: bool = field(default=False, repr=False)
879879
"""Live ownership guard; independent serialized copies require caller serialization."""
880880

881+
_terminal_unrecoverable: bool = field(default=False, repr=False)
882+
"""Set once a final output, its guardrails, and its terminal hooks have all completed.
883+
884+
It closes the state for the window where the run owns an accepted result that no resume can
885+
reproduce, and it is cleared only once that turn is fully persisted.
886+
"""
887+
881888
def __init__(
882889
self,
883890
context: RunContextWrapper[TContext],
@@ -920,6 +927,7 @@ def __init__(
920927
self._schema_version = CURRENT_SCHEMA_VERSION
921928
self._pending_session_write = None
922929
self._session_write_in_progress = False
930+
self._terminal_unrecoverable = False
923931
from .agent_tool_state import get_agent_tool_state_scope
924932

925933
self._agent_tool_state_scope_id = get_agent_tool_state_scope(context)
@@ -1909,6 +1917,8 @@ def to_json(
19091917
result["current_turn_persisted_item_count"] = self._current_turn_persisted_item_count
19101918
if self._pending_session_write is not None:
19111919
result["pending_session_write"] = copy.deepcopy(self._pending_session_write)
1920+
if self._terminal_unrecoverable:
1921+
result["terminal_unrecoverable"] = True
19121922
result["trace"] = self._serialize_trace_data(
19131923
include_tracing_api_key=include_tracing_api_key
19141924
)
@@ -4383,6 +4393,13 @@ async def _build_run_state_from_json(
43834393
):
43844394
raise validation_error_factory("Run state pending Session write is invalid", UserError)
43854395
state._pending_session_write = copy.deepcopy(cast(_PendingSessionWrite, pending_write))
4396+
terminal_unrecoverable = state_json.get("terminal_unrecoverable")
4397+
if terminal_unrecoverable is not None:
4398+
# An older label never wrote this marker, so honoring one would let a snapshot claim a
4399+
# resume boundary the schema it declares does not have.
4400+
if (schema_major, schema_minor) < (1, 17) or terminal_unrecoverable is not True:
4401+
raise validation_error_factory("Run state terminal marker is invalid", UserError)
4402+
state._terminal_unrecoverable = True
43864403
serialized_policy = state_json.get("reasoning_item_id_policy")
43874404
if serialized_policy in {"preserve", "omit"}:
43884405
state._reasoning_item_id_policy = cast(Literal["preserve", "omit"], serialized_policy)
@@ -5172,6 +5189,7 @@ def _clone_original_input(original_input: str | list[Any]) -> str | list[Any]:
51725189
"Run state agent not found in agent map",
51735190
"Run state pending_input must be a list",
51745191
"Run state pending Session write is invalid",
5192+
"Run state terminal marker is invalid",
51755193
"Run state references an agent identity that is not present in the restored graph",
51765194
(
51775195
"RunState context was serialized from a custom type; provide context_deserializer "

0 commit comments

Comments
 (0)