feat(model): per-attempt ModelEvent retry accounting and timing

CHANGELOG.md

@@ -1,6 +1,7 @@
 ## Unreleased
 
 - Logging: Local eval (`.eval`) and JSON (`.json`) log files are now written atomically (temp file + `fsync` + rename), preventing corruption from interrupted writes such as disk-full or process crash. (#2949)
+- Model API: `model.generate()` now emits one `ModelEvent` per attempt with per-attempt timing and retry accounting (`call_id`, `attempt`, `call_retries`, `http_retries`).
 - Checkpointing: In-sandbox restic artifacts moved from the world-listable `/opt/inspect-restic` to root-only `/root/.cache/inspect`, and egress scratch files no longer land in `/tmp`, so agents no longer see evidence of checkpointing.
 - Eval: `EvalSample` and `EvalSampleSummary` now record `turn_count` and the sample's token limit (`token_limit`, `token_limit_type`, and metered `token_limit_usage`).
 - Analysis: `samples_df` gains default `turn_count` and `token_limit_usage` columns, and `evals_df` configuration columns gain `token_limit_type`.

pyproject.toml

@@ -97,6 +97,12 @@ norecursedirs = [
 ]
 log_level = "WARNING"
 
+[tool.mutmut]
+paths_to_mutate = ["src/inspect_ai/model/_generate_accounting.py"]
+tests_dir = ["tests/model/test_generate_accounting.py"]
+pytest_add_cli_args = ["-q"]
+also_copy = ["src/inspect_ai"]
+
 [tool.mypy]
 exclude = [
   "tests/test_package",

requirements-dev.txt

@@ -8,13 +8,15 @@ google-genai>=1.69.0
 griffe>=2
 groq>=0.28.0
 huggingface_hub>=1.6.0
+hypothesis
 inspect-scout>=0.4.37
 ipython
 jsonpath-ng
 markdown
 mcp>=1.10.0
 mistralai>=2.4.5,
 moto[server]>=5.1.5
+mutmut
 mypy>=1.17.0
 nbformat
 openai>=2.45.0

src/inspect_ai/_util/retry.py

@@ -20,6 +20,7 @@ def report_http_retry(
     pausing scale-up but not triggering a cut.
     """
     from inspect_ai.log._samples import report_active_sample_retry
+    from inspect_ai.model._generate_accounting import current_model_generate_accounting
     from inspect_ai.util._concurrency import _active_controller, _request_had_retry
 
     # bump global counter
@@ -38,6 +39,11 @@ def report_http_retry(
         if controller is not None:
             controller.notify_retry(retry_after=retry_after)
 
+    # record on the per-generate accounting (separate from outer scheduled retries)
+    accounting = current_model_generate_accounting()
+    if accounting is not None:
+        accounting.record_http_retry()
+
 
 def http_retries_count() -> int:
     return _http_retries_count

src/inspect_ai/_view/inspect-openapi.json

@@ -7055,6 +7055,17 @@
       "ModelEvent": {
         "description": "Call to a language model.",
         "properties": {
+          "attempt": {
+            "anyOf": [
+              {
+                "type": "integer"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "title": "Attempt"
+          },
           "cache": {
             "anyOf": [
               {
@@ -7080,6 +7091,72 @@
               }
             ]
           },
+          "call_completed_at": {
+            "anyOf": [
+              {
+                "type": "string"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "title": "Call Completed At"
+          },
+          "call_id": {
+            "anyOf": [
+              {
+                "type": "string"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "title": "Call Id"
+          },
+          "call_retries": {
+            "anyOf": [
+              {
+                "type": "integer"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "title": "Call Retries"
+          },
+          "call_started_at": {
+            "anyOf": [
+              {
+                "type": "string"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "title": "Call Started At"
+          },
+          "call_working_start": {
+            "anyOf": [
+              {
+                "type": "number"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "title": "Call Working Start"
+          },
+          "call_working_time": {
+            "anyOf": [
+              {
+                "type": "number"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "title": "Call Working Time"
+          },
           "completed": {
             "anyOf": [
               {
@@ -7111,6 +7188,17 @@
             "title": "Event",
             "type": "string"
           },
+          "http_retries": {
+            "anyOf": [
+              {
+                "type": "integer"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "title": "Http Retries"
+          },
           "input": {
             "items": {
               "anyOf": [

src/inspect_ai/analysis/_dataframe/events/columns.py

@@ -1,11 +1,13 @@
 from datetime import datetime
-from typing import Any, Callable, Mapping, Type
+from typing import Any, Callable, Mapping, Type, cast
 
 from jsonpath_ng import JSONPath  # type: ignore
 from pydantic import JsonValue
 from typing_extensions import override
 
 from inspect_ai.event._event import Event
+from inspect_ai.event._model import ModelEvent
+from inspect_ai.event._tool import ToolEvent
 
 from ..columns import Column, ColumnType
 from .extract import (
@@ -44,6 +46,22 @@ def path_schema(self) -> Mapping[str, Any] | None:
         return None
 
 
+def _model_event_input_as_str(event: Event) -> str:
+    return model_event_input_as_str(cast(ModelEvent, event))
+
+
+def _tool_choice_as_str(event: Event) -> str:
+    return tool_choice_as_str(cast(ModelEvent, event))
+
+
+def _completion_as_str(event: Event) -> str:
+    return completion_as_str(cast(ModelEvent, event))
+
+
+def _tool_view_as_str(event: Event) -> str | None:
+    return tool_view_as_str(cast(ToolEvent, event))
+
+
 EventInfo: list[Column] = [
     EventColumn("event_id", path="uuid"),
     EventColumn("event", path="event"),
@@ -59,30 +77,48 @@ def path_schema(self) -> Mapping[str, Any] | None:
 ]
 """Event timing columns."""
 
-ModelEventColumns: list[Column] = [
-    EventColumn("model_event_model", path="model"),
-    EventColumn("model_event_role", path="role"),
-    EventColumn("model_event_input", path=model_event_input_as_str),
-    EventColumn("model_event_tools", path="tools"),
-    EventColumn("model_event_tool_choice", path=tool_choice_as_str),
-    EventColumn("model_event_config", path="config"),
-    EventColumn("model_event_usage", path="output.usage"),
-    EventColumn("model_event_time", path="output.time"),
-    EventColumn("model_event_completion", path=completion_as_str),
-    EventColumn("model_event_retries", path="retries"),
-    EventColumn("model_event_error", path="error"),
-    EventColumn("model_event_cache", path="cache"),
-    EventColumn("model_event_call", path="call"),
-]
+ModelEventColumns: list[Column] = cast(
+    list[Column],
+    [
+        EventColumn("model_event_model", path="model"),
+        EventColumn("model_event_role", path="role"),
+        EventColumn("model_event_input", path=_model_event_input_as_str),
+        EventColumn("model_event_tools", path="tools"),
+        EventColumn("model_event_tool_choice", path=_tool_choice_as_str),
+        EventColumn("model_event_config", path="config"),
+        EventColumn("model_event_usage", path="output.usage"),
+        EventColumn("model_event_time", path="output.time"),
+        EventColumn("model_event_completion", path=_completion_as_str),
+        EventColumn("model_event_retries", path="retries"),
+        EventColumn("model_event_error", path="error"),
+        EventColumn("model_event_cache", path="cache"),
+        EventColumn("model_event_call", path="call"),
+        EventColumn("model_event_call_id", path="call_id"),
+        EventColumn("model_event_attempt", path="attempt"),
+        EventColumn(
+            "model_event_call_started_at", path="call_started_at", type=datetime
+        ),
+        EventColumn(
+            "model_event_call_completed_at", path="call_completed_at", type=datetime
+        ),
+        EventColumn("model_event_call_working_start", path="call_working_start"),
+        EventColumn("model_event_call_working_time", path="call_working_time"),
+        EventColumn("model_event_call_retries", path="call_retries"),
+        EventColumn("model_event_http_retries", path="http_retries"),
+    ],
+)
 """Model event columns."""
 
-ToolEventColumns: list[Column] = [
-    EventColumn("tool_event_function", path="function"),
-    EventColumn("tool_event_arguments", path="arguments"),
-    EventColumn("tool_event_view", path=tool_view_as_str),
-    EventColumn("tool_event_result", path="result"),
-    EventColumn("tool_event_truncated", path="truncated"),
-    EventColumn("tool_event_error_type", path="error.type"),
-    EventColumn("tool_event_error_message", path="error.message"),
-]
+ToolEventColumns: list[Column] = cast(
+    list[Column],
+    [
+        EventColumn("tool_event_function", path="function"),
+        EventColumn("tool_event_arguments", path="arguments"),
+        EventColumn("tool_event_view", path=_tool_view_as_str),
+        EventColumn("tool_event_result", path="result"),
+        EventColumn("tool_event_truncated", path="truncated"),
+        EventColumn("tool_event_error_type", path="error.type"),
+        EventColumn("tool_event_error_message", path="error.message"),
+    ],
+)
 """Tool event columns."""

src/inspect_ai/event/_model.py

@@ -82,7 +82,11 @@ class ModelEvent(BaseEvent):
     """Output from model."""
 
     retries: int | None = Field(default=None)
-    """Retries for the model API request."""
+    """Legacy retry count.
+
+    On terminal post-fix events, mirrors ``http_retries`` if non-zero, else
+    ``call_retries``. Prefer ``call_retries`` and ``http_retries`` for new code.
+    """
 
     error: str | None = Field(default=None)
     """Error which occurred during model call."""
@@ -100,13 +104,49 @@ class ModelEvent(BaseEvent):
     """Raw call made to model API."""
 
     completed: UtcDatetime | None = Field(default=None)
-    """Time that model call completed (see `timestamp` for started)"""
+    """Time this attempt completed."""
 
     working_time: float | None = Field(default=None)
-    """working time for model call that succeeded (i.e. was not retried)."""
+    """Working-clock duration of this attempt."""
+
+    call_id: str | None = Field(default=None)
+    """Stable id shared by all ModelEvents from one logical generate call."""
+
+    attempt: int | None = Field(default=None)
+    """1-based outer-attempt number within ``call_id``. None on legacy logs."""
+
+    call_started_at: UtcDatetime | None = Field(default=None)
+    """Wall-clock time the logical generate call began. Terminal event only."""
+
+    call_completed_at: UtcDatetime | None = Field(default=None)
+    """Wall-clock time the logical generate call completed. Terminal event only."""
+
+    call_working_start: float | None = Field(default=None)
+    """Sample working-clock value when the logical generate call began."""
+
+    call_working_time: float | None = Field(default=None)
+    """Logical generate working-clock duration. Terminal event only."""
+
+    call_retries: int | None = Field(default=None)
+    """Number of outer Tenacity retries actually scheduled. Terminal event only."""
+
+    http_retries: int | None = Field(default=None)
+    """Total retry signals reported via ``report_http_retry``. Terminal event only."""
 
     @field_serializer("completed")
     def serialize_completed(self, dt: datetime | None) -> str | None:
         if dt is None:
             return None
         return datetime_to_iso_format_safe(dt)
+
+    @field_serializer("call_started_at")
+    def serialize_call_started_at(self, dt: datetime | None) -> str | None:
+        if dt is None:
+            return None
+        return datetime_to_iso_format_safe(dt)
+
+    @field_serializer("call_completed_at")
+    def serialize_call_completed_at(self, dt: datetime | None) -> str | None:
+        if dt is None:
+            return None
+        return datetime_to_iso_format_safe(dt)