test(sessions): add Session/Memory/Summary replay consistency harness

docs/issue_89_replay_consistency_design.md

@@ -0,0 +1,9 @@
+# Issue 89 Replay Consistency Design
+
+The replay harness loads public JSONL traces and applies the same operations to InMemory and SQLite-backed Session and Memory services. Each run writes user and assistant events, tool call and tool response parts, state deltas, memory store/search operations, and summary creation through the SDK public service APIs. The resulting backend state is converted into a normalized snapshot before comparison.
+
+Normalization removes non-business noise: SDK-generated event ids, raw timestamps, and JSON field ordering are not compared directly. Event order, authors, invocation ids, content parts, state deltas, memory search results, summary anchors, and summary coverage are still compared. Backend-specific tolerances are recorded as `allowed_diffs` rather than silently ignored.
+
+Summary comparison is split into text, metadata, and coverage checks. Summary text is compared after deterministic generation. Session ownership, anchor presence, observable revision state, replacement behavior, timestamp validity, and summary-plus-recent-event context are checked separately. The current SDK does not expose a dedicated persisted `summary.version` field, so the harness interprets version as observable revision state: v1/v2 generation order, current persisted summary text, anchor ownership, and coverage/projection consistency.
+
+SQLite is the default persistent backend for light mode, so no external database is required. Redis integration is environment-gated by `TRPC_AGENT_REPLAY_REDIS_URL` and skipped otherwise. The report `session_memory_summary_diff_report.json` records each case, backend pair, field path, session id, event index or summary id, and both compared values.

session_memory_summary_diff_report.json

@@ -0,0 +1,265 @@
+{
+  "comparisons": [
+    {
+      "actual_backend": "sqlite",
+      "allowed_diffs": [
+        {
+          "allowed": true,
+          "case_id": "single_turn_text",
+          "field_path": "sessions.*.events.*.id",
+          "reason": "event ids are SDK-generated unless fixed by the replay case"
+        },
+        {
+          "allowed": true,
+          "case_id": "single_turn_text",
+          "field_path": "sessions.*.events.*.timestamp",
+          "reason": "event timestamps are normalized away as non-business fields"
+        },
+        {
+          "allowed": true,
+          "case_id": "single_turn_text",
+          "field_path": "sessions.*.summary.anchor_timestamp",
+          "reason": "summary timestamps are compared by validity and order, not cross-backend equality"
+        }
+      ],
+      "case_id": "single_turn_text",
+      "diffs": [],
+      "expected_backend": "inmemory"
+    },
+    {
+      "actual_backend": "sqlite",
+      "allowed_diffs": [
+        {
+          "allowed": true,
+          "case_id": "multi_turn_text",
+          "field_path": "sessions.*.events.*.id",
+          "reason": "event ids are SDK-generated unless fixed by the replay case"
+        },
+        {
+          "allowed": true,
+          "case_id": "multi_turn_text",
+          "field_path": "sessions.*.events.*.timestamp",
+          "reason": "event timestamps are normalized away as non-business fields"
+        },
+        {
+          "allowed": true,
+          "case_id": "multi_turn_text",
+          "field_path": "sessions.*.summary.anchor_timestamp",
+          "reason": "summary timestamps are compared by validity and order, not cross-backend equality"
+        }
+      ],
+      "case_id": "multi_turn_text",
+      "diffs": [],
+      "expected_backend": "inmemory"
+    },
+    {
+      "actual_backend": "sqlite",
+      "allowed_diffs": [
+        {
+          "allowed": true,
+          "case_id": "tool_call_response",
+          "field_path": "sessions.*.events.*.id",
+          "reason": "event ids are SDK-generated unless fixed by the replay case"
+        },
+        {
+          "allowed": true,
+          "case_id": "tool_call_response",
+          "field_path": "sessions.*.events.*.timestamp",
+          "reason": "event timestamps are normalized away as non-business fields"
+        },
+        {
+          "allowed": true,
+          "case_id": "tool_call_response",
+          "field_path": "sessions.*.summary.anchor_timestamp",
+          "reason": "summary timestamps are compared by validity and order, not cross-backend equality"
+        }
+      ],
+      "case_id": "tool_call_response",
+      "diffs": [],
+      "expected_backend": "inmemory"
+    },
+    {
+      "actual_backend": "sqlite",
+      "allowed_diffs": [
+        {
+          "allowed": true,
+          "case_id": "state_overwrite",
+          "field_path": "sessions.*.events.*.id",
+          "reason": "event ids are SDK-generated unless fixed by the replay case"
+        },
+        {
+          "allowed": true,
+          "case_id": "state_overwrite",
+          "field_path": "sessions.*.events.*.timestamp",
+          "reason": "event timestamps are normalized away as non-business fields"
+        },
+        {
+          "allowed": true,
+          "case_id": "state_overwrite",
+          "field_path": "sessions.*.summary.anchor_timestamp",
+          "reason": "summary timestamps are compared by validity and order, not cross-backend equality"
+        }
+      ],
+      "case_id": "state_overwrite",
+      "diffs": [],
+      "expected_backend": "inmemory"
+    },
+    {
+      "actual_backend": "sqlite",
+      "allowed_diffs": [
+        {
+          "allowed": true,
+          "case_id": "memory_write_read",
+          "field_path": "sessions.*.events.*.id",
+          "reason": "event ids are SDK-generated unless fixed by the replay case"
+        },
+        {
+          "allowed": true,
+          "case_id": "memory_write_read",
+          "field_path": "sessions.*.events.*.timestamp",
+          "reason": "event timestamps are normalized away as non-business fields"
+        },
+        {
+          "allowed": true,
+          "case_id": "memory_write_read",
+          "field_path": "sessions.*.summary.anchor_timestamp",
+          "reason": "summary timestamps are compared by validity and order, not cross-backend equality"
+        }
+      ],
+      "case_id": "memory_write_read",
+      "diffs": [],
+      "expected_backend": "inmemory"
+    },
+    {
+      "actual_backend": "sqlite",
+      "allowed_diffs": [
+        {
+          "allowed": true,
+          "case_id": "summary_generation",
+          "field_path": "sessions.*.events.*.id",
+          "reason": "event ids are SDK-generated unless fixed by the replay case"
+        },
+        {
+          "allowed": true,
+          "case_id": "summary_generation",
+          "field_path": "sessions.*.events.*.timestamp",
+          "reason": "event timestamps are normalized away as non-business fields"
+        },
+        {
+          "allowed": true,
+          "case_id": "summary_generation",
+          "field_path": "sessions.*.summary.anchor_timestamp",
+          "reason": "summary timestamps are compared by validity and order, not cross-backend equality"
+        }
+      ],
+      "case_id": "summary_generation",
+      "diffs": [],
+      "expected_backend": "inmemory"
+    },
+    {
+      "actual_backend": "sqlite",
+      "allowed_diffs": [
+        {
+          "allowed": true,
+          "case_id": "summary_update",
+          "field_path": "sessions.*.events.*.id",
+          "reason": "event ids are SDK-generated unless fixed by the replay case"
+        },
+        {
+          "allowed": true,
+          "case_id": "summary_update",
+          "field_path": "sessions.*.events.*.timestamp",
+          "reason": "event timestamps are normalized away as non-business fields"
+        },
+        {
+          "allowed": true,
+          "case_id": "summary_update",
+          "field_path": "sessions.*.summary.anchor_timestamp",
+          "reason": "summary timestamps are compared by validity and order, not cross-backend equality"
+        }
+      ],
+      "case_id": "summary_update",
+      "diffs": [],
+      "expected_backend": "inmemory"
+    },
+    {
+      "actual_backend": "sqlite",
+      "allowed_diffs": [
+        {
+          "allowed": true,
+          "case_id": "summary_truncation_context",
+          "field_path": "sessions.*.events.*.id",
+          "reason": "event ids are SDK-generated unless fixed by the replay case"
+        },
+        {
+          "allowed": true,
+          "case_id": "summary_truncation_context",
+          "field_path": "sessions.*.events.*.timestamp",
+          "reason": "event timestamps are normalized away as non-business fields"
+        },
+        {
+          "allowed": true,
+          "case_id": "summary_truncation_context",
+          "field_path": "sessions.*.summary.anchor_timestamp",
+          "reason": "summary timestamps are compared by validity and order, not cross-backend equality"
+        }
+      ],
+      "case_id": "summary_truncation_context",
+      "diffs": [],
+      "expected_backend": "inmemory"
+    },
+    {
+      "actual_backend": "sqlite",
+      "allowed_diffs": [
+        {
+          "allowed": true,
+          "case_id": "duplicate_write_recovery",
+          "field_path": "sessions.*.events.*.id",
+          "reason": "event ids are SDK-generated unless fixed by the replay case"
+        },
+        {
+          "allowed": true,
+          "case_id": "duplicate_write_recovery",
+          "field_path": "sessions.*.events.*.timestamp",
+          "reason": "event timestamps are normalized away as non-business fields"
+        },
+        {
+          "allowed": true,
+          "case_id": "duplicate_write_recovery",
+          "field_path": "sessions.*.summary.anchor_timestamp",
+          "reason": "summary timestamps are compared by validity and order, not cross-backend equality"
+        }
+      ],
+      "case_id": "duplicate_write_recovery",
+      "diffs": [],
+      "expected_backend": "inmemory"
+    },
+    {
+      "actual_backend": "sqlite",
+      "allowed_diffs": [
+        {
+          "allowed": true,
+          "case_id": "wrong_session_recovery",
+          "field_path": "sessions.*.events.*.id",
+          "reason": "event ids are SDK-generated unless fixed by the replay case"
+        },
+        {
+          "allowed": true,
+          "case_id": "wrong_session_recovery",
+          "field_path": "sessions.*.events.*.timestamp",
+          "reason": "event timestamps are normalized away as non-business fields"
+        },
+        {
+          "allowed": true,
+          "case_id": "wrong_session_recovery",
+          "field_path": "sessions.*.summary.anchor_timestamp",
+          "reason": "summary timestamps are compared by validity and order, not cross-backend equality"
+        }
+      ],
+      "case_id": "wrong_session_recovery",
+      "diffs": [],
+      "expected_backend": "inmemory"
+    }
+  ],
+  "generated_at": "deterministic-test-run"
+}

tests/sessions/replay_cases/issue_89_replay_cases.jsonl

@@ -0,0 +1,10 @@
+{"case_id":"single_turn_text","description":"Single user input and agent text output.","operations":[{"op":"event","author":"user","text":"hello agent"},{"op":"event","author":"assistant","text":"hello user"}]}
+{"case_id":"multi_turn_text","description":"Multiple user and assistant turns appended to one session.","operations":[{"op":"event","author":"user","text":"turn one question"},{"op":"event","author":"assistant","text":"turn one answer"},{"op":"event","author":"user","text":"turn two question"},{"op":"event","author":"assistant","text":"turn two answer"}]}
+{"case_id":"tool_call_response","description":"Assistant function_call followed by tool function_response and final text.","operations":[{"op":"event","author":"user","text":"weather in shenzhen"},{"op":"tool_call","author":"assistant","call_id":"call-weather","name":"lookup_weather","args":{"city":"shenzhen"}},{"op":"tool_response","author":"tool","call_id":"call-weather","name":"lookup_weather","response":{"forecast":"sunny","temperature_c":30}},{"op":"event","author":"assistant","text":"shenzhen is sunny at 30c"}]}
+{"case_id":"state_overwrite","description":"Repeated session state writes and overwrite of the same key.","operations":[{"op":"state","key":"topic","value":"travel"},{"op":"state","key":"topic","value":"finance"},{"op":"state","key":"step","value":2},{"op":"event","author":"assistant","text":"state updated"}]}
+{"case_id":"memory_write_read","description":"Store preference and fact memories, then search them.","operations":[{"op":"event","author":"user","text":"memory preference tea oolong"},{"op":"memory_store"},{"op":"memory_search","query":"oolong"},{"op":"event","author":"user","text":"memory fact project codename river"},{"op":"memory_store"},{"op":"memory_search","query":"river"}]}
+{"case_id":"summary_generation","description":"Long conversation creates a v1 summary.","operations":[{"op":"event","author":"user","text":"summary seed one has enough context"},{"op":"event","author":"assistant","text":"summary response one has enough context"},{"op":"event","author":"user","text":"summary seed two has enough context"},{"op":"event","author":"assistant","text":"summary response two has enough context"},{"op":"event","author":"user","text":"summary seed three has enough context"},{"op":"summary","expected_revision":"v1"}]}
+{"case_id":"summary_update","description":"A second summary generation replaces the previous observable revision.","operations":[{"op":"event","author":"user","text":"summary update one has enough context"},{"op":"event","author":"assistant","text":"summary update answer one has enough context"},{"op":"event","author":"user","text":"summary update two has enough context"},{"op":"event","author":"assistant","text":"summary update answer two has enough context"},{"op":"summary","expected_revision":"v1"},{"op":"event","author":"user","text":"summary update three has enough context"},{"op":"event","author":"assistant","text":"summary update answer three has enough context"},{"op":"event","author":"user","text":"summary update four has enough context"},{"op":"summary","expected_revision":"v2"}]}
+{"case_id":"summary_truncation_context","description":"Summary anchor plus retained events and new events restore active context.","operations":[{"op":"event","author":"user","text":"context alpha decision keep"},{"op":"event","author":"assistant","text":"context beta answer keep"},{"op":"event","author":"user","text":"context gamma followup keep"},{"op":"event","author":"assistant","text":"context delta answer keep"},{"op":"event","author":"user","text":"context epsilon recent keep"},{"op":"summary","expected_revision":"v1"},{"op":"event","author":"assistant","text":"post summary continuation"}]}
+{"case_id":"duplicate_write_recovery","description":"Repeated session and memory writes should not create replay-visible duplicates.","operations":[{"op":"event","event_id":"duplicate-event-1","author":"user","text":"duplicate recovery input"},{"op":"event","author":"assistant","text":"duplicate recovery output"},{"op":"duplicate_update"},{"op":"memory_store"},{"op":"memory_store"},{"op":"memory_search","query":"duplicate"}]}
+{"case_id":"wrong_session_recovery","description":"Two sessions keep summary ownership isolated.","operations":[{"op":"event","author":"user","text":"primary session content one"},{"op":"event","author":"assistant","text":"primary session content two"},{"op":"event","author":"user","text":"primary session content three"},{"op":"summary","expected_revision":"v1"},{"op":"other_session_event","session_id":"other-session","author":"user","text":"other session content one"},{"op":"other_session_event","session_id":"other-session","author":"assistant","text":"other session content two"},{"op":"other_session_event","session_id":"other-session","author":"user","text":"other session content three"},{"op":"other_session_summary","session_id":"other-session","expected_revision":"v1"}]}