diff --git a/agent/db.py b/agent/db.py index e41af26..c3f21ad 100644 --- a/agent/db.py +++ b/agent/db.py @@ -119,6 +119,37 @@ def _row(r: dict) -> dict: ) """) +# Append-only usage log for feature/command adoption reporting (see +# scripts/usage_report.py) — deliberately no event payload beyond a +# structural name/source, never message content or expense/income +# descriptions, since this is financial household data. +_run(""" + CREATE TABLE IF NOT EXISTS usage_events ( + id SERIAL PRIMARY KEY, + user_id INTEGER REFERENCES users(id), + event_type TEXT NOT NULL, + event_name TEXT NOT NULL, + source TEXT, + created_at TIMESTAMPTZ DEFAULT NOW() + ) +""") + +_run("CREATE INDEX IF NOT EXISTS usage_events_name_idx ON usage_events (event_name)") + +# Chat session history, externalized from the in-process dict it used to +# live in (agent/main.py's old _sessions) — that dict silently dropped every +# in-progress conversation on every redeploy/restart, since nothing survived +# process memory. One row per user; the whole message list is replaced on +# every save rather than appended in SQL, since the caller always has the +# full up-to-date list in hand already. +_run(""" + CREATE TABLE IF NOT EXISTS chat_sessions ( + user_id INTEGER PRIMARY KEY REFERENCES users(id), + messages JSONB NOT NULL DEFAULT '[]', + updated_at TIMESTAMPTZ DEFAULT NOW() + ) +""") + def get_user_by_username(username: str) -> dict | None: cur = _run("SELECT * FROM users WHERE username = %s", (username,)) @@ -898,3 +929,61 @@ def increment_api_call_count(user_id: int, date: str) -> None: """, (user_id, date), ) + + +def record_usage(user_id: int | None, event_type: str, event_name: str, source: str | None = None) -> None: + """Best-effort usage logging — never let a logging failure break the + actual request it's attached to (an edit/delete/tool-call succeeding + is what matters; the analytics record is secondary).""" + try: + _run( + "INSERT INTO usage_events (user_id, event_type, event_name, source) VALUES (%s, %s, %s, %s)", + (user_id, event_type, event_name, source), + ) + except Exception: + pass + + +def load_chat_session(user_id: int) -> list: + cur = _run("SELECT messages FROM chat_sessions WHERE user_id = %s", (user_id,)) + row = cur.fetchone() + return row["messages"] if row else [] + + +def save_chat_session(user_id: int, messages: list) -> None: + _run( + """ + INSERT INTO chat_sessions (user_id, messages, updated_at) VALUES (%s, %s, NOW()) + ON CONFLICT (user_id) DO UPDATE SET messages = EXCLUDED.messages, updated_at = NOW() + """, + (user_id, psycopg2.extras.Json(messages)), + ) + + +def clear_chat_session(user_id: int) -> None: + _run("DELETE FROM chat_sessions WHERE user_id = %s", (user_id,)) + + +def get_usage_summary(since: str | None = None) -> list[dict]: + """Event counts grouped by type/name/source, for scripts/usage_report.py.""" + if since: + cur = _run( + """ + SELECT event_type, event_name, source, COUNT(*) AS count + FROM usage_events + WHERE created_at >= %s + GROUP BY event_type, event_name, source + ORDER BY count DESC + """, + (since,), + ) + else: + cur = _run( + """ + SELECT event_type, event_name, source, COUNT(*) AS count + FROM usage_events + GROUP BY event_type, event_name, source + ORDER BY count DESC + """ + ) + return [dict(r) for r in cur.fetchall()] diff --git a/agent/main.py b/agent/main.py index 1c60167..8415923 100644 --- a/agent/main.py +++ b/agent/main.py @@ -1,11 +1,13 @@ import logging import os +import threading import traceback from datetime import date import anthropic from dotenv import load_dotenv +from agent import db from agent.categories import CATEGORY_HINTS, INCOME_CATEGORY_HINTS from agent.tools import TOOL_DEFINITIONS, TOOL_HANDLERS @@ -75,8 +77,26 @@ {category_hints}""" -# Conversation history keyed by user_id. -_sessions: dict[str, list] = {} +# Conversation history is persisted in Postgres (agent/db.py's chat_sessions +# table), not kept in process memory — an in-memory dict here used to drop +# every in-progress conversation on every redeploy/restart. One lock per +# user_id serializes that user's own concurrent requests (e.g. a double-tap, +# or two tabs) around the load -> mutate -> save round trip, so one request's +# save can't silently clobber another's; this only guards same-instance +# concurrency, which is the actual deployment today — true cross-instance +# locking would need a DB-level lock and isn't worth it at this scale yet. +_session_locks: dict[str, threading.Lock] = {} +_session_locks_guard = threading.Lock() + + +def _get_session_lock(user_id: int) -> threading.Lock: + key = str(user_id) + with _session_locks_guard: + if key not in _session_locks: + _session_locks[key] = threading.Lock() + return _session_locks[key] + + HISTORY_LIMIT = 30 # max messages passed to the API per turn _SESSION_CAP = 120 # trim stored history once it exceeds this to prevent unbounded growth API_TIMEOUT = 120.0 # seconds before giving up on a Claude API call @@ -120,20 +140,35 @@ def _trim_session(messages: list) -> None: messages[:] = messages[-HISTORY_LIMIT:] +def _serialize_block(block) -> dict: + """Normalize one response content block to a plain JSON-serializable + dict. In production this is always an Anthropic SDK pydantic object + (has model_dump()); test doubles use plain dicts or SimpleNamespace. + Required both to persist assistant turns to Postgres and to keep + _repair_dangling_tool_use's dict-style access correct now that history + can come from a DB reload rather than always being this process's own + freshly-returned API response.""" + if isinstance(block, dict): + return block + if hasattr(block, "model_dump"): + return block.model_dump() + return dict(vars(block)) + + def _repair_dangling_tool_use(messages: list) -> None: """Drop a trailing assistant tool_use turn that never got its tool_result appended. This can only happen from a session that was corrupted by a since-fixed bug (a tool handler exception used to abort _run_tools before - it appended the matching tool_result) — but since _sessions lives only in - memory, any session already in that broken state stays broken forever - without this: every future turn would keep failing the same way, since + it appended the matching tool_result) — but since sessions are persisted, + any session already in that broken state stays broken forever without + this: every future turn would keep failing the same way, since Anthropic's API rejects a tool_use block with no following tool_result. Safe going forward too, in case some other future bug reintroduces the same failure mode.""" if not messages: return last = messages[-1] - if last.get("role") == "assistant" and any(getattr(b, "type", None) == "tool_use" for b in last.get("content", [])): + if last.get("role") == "assistant" and any(b.get("type") == "tool_use" for b in last.get("content", [])): messages.pop() @@ -192,7 +227,7 @@ def _build_user_content(user_input: str, images: list[dict] | None) -> str: return "\n\n".join(parts) -def _run_tools(response_content: list, user_id: int, on_result=None) -> list: +def _run_tools(response_content: list, user_id: int, on_result=None, source: str | None = None) -> list: # Every tool_use block below MUST produce a tool_result, even if the # handler raises. The assistant message containing these tool_use blocks # is already appended to the persistent session history by the caller @@ -215,6 +250,7 @@ def _run_tools(response_content: list, user_id: int, on_result=None) -> list: except Exception: logger.error("tool %s(%s) failed:\n%s", block.name, kwargs, traceback.format_exc()) result = {"status": "error", "message": f"{block.name} failed unexpectedly — tell the user and don't retry automatically."} + db.record_usage(user_id, "tool", block.name, source) print(f"[tool] {block.name}({kwargs}) -> {result}") if on_result: on_result(block.name, result) @@ -227,102 +263,110 @@ def _run_tools(response_content: list, user_id: int, on_result=None) -> list: def clear_session(user_id: int) -> None: - _sessions.pop(str(user_id), None) - - -def chat(user_input: str, user_id: int, username: str = "user", images: list[dict] | None = None) -> str: - messages = _sessions.setdefault(str(user_id), []) - _repair_dangling_tool_use(messages) - content = _build_user_content(user_input, images) - _append_user_turn(messages, content) - _trim_session(messages) - - while True: - response = client.messages.create( - model=MODEL_DEFAULT, - max_tokens=2048, - timeout=API_TIMEOUT, - system=SYSTEM.format( - today=date.today().isoformat(), - username=username, - category_hints=CATEGORY_HINTS, - income_category_hints=INCOME_CATEGORY_HINTS, - ), - tools=TOOL_DEFINITIONS, - messages=messages[-HISTORY_LIMIT:], - ) - - messages.append({"role": "assistant", "content": response.content}) - - if response.stop_reason == "end_turn": - for block in response.content: - if hasattr(block, "text"): - return block.text - - if response.stop_reason == "tool_use": - tool_results = _run_tools(response.content, user_id) - messages.append({"role": "user", "content": tool_results}) - - -def stream_chat(user_input: str, user_id: int, username: str = "user", images: list[dict] | None = None): - messages = _sessions.setdefault(str(user_id), []) - _repair_dangling_tool_use(messages) - if images: - yield {"status": "Reading image…" if len(images) == 1 else "Reading images…"} - content = _build_user_content(user_input, images) - _append_user_turn(messages, content) - _trim_session(messages) - - last_char = "" - while True: - with client.messages.stream( - model=MODEL_DEFAULT, - max_tokens=2048, - timeout=API_TIMEOUT, - system=SYSTEM.format( - today=date.today().isoformat(), - username=username, - category_hints=CATEGORY_HINTS, - income_category_hints=INCOME_CATEGORY_HINTS, - ), - tools=TOOL_DEFINITIONS, - messages=messages[-HISTORY_LIMIT:], - ) as stream: - first_chunk_of_turn = True - for chunk in stream.text_stream: - if not chunk: - continue - # Each turn streams independently, so the model can end one turn - # with "...today." and start the next (post-tool_use) with "Done!" - # with no space in between. Only check at the turn boundary — - # chunks within a single turn are exact slices of one continuous - # string and always join up correctly on their own. - if first_chunk_of_turn and last_char and not last_char.isspace() and not chunk[0].isspace(): - yield {"text": " "} - yield {"text": chunk} - last_char = chunk[-1] - first_chunk_of_turn = False - final = stream.get_final_message() - - messages.append({"role": "assistant", "content": final.content}) - - if final.stop_reason == "end_turn": - break - - if final.stop_reason == "tool_use": - for block in final.content: - if block.type == "tool_use": - yield {"status": TOOL_STATUS_LABELS.get(block.name, "Working…")} - - rich_events = [] - - def _capture_rich_result(name, result): - if name == "get_category_breakdown": - rich_events.append({"breakdown": result}) - - tool_results = _run_tools(final.content, user_id, on_result=_capture_rich_result) - messages.append({"role": "user", "content": tool_results}) - yield from rich_events + db.clear_chat_session(user_id) + + +def chat(user_input: str, user_id: int, username: str = "user", images: list[dict] | None = None, source: str | None = None) -> str: + with _get_session_lock(user_id): + messages = db.load_chat_session(user_id) + _repair_dangling_tool_use(messages) + content = _build_user_content(user_input, images) + _append_user_turn(messages, content) + _trim_session(messages) + db.save_chat_session(user_id, messages) + + while True: + response = client.messages.create( + model=MODEL_DEFAULT, + max_tokens=2048, + timeout=API_TIMEOUT, + system=SYSTEM.format( + today=date.today().isoformat(), + username=username, + category_hints=CATEGORY_HINTS, + income_category_hints=INCOME_CATEGORY_HINTS, + ), + tools=TOOL_DEFINITIONS, + messages=messages[-HISTORY_LIMIT:], + ) + + messages.append({"role": "assistant", "content": [_serialize_block(b) for b in response.content]}) + db.save_chat_session(user_id, messages) + + if response.stop_reason == "end_turn": + for block in response.content: + if hasattr(block, "text"): + return block.text + + if response.stop_reason == "tool_use": + tool_results = _run_tools(response.content, user_id, source=source) + messages.append({"role": "user", "content": tool_results}) + db.save_chat_session(user_id, messages) + + +def stream_chat(user_input: str, user_id: int, username: str = "user", images: list[dict] | None = None, source: str | None = None): + with _get_session_lock(user_id): + messages = db.load_chat_session(user_id) + _repair_dangling_tool_use(messages) + if images: + yield {"status": "Reading image…" if len(images) == 1 else "Reading images…"} + content = _build_user_content(user_input, images) + _append_user_turn(messages, content) + _trim_session(messages) + db.save_chat_session(user_id, messages) + + last_char = "" + while True: + with client.messages.stream( + model=MODEL_DEFAULT, + max_tokens=2048, + timeout=API_TIMEOUT, + system=SYSTEM.format( + today=date.today().isoformat(), + username=username, + category_hints=CATEGORY_HINTS, + income_category_hints=INCOME_CATEGORY_HINTS, + ), + tools=TOOL_DEFINITIONS, + messages=messages[-HISTORY_LIMIT:], + ) as stream: + first_chunk_of_turn = True + for chunk in stream.text_stream: + if not chunk: + continue + # Each turn streams independently, so the model can end one turn + # with "...today." and start the next (post-tool_use) with "Done!" + # with no space in between. Only check at the turn boundary — + # chunks within a single turn are exact slices of one continuous + # string and always join up correctly on their own. + if first_chunk_of_turn and last_char and not last_char.isspace() and not chunk[0].isspace(): + yield {"text": " "} + yield {"text": chunk} + last_char = chunk[-1] + first_chunk_of_turn = False + final = stream.get_final_message() + + messages.append({"role": "assistant", "content": [_serialize_block(b) for b in final.content]}) + db.save_chat_session(user_id, messages) + + if final.stop_reason == "end_turn": + break + + if final.stop_reason == "tool_use": + for block in final.content: + if block.type == "tool_use": + yield {"status": TOOL_STATUS_LABELS.get(block.name, "Working…")} + + rich_events = [] + + def _capture_rich_result(name, result): + if name == "get_category_breakdown": + rich_events.append({"breakdown": result}) + + tool_results = _run_tools(final.content, user_id, on_result=_capture_rich_result, source=source) + messages.append({"role": "user", "content": tool_results}) + db.save_chat_session(user_id, messages) + yield from rich_events if __name__ == "__main__": diff --git a/api/server.py b/api/server.py index 7ab80cb..346a5f8 100644 --- a/api/server.py +++ b/api/server.py @@ -26,6 +26,7 @@ get_recurring_expenses, get_user_by_username, increment_api_call_count, + record_usage, set_budget, update_expense, update_income, @@ -85,6 +86,12 @@ class ImageInput(BaseModel): class ChatRequest(BaseModel): message: str images: list[ImageInput] | None = None + # Where this message came from — a suggested chip, a slash command, or + # freeform typing (default). Lets usage reporting tell "clicked /budget" + # apart from "typed the same words", which look identical server-side + # otherwise. Frontend-supplied and unvalidated (analytics only, never + # used for auth/business logic), so no enum constraint here. + source: str | None = None def _images_payload(req: ChatRequest) -> list[dict] | None: @@ -99,7 +106,7 @@ def chat_stream_endpoint( ): def generate(): try: - for event in stream_chat(req.message, user_id, username, _images_payload(req)): + for event in stream_chat(req.message, user_id, username, _images_payload(req), req.source): yield f"data: {json.dumps(event)}\n\n" except Exception: logger.error("stream_chat error:\n%s", traceback.format_exc()) @@ -167,11 +174,13 @@ def insights_endpoint(user_id: int = Depends(get_current_user)): @app.put("/budgets/{category}") def set_budget_endpoint(category: str, req: BudgetRequest, user_id: int = Depends(get_current_user)): + record_usage(user_id, "ui", "set_budget") return set_budget(category, req.monthly_limit) @app.delete("/budgets/{category}") def delete_budget_endpoint(category: str, user_id: int = Depends(get_current_user)): + record_usage(user_id, "ui", "delete_budget") return delete_budget(category) @@ -202,6 +211,7 @@ def validate_category(cls, v): @app.patch("/expenses/{id}") def update_expense_endpoint(id: int, req: UpdateRequest, user_id: int = Depends(get_current_user)): + record_usage(user_id, "ui", "update_expense") result = update_expense(id, req.amount, req.category, req.description, req.date, req.flagged) if result.get("status") == "not_found": raise HTTPException(status_code=404, detail="Expense not found") @@ -210,6 +220,7 @@ def update_expense_endpoint(id: int, req: UpdateRequest, user_id: int = Depends( @app.delete("/expenses/{id}") def delete_expense_endpoint(id: int, user_id: int = Depends(get_current_user)): + record_usage(user_id, "ui", "delete_expense") result = delete_expense(id) if result.get("status") == "not_found": raise HTTPException(status_code=404, detail="Expense not found") @@ -232,6 +243,7 @@ def validate_category(cls, v): @app.patch("/income/{id}") def update_income_endpoint(id: int, req: IncomeUpdateRequest, user_id: int = Depends(get_current_user)): + record_usage(user_id, "ui", "update_income") result = update_income(id, req.amount, req.category, req.description, req.date) if result.get("status") == "not_found": raise HTTPException(status_code=404, detail="Income entry not found") @@ -240,6 +252,7 @@ def update_income_endpoint(id: int, req: IncomeUpdateRequest, user_id: int = Dep @app.delete("/income/{id}") def delete_income_endpoint(id: int, user_id: int = Depends(get_current_user)): + record_usage(user_id, "ui", "delete_income") result = delete_income(id) if result.get("status") == "not_found": raise HTTPException(status_code=404, detail="Income entry not found") @@ -248,6 +261,7 @@ def delete_income_endpoint(id: int, user_id: int = Depends(get_current_user)): @app.get("/expenses/export") def expenses_export(user_id: int = Depends(get_current_user)): + record_usage(user_id, "ui", "export_csv") rows = get_expenses() output = io.StringIO() writer = csv.DictWriter( diff --git a/frontend/src/components/Chat.jsx b/frontend/src/components/Chat.jsx index e6949b2..dda7e5a 100644 --- a/frontend/src/components/Chat.jsx +++ b/frontend/src/components/Chat.jsx @@ -288,7 +288,7 @@ export default function Chat({ onExpenseChange, className = "", token, username, setImages((prev) => prev.filter((_, i) => i !== index)); }; - const sendMessage = async (text, displayText = null) => { + const sendMessage = async (text, displayText = null, source = null) => { if (!text || loading) return; if (!hasOnboarded) { localStorage.setItem(onboardedKey, "1"); @@ -338,6 +338,7 @@ export default function Chat({ onExpenseChange, className = "", token, username, images: attachedImages.length ? attachedImages.map((img) => ({ data: img.data, media_type: img.mediaType })) : null, + source, }), signal: controller.signal, }); @@ -404,7 +405,7 @@ export default function Chat({ onExpenseChange, className = "", token, username, const runCommand = (cmd) => { setInput(""); - sendMessage(cmd.prompt, cmd.label); + sendMessage(cmd.prompt, cmd.label, `command:${cmd.command}`); }; const send = () => { @@ -430,7 +431,8 @@ export default function Chat({ onExpenseChange, className = "", token, username, const start = `${year}-${String(now.getMonth() + 1).padStart(2, "0")}-01`; sendMessage( `Summarize my expenses from ${start} to today. Show a breakdown by category with amounts, a total, and one observation about my spending.`, - `Summarize ${month} ${year}` + `Summarize ${month} ${year}`, + "chip:monthly_summary" ); }; @@ -643,7 +645,7 @@ export default function Chat({ onExpenseChange, className = "", token, username, {suggestions.map((s) => (