diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..4a4d94a --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,395 @@ +# CLAUDE.md — OpenDevOps Agent + +## Project Overview + +OpenDevOps Agent is an open-source AWS incident investigation tool powered by any LLM via LiteLLM. It runs a LangGraph ReAct loop (via DeepAgents) that calls 27 tools — 21 structured boto3 AWS tools plus bash, history analytics, skills, and a structured final-answer tool — then streams results to a React/Vite chat UI over SSE. Auth, multi-user RBAC, event-driven incident detection (EventBridge → SQS), and proactive anomaly polling are all built in and optional. + +--- + +## Repo Structure + +``` +apps/ + core/ Installable package `opendevops-core` — the shared agent brain: + src/opendevops_core/{agent, tools, providers, models, skills, + integrations, migrations, config.py}. Has no web/CLI layer. + Consumed by the OSS backend (and, later, the SaaS product repo). + backend/ OSS web app + CLI — src/{api, cli, config, mcp_server.py}, tests/, + scripts/, pyproject.toml. Depends on opendevops-core via a uv path source. + frontend/ React/Vite UI — src/, package.json, vite.config.ts + documentation/ Markdown docs (future hosted docs site) +deployment/ + docker-compose/ docker-compose.yml (PostgreSQL + backend + frontend) + railway/ Dockerfile.railway + railway.toml (combined single-image deploy) +design-system/ Cross-cutting design reference (colors, typography, UI kits) +demos/ Reproducible AWS incident scripts for local testing +Makefile Root convenience targets — wraps `cd apps/backend && uv run ...` +``` + +All Python commands run from `apps/backend/` (or via `make ` at repo root). +`uv sync` from `apps/backend/` installs `opendevops-core` editable from `../core`, so edits +to core are live without republishing. + +### Core vs app boundary +- **Reusable agent logic lives in `apps/core` (`opendevops_core`)** — the DeepAgents loop, + tools, providers, models, skills, integrations, DB backends, and baseline migrations. +- **Web/CLI-only code stays in `apps/backend`** — FastAPI routers, auth, the CLI, `mcp_server.py`. +- **Config injection:** core reads settings through the `settings` proxy in + `opendevops_core/config.py`, which delegates to whatever instance the host app registers via + `configure()`. The OSS app's `Settings(CoreSettings)` (in `apps/backend/src/config/appsettings.py`) + adds web/auth-only fields (e.g. `jwt_*`) and calls `configure(settings)` at startup. + +--- + +## Automatic Behavior Rules + +**Always do these when making changes:** + +- **New env var:** if core reads it, add the Pydantic field to `CoreSettings` in `apps/core/src/opendevops_core/config.py`; if it's web/auth-only, add it to `Settings(CoreSettings)` in `apps/backend/src/config/appsettings.py`. Either way mirror it in `.env.example` (with a comment). Never read env vars directly — always go through `settings`. +- **New DB column or table:** add a new numbered migration. Core-domain schema (tables core code reads/writes) goes in `apps/core/src/opendevops_core/migrations/` (e.g. `014_name.sql`); OSS-app-only schema goes in `apps/backend/migrations/`. The runner applies core-then-app, tracked in the `schema_migrations(source, version)` ledger. Never add columns inline in Python code. +- **New tool:** add it to `ALL_TOOLS` in `apps/core/src/opendevops_core/agent/core.py`. Tool functions must be plain synchronous Python functions — DeepAgents infers the JSON schema from type hints and docstrings. +- **New API route that matches a React Router path:** prefix it with `/api/` to avoid the SPA fallback conflict. The `/{full_path:path}` catch-all in `apps/backend/src/api/app.py` intercepts any GET that matches a registered FastAPI route first. +- **New skill:** drop a `SKILL.md` file into `apps/core/src/opendevops_core/skills//SKILL.md`. It is picked up automatically at startup (and bundled into the core wheel) — no code changes needed. Use the frontmatter format (`name`, `description`) from the existing `lambda-throttling` skill. +- **Docs sync:** if a feature has a corresponding file in `apps/documentation/`, update it when the feature changes. The `apps/documentation/` folder is the public documentation source. + +--- + +## Common Commands + +```bash +# Install / update dependencies +cd apps/backend && uv sync # or: make install + +# Development server — FastAPI with hot reload +cd apps/backend && uv run dev # or: make dev + +# Production web UI (FastAPI backend + serves built frontend, no reload) +cd apps/backend && uv run devops-agent ui # or: make ui + +# Apply SQL migrations to PostgreSQL (requires CHECKPOINT_BACKEND=postgres + DATABASE_URL) +cd apps/backend && uv run migrate # or: make migrate + +# CLI investigation +cd apps/backend && uv run devops-agent investigate "Lambda high error rate on payment service" +cd apps/backend && uv run devops-agent ask "Why would an ECS task OOM?" +cd apps/backend && uv run devops-agent report --days 7 + +# MCP server +cd apps/backend && uv run devops-agent mcp # stdio transport (Claude Desktop, Cursor) +cd apps/backend && uv run devops-agent mcp --http # HTTP+SSE transport, port 8001 + +# Tests +cd apps/backend && uv run pytest # or: make test + +# Lint / format (covers both the app and the core package) +cd apps/backend && uv run ruff check src/ ../core/src +cd apps/backend && uv run ruff format src/ ../core/src # or: make lint / make lint-fix + +# Full stack with PostgreSQL (Docker Compose) +docker compose -f deployment/docker-compose/docker-compose.yml up --build # or: make compose-up + +# Frontend dev server (port 5173, proxies API to localhost:8000) +cd apps/frontend && npm install && npm run dev # or: make frontend-dev + +# Frontend production build (output to apps/frontend/dist/ — served by FastAPI) +cd apps/frontend && npm run build # or: make frontend-build +``` + +--- + +## Current State + +Everything below is built and working in the codebase: + +### Agent & Tools +- **Framework:** DeepAgents (`create_deep_agent`) wrapping a LangGraph ReAct loop. `ChatLiteLLM` as the model interface — supports OpenRouter, Anthropic, OpenAI, Groq, Ollama, and any OpenAI-compatible endpoint via a single `LLM_MODEL` env var. +- **27 tools total** registered in `ALL_TOOLS` in `apps/core/src/opendevops_core/agent/core.py`: + - CloudWatch (6): `get_alarms`, `get_alarm_history`, `get_metric_data`, `get_log_events`, `describe_log_groups`, `query_logs_insights` + - CloudTrail (2): trail events + event lookup + - ECS (4): clusters, services, service detail, tasks + - Lambda (4): list, config, error rate, concurrent executions + - EC2 (2): list instances, instance details + - RDS (2): list DBs, DB details + - IAM (1): describe role + policies + - Bash (1): `run_bash_command` — allowlisted read-only `aws`, `kubectl`, `docker` commands; never `shell=True`; 30s hard timeout + - History (2): `get_investigation_history`, `search_past_investigations` + - Skills (2): `list_skills`, `use_skill` + - Final answer (1): `submit_investigation` — structured output required to end every investigation +- **Tool response capping:** `with_cap()` wraps every tool at startup when `TOOL_RESPONSE_MAX_CHARS > 0`; truncates oversized responses and appends a notice to the LLM. +- **Tool caching:** `@tool_cached` — in-process TTL LRU cache (2-min TTL, 256 entries max); cache key includes function name + AWS profile + region. +- **Skills system:** several skills ship (e.g. `lambda-throttling`). The system prompt is built at import time by scanning `apps/core/src/opendevops_core/skills/*/SKILL.md` — skill names are injected; full content is loaded lazily when the agent calls `use_skill(name)`. +- **Summarization:** `maybe_summarize()` runs before each agent call; compacts old messages when total chars exceed `SUMMARIZATION_THRESHOLD_CHARS`; tracks the event in `usage_events` with `metadata.summarization=True`. +- **Cancellation:** `DELETE /chat/{session_id}` sets an `asyncio.Event` that stops the streaming loop at the next chunk boundary. + +### Storage +- Three backends all implementing `DatabaseBackend` ABC: `memory` (default, zero config), `sqlite` (aiosqlite + LangGraph SQLite checkpointer), `postgres` (psycopg3 async + `AsyncPostgresSaver`). +- LangGraph checkpointer tables are created automatically by `AsyncPostgresSaver.setup()`. Application tables come from the bundled core migrations in `apps/core/src/opendevops_core/migrations/*.sql`. +- Schema tables: `organizations`, `users`, `aws_profiles`, `sessions`, `messages`, `tool_calls`, `usage_events`, `findings`, `api_keys`, `alerts`. +- Soft delete is in place on sessions (`is_deleted`, `deleted_at` from migration 002). +- Multi-tenant scoping: `upsert_session`, `list_sessions`, and `get_messages` accept an optional `org_id` (default `None` = unscoped). Postgres enforces it (filters lists, denies cross-org `get_messages`); memory scopes it; sqlite accepts-but-ignores it (single-tenant). `None` preserves single-tenant OSS behavior — the scoping exists for downstream multi-tenant consumers (the SaaS product). + +### API +- FastAPI SSE endpoint at `POST /chat`; streams `token`, `tool_status`, `tool_call`, `error`, `done`, `cancelled` events. +- SPA fallback: `GET /{full_path:path}` first serves a matching root-level file from `apps/frontend/dist/` (favicon, logos, and any other `public/` asset Vite copies to the build root), and otherwise returns `index.html` so React Router works on refresh. Vite's hashed JS/CSS is served separately via the `/assets` mount. **Don't narrow this back to "always index.html"** — root-level `public/` assets (e.g. `/favicon.svg`, `/Emblem.svg`) would then be served as HTML and render as broken images. +- All routes that could conflict with React Router paths use the `/api/` prefix: `/api/settings`, `/api/users`, `/api/history`, `/api/monitoring`, `/api/init`. +- Auth: optional JWT (HS256 via python-jose + bcrypt). Disabled when `JWT_SECRET` is unset — `get_current_user()` returns `None` in that case, meaning all routes are public. + +### Evidence Pack (replayable, read-side) +- `GET /api/sessions/{id}/evidence` (router `apps/backend/src/api/routers/evidence.py`) returns the replayable evidence pack: ranked hypotheses each with their cited evidence, every evidence item linked to the supporting tool call, the exact query/command that ran, and a deterministic AWS-console deeplink. Read-only — it never touches the SSE contract. +- **The investigation conclusion lives in `tool_calls`, not `findings`.** It is the `tool_calls` row with `tool_name='submit_investigation'` whose `args` are the structured output. The `findings` table is currently unwritten (placeholder schema). The pack builder reads the conclusion + supporting calls from `tool_calls` via `db.get_evidence(session_id)` (added to the `DatabaseBackend` ABC with a default + all three backends). +- Pure builder + deeplink logic is in `apps/core/src/opendevops_core/agent/evidence.py` (`build_evidence_pack`, `console_deeplink`, `exact_command`). Evidence→tool-call linking is a deterministic best-effort substring match on the call's arg identifiers. `exact_command` only returns a string for `query_logs_insights` (the Logs Insights query) and `run_bash_command` (the `az`/`kubectl` command); Azure has no console deeplink, so the command string IS the replay artifact. +- **AWS console hash-object encoder** (used for Logs Insights `queryDetail` and Metrics `graph` deeplinks): objects serialize as `~(k~v~...)` (leading tilde), arrays as `(~e1~e2)` (no leading tilde of their own — each element is tilde-prefixed), strings as `'` + chars with non-`[A-Za-z0-9-._]` escaped to `*xx`. Log-group/path segments use the simpler `quote(...).replace('%','$25')` form. Don't "simplify" these — the nesting and the tilde placement are exactly what the console parser expects. + +### Ranked Hypotheses (conclusion schema) +- `submit_investigation` emits `hypotheses: list[dict]` (each `{"hypothesis", "evidence": list[str], "confidence"}`) alongside the legacy `root_cause_summary` + flat `evidence[]`, which are preserved for backward compatibility. Kept as `list[dict]` so DeepAgents still infers the tool schema — do not make it a Pydantic-typed param. `InvestigationResult.hypotheses` reuses the existing `Finding` model. Migration `015_findings_hypotheses.sql` adds the `findings.hypotheses` JSONB column (postgres only; sqlite has no `findings` table). The pack builder falls back to a single synthetic hypothesis from the legacy fields when `hypotheses` is absent, so pre-existing investigations still render. + +### Event-Driven Detection +- `event_consumer_loop()` long-polls SQS (20s wait), processes EventBridge events (CloudWatch alarm, ECS task failure, Lambda async error, RDS event, EC2 state change, CodePipeline failure, AWS Health), runs a full agent investigation per event, delivers results to SNS + Slack, persists to `alerts` table. +- `context_collectors.collect_context()` pre-fetches resource facts deterministically before the LLM runs to reduce tool call count. +- Starts automatically on app startup if `event_consumer_enabled=True`, `sqs_queue_url` is set, or database-backed app config marks event infrastructure as enabled. Autonomous monitoring requires SQLite or PostgreSQL; memory mode is disabled for poller/consumer runs. + +### Proactive Polling +- `polling_loop()` runs every `POLL_INTERVAL_SECONDS` seconds (disabled by default at 0); checks CloudWatch alarms in ALARM state and Lambda error rates above `POLL_ERROR_THRESHOLD`; auto-investigates new anomalies and posts to Slack/Telegram. Dedup uses canonical incident keys and durable DB-backed claims. + +### Frontend +- React 18 + TypeScript + Vite + Tailwind CSS + `@tailwindcss/typography` +- Font: Inter Variable (Google Fonts) with a full system fallback stack +- Routes: `/`, `/chat/:sessionId`, `/dashboard`, `/monitoring`, `/monitoring/:alertId`, `/history`, `/settings`, `/users`, `/login` +- Chat page: SSE streaming, tool call inspector, cost/latency card, stop button, suggestion chips on empty state, `?prompt=` deeplink support +- Sidebar: paginated session list (15 at a time), three-dot menu with rename + delete (portal-based, no overflow clipping), real `` links for native right-click +- Settings: Environment, Agent Config, Integrations (UI stubs), AWS Configuration (editable, admin only), Preferences (dark mode) + +### Auth & MCP +- RBAC: `admin` and `user` roles. First registered user auto-becomes admin. `JWT_SECRET` unset = auth disabled. +- MCP server via fastmcp: `investigate`, `ask`, `list_sessions` tools. Stdio and HTTP+SSE transports. + +--- + +## Current Priorities + +Incomplete items from the README roadmap (do not mark complete here — update README when done): + +- **Custom tools via URL** — register external tools by OpenAPI endpoint; agent discovers them alongside built-in tools +- **Bash sandbox Phase 2** — throwaway Docker container per command: `--network none`, read-only FS, non-root, `--memory 256m`, killed immediately after; current Phase 1 (subprocess allowlist) is in `apps/core/src/opendevops_core/tools/bash_tool.py` +- **Optimize tool loading** — pass only contextually relevant tools instead of the full 27-tool set per invocation +- **OpenTelemetry traces** — spans for agent steps, tool call latency, token usage; OTLP export +- **Follow-up question suggestions** — add `follow_up_questions: list[str]` to `submit_investigation` schema (same call, no extra LLM cost); surface as chips in the chat UI after investigation completes +- **Session / user feedback loop** — thumbs up/down on investigations; `feedback` column in `usage_events` (needs migration 006) +- **Slack Integration UI** — Slack backend is fully implemented (`apps/core/src/opendevops_core/integrations/slack_webhook.py`); Settings → Integrations "Connect" button is currently a non-functional stub +- **Session rename** — `PATCH /sessions/{id}` + inline edit in sidebar three-dot menu +- **Multi-account AWS** — `aws_profiles` table already in schema; needs Settings UI + per-session profile selector +- **Knowledge base** — attach runbooks, post-mortems, architecture docs beyond the skills system + +--- + +## What NOT to Change Without Discussion + +| Contract | Why it matters | +|---|---| +| **SSE event types:** `token`, `tool_status`, `tool_call`, `error`, `done`, `cancelled` | Frontend `ChatPage.tsx` switches on these exact strings. Renaming or adding new required fields is a breaking change. | +| **Tool function signatures** | DeepAgents infers JSON schema from Python type hints + docstrings. Adding `*args`, `**kwargs`, removing type hints, or making parameters non-primitive breaks schema inference silently. | +| **`DatabaseBackend` ABC** (`apps/core/src/opendevops_core/agent/db/base.py`) | All three backends must implement the same interface. Adding a method requires implementing it in all three backends plus `memory.py` defaults. | +| **LangGraph checkpointer wiring** | The checkpointer is passed into `create_deep_agent()` and drives session continuity via `thread_id = session_id`. Do not write messages to the DB outside `save_*` calls or bypass the checkpointer. | +| **Agent framework (DeepAgents + LangGraph)** | The ReAct loop, tool dispatch, checkpointing, and `recursion_limit` contract all depend on this. Do not swap. | +| **DB schema migrations** | Tables have a defined shape. New columns need a new file in `migrations/`. Never add columns inline in Python or modify existing migration files. | +| **Auth opt-out pattern** | `get_current_user()` returns `None` when `jwt_secret` is unset (dev/memory mode). New routes that call `Depends(get_current_user)` must handle `None` gracefully — do not hard-require auth in non-admin routes. | +| **psycopg3 placeholder syntax** | psycopg3 uses `%s` (not `$1`/`$2`). Using asyncpg-style params causes silent failures with no Python exception. | + +--- + +## Tech Stack + +| Layer | Library / Version | +|---|---| +| Language | Python 3.11+ | +| Agent framework | `deepagents` + `langgraph>=0.2.0` | +| LLM abstraction | `litellm>=1.83.0` via `langchain-litellm` (`ChatLiteLLM`) | +| AWS SDK | `boto3>=1.34.0` (sync; all tools are synchronous) | +| Web backend | `fastapi>=0.111.0` + `uvicorn>=0.30.0` | +| CLI | `typer>=0.12.0` + `rich>=13.7.0` | +| Config | `pydantic-settings>=2.3.0` + `pydantic>=2.7.0` | +| Storage | `aiosqlite>=0.20.0` / `psycopg[binary,pool]>=3.1.0` + LangGraph checkpointers | +| Auth | `python-jose[cryptography]>=3.3.0` + `bcrypt>=5.0.0` | +| MCP server | `fastmcp>=3.2.4` | +| Tool cache | `cachetools>=5.3.0` (TTLCache, in-process) | +| Logging | `loguru>=0.7.0` (never `print`) | +| HTTP client | `httpx>=0.27.0` | +| Testing | `pytest>=8.0.0`, `pytest-asyncio>=0.23.0`, `moto>=5.0.0`, `pytest-mock>=3.14.0` | +| Linting | `ruff>=0.4.0` (line-length 100, Python 3.11 target, `asyncio_mode = "auto"`) | +| Package manager | **uv** — always `uv run` and `uv add`, never bare `pip` | +| Frontend | React 18, TypeScript, Vite, Tailwind CSS 3, `@tailwindcss/typography` | +| Font | Inter Variable (Google Fonts) — full system fallback stack in `apps/frontend/tailwind.config.js` | + +--- + +## Environment Variables + +Agent/core variables are defined on `CoreSettings` in `apps/core/src/opendevops_core/config.py`; web/auth-only variables (e.g. `JWT_SECRET`) live on `Settings(CoreSettings)` in `apps/backend/src/config/appsettings.py`. `.env.example` mirrors every variable. The OSS app instantiates `Settings` and registers it via `configure()` so core sees it at runtime. + +```bash +# LLM — LiteLLM model string format +LLM_MODEL=openrouter/openai/gpt-4o # default +LLM_API_BASE= # optional custom base URL +LLM_API_KEY= # optional custom API key +OPENROUTER_API_KEY= # used when LLM_MODEL starts with "openrouter/" + +# AWS +AWS_REGION=us-east-1 # default +AWS_PROFILE= # optional named ~/.aws profile + +# Agent behavior +MAX_TOOL_CALLS=20 # recursion_limit = MAX_TOOL_CALLS * 3 + 15 +INVESTIGATION_TIMEOUT=120 # seconds before asyncio.TimeoutError +LOG_LEVEL=INFO +LOG_CONSOLE_ENABLED=true # false = suppress all console output +LOG_CONSOLE_COLORIZE=true # false = strip ANSI colours (CI / Docker) +TOOL_RESPONSE_MAX_CHARS=40000 # 0 = disabled; ~10K tokens at 4 chars/token + +# Storage backend — pick exactly one +CHECKPOINT_BACKEND=memory # memory | sqlite | postgres +SQLITE_PATH=./data/agent.db # only when backend=sqlite +DATABASE_URL= # only when backend=postgres (psycopg3 DSN) + +# Conversation summarization +SUMMARIZATION_ENABLED=true +SUMMARIZATION_THRESHOLD_CHARS=60000 # trigger when session exceeds this (~15K tokens) +SUMMARIZATION_KEEP_CHARS=20000 # preserve this many recent chars intact (~5K tokens) + +# Auth — leave unset to disable entirely (all routes public) +JWT_SECRET= # set to enable auth; required for /api/users +JWT_EXPIRE_MINUTES=1440 # 24 hours + +# Slack notifications +SLACK_WEBHOOK_URL= # leave unset to disable + +# Proactive polling +POLL_INTERVAL_SECONDS=0 # 0 = disabled; set to e.g. 300 (5 min) to enable +POLL_ERROR_THRESHOLD=5.0 # Lambda error rate % to trigger investigation +POLL_REINVESTIGATE_HOURS=1 # dedup window + +# Event-driven detection +SNS_TOPIC_ARN= # SNS publish target after investigations +SQS_QUEUE_URL= # SQS queue for EventBridge events +EVENT_CONSUMER_ENABLED=false # also auto-starts if SQS_QUEUE_URL is set or app config enables infra + +# Misc +DATA_DIR=data # reserved for future file-based state +``` + +--- + +## Core Architecture + +### Request flow (web chat) +``` +POST /chat → maybe_summarize() → agent.astream() + LangGraph ReAct loop: + LLM reasons → picks tool → @tool_cached check → with_cap() → boto3 / subprocess + result injected back to LLM → repeat until submit_investigation() or MAX_TOOL_CALLS + SSE events streamed per chunk: + token | tool_status | tool_call | error | done | cancelled + After stream ends: + save_turn() → upsert_session + save_message + save_tool_calls + save_usage_event + notify_slack() → only if submit_investigation was called +``` + +### Event-driven flow +``` +EventBridge rules (9 event types) + → SQS queue → event_consumer_loop() (long-poll, 20s wait) + → _is_real_failure() filter + → collect_context() (deterministic boto3 pre-fetch, no LLM) + → agent.ainvoke() (full ReAct loop) + → _deliver(): SNS publish + Slack post + add_alert() → alerts table +``` + +### Startup sequence +``` +db.init() → init_agent(checkpointer) + optional: asyncio.create_task(polling_loop()) if POLL_INTERVAL_SECONDS > 0 + optional: asyncio.create_task(event_consumer_loop()) if SQS configured or app config enables infra +``` + +### Session continuity +The LangGraph checkpointer stores full thread state keyed by `session_id`. Every `/chat` call resumes the thread by passing `thread_id = session_id` in config — the agent sees complete history without the API explicitly passing messages. + +--- + +## Code Style Rules + +- **Type hints everywhere** — no untyped functions, no `Any` without import +- **Tool functions are synchronous** — boto3 is sync; `async` lives only in the API and DB layers +- **Tools always return `dict`, never raise** — catch `BotoCoreError`, `ClientError`, and `Exception`; return `{"error": str(e), ...}` with safe empty defaults +- **Never use `shell=True`** in subprocess — `bash_tool.py` uses `shlex.split()` + list-form `subprocess.run()` +- **Credentials via env or AWS profiles only** — never hardcoded, never in source +- **`logger` (Loguru) for all output** — never `print()` +- **`uv run` / `uv add`** — never bare `pip install` +- **ruff** for lint and format — `line-length = 100`, `target-version = "py311"`, rules `E F I UP` +- **psycopg3 uses `%s` placeholders** — not `$1`/`$2` (that is asyncpg) +- **Route prefix rule** — any API route whose path matches a React Router path must use `/api/` prefix to avoid the SPA fallback catch-all intercepting browser GETs on refresh +- **Migrations are append-only** — never modify existing `.sql` files; add a new numbered file +- **No `shell=True`, no write commands in bash tool** — the allowlist is the contract; never bypass it + +Behavioral guidelines to reduce common LLM coding mistakes. Merge with project-specific instructions as needed. + +**Tradeoff:** These guidelines bias toward caution over speed. For trivial tasks, use judgment. + +## 1. Think Before Coding + +**Don't assume. Don't hide confusion. Surface tradeoffs.** + +Before implementing: +- State your assumptions explicitly. If uncertain, ask. +- If multiple interpretations exist, present them - don't pick silently. +- If a simpler approach exists, say so. Push back when warranted. +- If something is unclear, stop. Name what's confusing. Ask. + +## 2. Simplicity First + +**Minimum code that solves the problem. Nothing speculative.** + +- No features beyond what was asked. +- No abstractions for single-use code. +- No "flexibility" or "configurability" that wasn't requested. +- No error handling for impossible scenarios. +- If you write 200 lines and it could be 50, rewrite it. + +Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify. + +## 3. Surgical Changes + +**Touch only what you must. Clean up only your own mess.** + +When editing existing code: +- Don't "improve" adjacent code, comments, or formatting. +- Don't refactor things that aren't broken. +- Match existing style, even if you'd do it differently. +- If you notice unrelated dead code, mention it - don't delete it. + +When your changes create orphans: +- Remove imports/variables/functions that YOUR changes made unused. +- Don't remove pre-existing dead code unless asked. + +The test: Every changed line should trace directly to the user's request. + +## 4. Goal-Driven Execution + +**Define success criteria. Loop until verified.** + +Transform tasks into verifiable goals: +- "Add validation" → "Write tests for invalid inputs, then make them pass" +- "Fix the bug" → "Write a test that reproduces it, then make it pass" +- "Refactor X" → "Ensure tests pass before and after" + +For multi-step tasks, state a brief plan: +``` +1. [Step] → verify: [check] +2. [Step] → verify: [check] +3. [Step] → verify: [check] +``` + +Strong success criteria let you loop independently. Weak criteria ("make it work") require constant clarification. + +--- + +**These guidelines are working if:** fewer unnecessary changes in diffs, fewer rewrites due to overcomplication, and clarifying questions come before implementation rather than after mistakes. \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d07d2a..4e9d866 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,21 @@ Notable changes to OpenDevOps Agent (open-source core + backend). (`None` = unscoped — OSS behavior unchanged). - `CREDENTIALS_ENCRYPTION_KEY` (Fernet) for encrypting stored account secrets. +### Added — Replayable evidence pack & ranked hypotheses +- **Evidence pack endpoint** — read-only `GET /api/sessions/{id}/evidence` returns the + investigation's ranked hypotheses, each with cited evidence linked to the supporting tool + call, the exact query/command that ran, and a deterministic AWS-console deeplink. Reads the + conclusion from the persisted `submit_investigation` tool call (the `findings` table stays + an unwritten placeholder). Pure builder + console-deeplink encoder live in + `opendevops_core/agent/evidence.py`; `db.get_evidence()` added to the `DatabaseBackend` ABC + (default + all three backends). Frontend `EvidencePanel` renders grouped hypotheses + replay + cards with copy-to-clipboard and JSON export. See `apps/documentation/evidence_pack.md`. +- **Ranked hypotheses conclusion schema** — `submit_investigation` now emits + `hypotheses: list[dict]` (`{hypothesis, evidence, confidence}`) alongside the legacy + `root_cause_summary` + flat `evidence[]` (preserved for backward compatibility). Migration + `015` adds `findings.hypotheses JSONB` (postgres only). The builder falls back to one + synthetic hypothesis when `hypotheses` is absent so pre-existing investigations still render. + ### Changed - **README** reframed as multi-cloud (AWS + Azure) with links to both setup guides. - **`demos/`** reorganized into `demos/aws/` + `demos/azure/` with a top-level index. diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index a799958..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,386 +0,0 @@ -# CLAUDE.md — OpenDevOps Agent - -## Project Overview - -OpenDevOps Agent is an open-source AWS incident investigation tool powered by any LLM via LiteLLM. It runs a LangGraph ReAct loop (via DeepAgents) that calls 27 tools — 21 structured boto3 AWS tools plus bash, history analytics, skills, and a structured final-answer tool — then streams results to a React/Vite chat UI over SSE. Auth, multi-user RBAC, event-driven incident detection (EventBridge → SQS), and proactive anomaly polling are all built in and optional. - ---- - -## Repo Structure - -``` -apps/ - core/ Installable package `opendevops-core` — the shared agent brain: - src/opendevops_core/{agent, tools, providers, models, skills, - integrations, migrations, config.py}. Has no web/CLI layer. - Consumed by the OSS backend (and, later, the SaaS product repo). - backend/ OSS web app + CLI — src/{api, cli, config, mcp_server.py}, tests/, - scripts/, pyproject.toml. Depends on opendevops-core via a uv path source. - frontend/ React/Vite UI — src/, package.json, vite.config.ts - documentation/ Markdown docs (future hosted docs site) -deployment/ - docker-compose/ docker-compose.yml (PostgreSQL + backend + frontend) - railway/ Dockerfile.railway + railway.toml (combined single-image deploy) -design-system/ Cross-cutting design reference (colors, typography, UI kits) -demos/ Reproducible AWS incident scripts for local testing -Makefile Root convenience targets — wraps `cd apps/backend && uv run ...` -``` - -All Python commands run from `apps/backend/` (or via `make ` at repo root). -`uv sync` from `apps/backend/` installs `opendevops-core` editable from `../core`, so edits -to core are live without republishing. - -### Core vs app boundary -- **Reusable agent logic lives in `apps/core` (`opendevops_core`)** — the DeepAgents loop, - tools, providers, models, skills, integrations, DB backends, and baseline migrations. -- **Web/CLI-only code stays in `apps/backend`** — FastAPI routers, auth, the CLI, `mcp_server.py`. -- **Config injection:** core reads settings through the `settings` proxy in - `opendevops_core/config.py`, which delegates to whatever instance the host app registers via - `configure()`. The OSS app's `Settings(CoreSettings)` (in `apps/backend/src/config/appsettings.py`) - adds web/auth-only fields (e.g. `jwt_*`) and calls `configure(settings)` at startup. - ---- - -## Automatic Behavior Rules - -**Always do these when making changes:** - -- **New env var:** if core reads it, add the Pydantic field to `CoreSettings` in `apps/core/src/opendevops_core/config.py`; if it's web/auth-only, add it to `Settings(CoreSettings)` in `apps/backend/src/config/appsettings.py`. Either way mirror it in `.env.example` (with a comment). Never read env vars directly — always go through `settings`. -- **New DB column or table:** add a new numbered migration. Core-domain schema (tables core code reads/writes) goes in `apps/core/src/opendevops_core/migrations/` (e.g. `014_name.sql`); OSS-app-only schema goes in `apps/backend/migrations/`. The runner applies core-then-app, tracked in the `schema_migrations(source, version)` ledger. Never add columns inline in Python code. -- **New tool:** add it to `ALL_TOOLS` in `apps/core/src/opendevops_core/agent/core.py`. Tool functions must be plain synchronous Python functions — DeepAgents infers the JSON schema from type hints and docstrings. -- **New API route that matches a React Router path:** prefix it with `/api/` to avoid the SPA fallback conflict. The `/{full_path:path}` catch-all in `apps/backend/src/api/app.py` intercepts any GET that matches a registered FastAPI route first. -- **New skill:** drop a `SKILL.md` file into `apps/core/src/opendevops_core/skills//SKILL.md`. It is picked up automatically at startup (and bundled into the core wheel) — no code changes needed. Use the frontmatter format (`name`, `description`) from the existing `lambda-throttling` skill. -- **Docs sync:** if a feature has a corresponding file in `apps/documentation/`, update it when the feature changes. The `apps/documentation/` folder is the public documentation source. - ---- - -## Common Commands - -```bash -# Install / update dependencies -cd apps/backend && uv sync # or: make install - -# Development server — FastAPI with hot reload -cd apps/backend && uv run dev # or: make dev - -# Production web UI (FastAPI backend + serves built frontend, no reload) -cd apps/backend && uv run devops-agent ui # or: make ui - -# Apply SQL migrations to PostgreSQL (requires CHECKPOINT_BACKEND=postgres + DATABASE_URL) -cd apps/backend && uv run migrate # or: make migrate - -# CLI investigation -cd apps/backend && uv run devops-agent investigate "Lambda high error rate on payment service" -cd apps/backend && uv run devops-agent ask "Why would an ECS task OOM?" -cd apps/backend && uv run devops-agent report --days 7 - -# MCP server -cd apps/backend && uv run devops-agent mcp # stdio transport (Claude Desktop, Cursor) -cd apps/backend && uv run devops-agent mcp --http # HTTP+SSE transport, port 8001 - -# Tests -cd apps/backend && uv run pytest # or: make test - -# Lint / format (covers both the app and the core package) -cd apps/backend && uv run ruff check src/ ../core/src -cd apps/backend && uv run ruff format src/ ../core/src # or: make lint / make lint-fix - -# Full stack with PostgreSQL (Docker Compose) -docker compose -f deployment/docker-compose/docker-compose.yml up --build # or: make compose-up - -# Frontend dev server (port 5173, proxies API to localhost:8000) -cd apps/frontend && npm install && npm run dev # or: make frontend-dev - -# Frontend production build (output to apps/frontend/dist/ — served by FastAPI) -cd apps/frontend && npm run build # or: make frontend-build -``` - ---- - -## Current State - -Everything below is built and working in the codebase: - -### Agent & Tools -- **Framework:** DeepAgents (`create_deep_agent`) wrapping a LangGraph ReAct loop. `ChatLiteLLM` as the model interface — supports OpenRouter, Anthropic, OpenAI, Groq, Ollama, and any OpenAI-compatible endpoint via a single `LLM_MODEL` env var. -- **27 tools total** registered in `ALL_TOOLS` in `apps/core/src/opendevops_core/agent/core.py`: - - CloudWatch (6): `get_alarms`, `get_alarm_history`, `get_metric_data`, `get_log_events`, `describe_log_groups`, `query_logs_insights` - - CloudTrail (2): trail events + event lookup - - ECS (4): clusters, services, service detail, tasks - - Lambda (4): list, config, error rate, concurrent executions - - EC2 (2): list instances, instance details - - RDS (2): list DBs, DB details - - IAM (1): describe role + policies - - Bash (1): `run_bash_command` — allowlisted read-only `aws`, `kubectl`, `docker` commands; never `shell=True`; 30s hard timeout - - History (2): `get_investigation_history`, `search_past_investigations` - - Skills (2): `list_skills`, `use_skill` - - Final answer (1): `submit_investigation` — structured output required to end every investigation -- **Tool response capping:** `with_cap()` wraps every tool at startup when `TOOL_RESPONSE_MAX_CHARS > 0`; truncates oversized responses and appends a notice to the LLM. -- **Tool caching:** `@tool_cached` — in-process TTL LRU cache (2-min TTL, 256 entries max); cache key includes function name + AWS profile + region. -- **Skills system:** several skills ship (e.g. `lambda-throttling`). The system prompt is built at import time by scanning `apps/core/src/opendevops_core/skills/*/SKILL.md` — skill names are injected; full content is loaded lazily when the agent calls `use_skill(name)`. -- **Summarization:** `maybe_summarize()` runs before each agent call; compacts old messages when total chars exceed `SUMMARIZATION_THRESHOLD_CHARS`; tracks the event in `usage_events` with `metadata.summarization=True`. -- **Cancellation:** `DELETE /chat/{session_id}` sets an `asyncio.Event` that stops the streaming loop at the next chunk boundary. - -### Storage -- Three backends all implementing `DatabaseBackend` ABC: `memory` (default, zero config), `sqlite` (aiosqlite + LangGraph SQLite checkpointer), `postgres` (psycopg3 async + `AsyncPostgresSaver`). -- LangGraph checkpointer tables are created automatically by `AsyncPostgresSaver.setup()`. Application tables come from the bundled core migrations in `apps/core/src/opendevops_core/migrations/*.sql`. -- Schema tables: `organizations`, `users`, `aws_profiles`, `sessions`, `messages`, `tool_calls`, `usage_events`, `findings`, `api_keys`, `alerts`. -- Soft delete is in place on sessions (`is_deleted`, `deleted_at` from migration 002). -- Multi-tenant scoping: `upsert_session`, `list_sessions`, and `get_messages` accept an optional `org_id` (default `None` = unscoped). Postgres enforces it (filters lists, denies cross-org `get_messages`); memory scopes it; sqlite accepts-but-ignores it (single-tenant). `None` preserves single-tenant OSS behavior — the scoping exists for downstream multi-tenant consumers (the SaaS product). - -### API -- FastAPI SSE endpoint at `POST /chat`; streams `token`, `tool_status`, `tool_call`, `error`, `done`, `cancelled` events. -- SPA fallback: `GET /{full_path:path}` first serves a matching root-level file from `apps/frontend/dist/` (favicon, logos, and any other `public/` asset Vite copies to the build root), and otherwise returns `index.html` so React Router works on refresh. Vite's hashed JS/CSS is served separately via the `/assets` mount. **Don't narrow this back to "always index.html"** — root-level `public/` assets (e.g. `/favicon.svg`, `/Emblem.svg`) would then be served as HTML and render as broken images. -- All routes that could conflict with React Router paths use the `/api/` prefix: `/api/settings`, `/api/users`, `/api/history`, `/api/monitoring`, `/api/init`. -- Auth: optional JWT (HS256 via python-jose + bcrypt). Disabled when `JWT_SECRET` is unset — `get_current_user()` returns `None` in that case, meaning all routes are public. - -### Event-Driven Detection -- `event_consumer_loop()` long-polls SQS (20s wait), processes EventBridge events (CloudWatch alarm, ECS task failure, Lambda async error, RDS event, EC2 state change, CodePipeline failure, AWS Health), runs a full agent investigation per event, delivers results to SNS + Slack, persists to `alerts` table. -- `context_collectors.collect_context()` pre-fetches resource facts deterministically before the LLM runs to reduce tool call count. -- Starts automatically on app startup if `event_consumer_enabled=True`, `sqs_queue_url` is set, or database-backed app config marks event infrastructure as enabled. Autonomous monitoring requires SQLite or PostgreSQL; memory mode is disabled for poller/consumer runs. - -### Proactive Polling -- `polling_loop()` runs every `POLL_INTERVAL_SECONDS` seconds (disabled by default at 0); checks CloudWatch alarms in ALARM state and Lambda error rates above `POLL_ERROR_THRESHOLD`; auto-investigates new anomalies and posts to Slack/Telegram. Dedup uses canonical incident keys and durable DB-backed claims. - -### Frontend -- React 18 + TypeScript + Vite + Tailwind CSS + `@tailwindcss/typography` -- Font: Inter Variable (Google Fonts) with a full system fallback stack -- Routes: `/`, `/chat/:sessionId`, `/dashboard`, `/monitoring`, `/monitoring/:alertId`, `/history`, `/settings`, `/users`, `/login` -- Chat page: SSE streaming, tool call inspector, cost/latency card, stop button, suggestion chips on empty state, `?prompt=` deeplink support -- Sidebar: paginated session list (15 at a time), three-dot menu with rename + delete (portal-based, no overflow clipping), real `` links for native right-click -- Settings: Environment, Agent Config, Integrations (UI stubs), AWS Configuration (editable, admin only), Preferences (dark mode) - -### Auth & MCP -- RBAC: `admin` and `user` roles. First registered user auto-becomes admin. `JWT_SECRET` unset = auth disabled. -- MCP server via fastmcp: `investigate`, `ask`, `list_sessions` tools. Stdio and HTTP+SSE transports. - ---- - -## Current Priorities - -Incomplete items from the README roadmap (do not mark complete here — update README when done): - -- **Custom tools via URL** — register external tools by OpenAPI endpoint; agent discovers them alongside built-in tools -- **Bash sandbox Phase 2** — throwaway Docker container per command: `--network none`, read-only FS, non-root, `--memory 256m`, killed immediately after; current Phase 1 (subprocess allowlist) is in `apps/core/src/opendevops_core/tools/bash_tool.py` -- **Optimize tool loading** — pass only contextually relevant tools instead of the full 27-tool set per invocation -- **OpenTelemetry traces** — spans for agent steps, tool call latency, token usage; OTLP export -- **Follow-up question suggestions** — add `follow_up_questions: list[str]` to `submit_investigation` schema (same call, no extra LLM cost); surface as chips in the chat UI after investigation completes -- **Session / user feedback loop** — thumbs up/down on investigations; `feedback` column in `usage_events` (needs migration 006) -- **Slack Integration UI** — Slack backend is fully implemented (`apps/core/src/opendevops_core/integrations/slack_webhook.py`); Settings → Integrations "Connect" button is currently a non-functional stub -- **Session rename** — `PATCH /sessions/{id}` + inline edit in sidebar three-dot menu -- **Multi-account AWS** — `aws_profiles` table already in schema; needs Settings UI + per-session profile selector -- **Knowledge base** — attach runbooks, post-mortems, architecture docs beyond the skills system - ---- - -## What NOT to Change Without Discussion - -| Contract | Why it matters | -|---|---| -| **SSE event types:** `token`, `tool_status`, `tool_call`, `error`, `done`, `cancelled` | Frontend `ChatPage.tsx` switches on these exact strings. Renaming or adding new required fields is a breaking change. | -| **Tool function signatures** | DeepAgents infers JSON schema from Python type hints + docstrings. Adding `*args`, `**kwargs`, removing type hints, or making parameters non-primitive breaks schema inference silently. | -| **`DatabaseBackend` ABC** (`apps/core/src/opendevops_core/agent/db/base.py`) | All three backends must implement the same interface. Adding a method requires implementing it in all three backends plus `memory.py` defaults. | -| **LangGraph checkpointer wiring** | The checkpointer is passed into `create_deep_agent()` and drives session continuity via `thread_id = session_id`. Do not write messages to the DB outside `save_*` calls or bypass the checkpointer. | -| **Agent framework (DeepAgents + LangGraph)** | The ReAct loop, tool dispatch, checkpointing, and `recursion_limit` contract all depend on this. Do not swap. | -| **DB schema migrations** | Tables have a defined shape. New columns need a new file in `migrations/`. Never add columns inline in Python or modify existing migration files. | -| **Auth opt-out pattern** | `get_current_user()` returns `None` when `jwt_secret` is unset (dev/memory mode). New routes that call `Depends(get_current_user)` must handle `None` gracefully — do not hard-require auth in non-admin routes. | -| **psycopg3 placeholder syntax** | psycopg3 uses `%s` (not `$1`/`$2`). Using asyncpg-style params causes silent failures with no Python exception. | - ---- - -## Tech Stack - -| Layer | Library / Version | -|---|---| -| Language | Python 3.11+ | -| Agent framework | `deepagents` + `langgraph>=0.2.0` | -| LLM abstraction | `litellm>=1.83.0` via `langchain-litellm` (`ChatLiteLLM`) | -| AWS SDK | `boto3>=1.34.0` (sync; all tools are synchronous) | -| Web backend | `fastapi>=0.111.0` + `uvicorn>=0.30.0` | -| CLI | `typer>=0.12.0` + `rich>=13.7.0` | -| Config | `pydantic-settings>=2.3.0` + `pydantic>=2.7.0` | -| Storage | `aiosqlite>=0.20.0` / `psycopg[binary,pool]>=3.1.0` + LangGraph checkpointers | -| Auth | `python-jose[cryptography]>=3.3.0` + `bcrypt>=5.0.0` | -| MCP server | `fastmcp>=3.2.4` | -| Tool cache | `cachetools>=5.3.0` (TTLCache, in-process) | -| Logging | `loguru>=0.7.0` (never `print`) | -| HTTP client | `httpx>=0.27.0` | -| Testing | `pytest>=8.0.0`, `pytest-asyncio>=0.23.0`, `moto>=5.0.0`, `pytest-mock>=3.14.0` | -| Linting | `ruff>=0.4.0` (line-length 100, Python 3.11 target, `asyncio_mode = "auto"`) | -| Package manager | **uv** — always `uv run` and `uv add`, never bare `pip` | -| Frontend | React 18, TypeScript, Vite, Tailwind CSS 3, `@tailwindcss/typography` | -| Font | Inter Variable (Google Fonts) — full system fallback stack in `apps/frontend/tailwind.config.js` | - ---- - -## Environment Variables - -Agent/core variables are defined on `CoreSettings` in `apps/core/src/opendevops_core/config.py`; web/auth-only variables (e.g. `JWT_SECRET`) live on `Settings(CoreSettings)` in `apps/backend/src/config/appsettings.py`. `.env.example` mirrors every variable. The OSS app instantiates `Settings` and registers it via `configure()` so core sees it at runtime. - -```bash -# LLM — LiteLLM model string format -LLM_MODEL=openrouter/openai/gpt-4o # default -LLM_API_BASE= # optional custom base URL -LLM_API_KEY= # optional custom API key -OPENROUTER_API_KEY= # used when LLM_MODEL starts with "openrouter/" - -# AWS -AWS_REGION=us-east-1 # default -AWS_PROFILE= # optional named ~/.aws profile - -# Agent behavior -MAX_TOOL_CALLS=20 # recursion_limit = MAX_TOOL_CALLS * 3 + 15 -INVESTIGATION_TIMEOUT=120 # seconds before asyncio.TimeoutError -LOG_LEVEL=INFO -LOG_CONSOLE_ENABLED=true # false = suppress all console output -LOG_CONSOLE_COLORIZE=true # false = strip ANSI colours (CI / Docker) -TOOL_RESPONSE_MAX_CHARS=40000 # 0 = disabled; ~10K tokens at 4 chars/token - -# Storage backend — pick exactly one -CHECKPOINT_BACKEND=memory # memory | sqlite | postgres -SQLITE_PATH=./data/agent.db # only when backend=sqlite -DATABASE_URL= # only when backend=postgres (psycopg3 DSN) - -# Conversation summarization -SUMMARIZATION_ENABLED=true -SUMMARIZATION_THRESHOLD_CHARS=60000 # trigger when session exceeds this (~15K tokens) -SUMMARIZATION_KEEP_CHARS=20000 # preserve this many recent chars intact (~5K tokens) - -# Auth — leave unset to disable entirely (all routes public) -JWT_SECRET= # set to enable auth; required for /api/users -JWT_EXPIRE_MINUTES=1440 # 24 hours - -# Slack notifications -SLACK_WEBHOOK_URL= # leave unset to disable - -# Proactive polling -POLL_INTERVAL_SECONDS=0 # 0 = disabled; set to e.g. 300 (5 min) to enable -POLL_ERROR_THRESHOLD=5.0 # Lambda error rate % to trigger investigation -POLL_REINVESTIGATE_HOURS=1 # dedup window - -# Event-driven detection -SNS_TOPIC_ARN= # SNS publish target after investigations -SQS_QUEUE_URL= # SQS queue for EventBridge events -EVENT_CONSUMER_ENABLED=false # also auto-starts if SQS_QUEUE_URL is set or app config enables infra - -# Misc -DATA_DIR=data # reserved for future file-based state -``` - ---- - -## Core Architecture - -### Request flow (web chat) -``` -POST /chat → maybe_summarize() → agent.astream() - LangGraph ReAct loop: - LLM reasons → picks tool → @tool_cached check → with_cap() → boto3 / subprocess - result injected back to LLM → repeat until submit_investigation() or MAX_TOOL_CALLS - SSE events streamed per chunk: - token | tool_status | tool_call | error | done | cancelled - After stream ends: - save_turn() → upsert_session + save_message + save_tool_calls + save_usage_event - notify_slack() → only if submit_investigation was called -``` - -### Event-driven flow -``` -EventBridge rules (9 event types) - → SQS queue → event_consumer_loop() (long-poll, 20s wait) - → _is_real_failure() filter - → collect_context() (deterministic boto3 pre-fetch, no LLM) - → agent.ainvoke() (full ReAct loop) - → _deliver(): SNS publish + Slack post + add_alert() → alerts table -``` - -### Startup sequence -``` -db.init() → init_agent(checkpointer) - optional: asyncio.create_task(polling_loop()) if POLL_INTERVAL_SECONDS > 0 - optional: asyncio.create_task(event_consumer_loop()) if SQS configured or app config enables infra -``` - -### Session continuity -The LangGraph checkpointer stores full thread state keyed by `session_id`. Every `/chat` call resumes the thread by passing `thread_id = session_id` in config — the agent sees complete history without the API explicitly passing messages. - ---- - -## Code Style Rules - -- **Type hints everywhere** — no untyped functions, no `Any` without import -- **Tool functions are synchronous** — boto3 is sync; `async` lives only in the API and DB layers -- **Tools always return `dict`, never raise** — catch `BotoCoreError`, `ClientError`, and `Exception`; return `{"error": str(e), ...}` with safe empty defaults -- **Never use `shell=True`** in subprocess — `bash_tool.py` uses `shlex.split()` + list-form `subprocess.run()` -- **Credentials via env or AWS profiles only** — never hardcoded, never in source -- **`logger` (Loguru) for all output** — never `print()` -- **`uv run` / `uv add`** — never bare `pip install` -- **ruff** for lint and format — `line-length = 100`, `target-version = "py311"`, rules `E F I UP` -- **psycopg3 uses `%s` placeholders** — not `$1`/`$2` (that is asyncpg) -- **Route prefix rule** — any API route whose path matches a React Router path must use `/api/` prefix to avoid the SPA fallback catch-all intercepting browser GETs on refresh -- **Migrations are append-only** — never modify existing `.sql` files; add a new numbered file -- **No `shell=True`, no write commands in bash tool** — the allowlist is the contract; never bypass it - -Behavioral guidelines to reduce common LLM coding mistakes. Merge with project-specific instructions as needed. - -**Tradeoff:** These guidelines bias toward caution over speed. For trivial tasks, use judgment. - -## 1. Think Before Coding - -**Don't assume. Don't hide confusion. Surface tradeoffs.** - -Before implementing: -- State your assumptions explicitly. If uncertain, ask. -- If multiple interpretations exist, present them - don't pick silently. -- If a simpler approach exists, say so. Push back when warranted. -- If something is unclear, stop. Name what's confusing. Ask. - -## 2. Simplicity First - -**Minimum code that solves the problem. Nothing speculative.** - -- No features beyond what was asked. -- No abstractions for single-use code. -- No "flexibility" or "configurability" that wasn't requested. -- No error handling for impossible scenarios. -- If you write 200 lines and it could be 50, rewrite it. - -Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify. - -## 3. Surgical Changes - -**Touch only what you must. Clean up only your own mess.** - -When editing existing code: -- Don't "improve" adjacent code, comments, or formatting. -- Don't refactor things that aren't broken. -- Match existing style, even if you'd do it differently. -- If you notice unrelated dead code, mention it - don't delete it. - -When your changes create orphans: -- Remove imports/variables/functions that YOUR changes made unused. -- Don't remove pre-existing dead code unless asked. - -The test: Every changed line should trace directly to the user's request. - -## 4. Goal-Driven Execution - -**Define success criteria. Loop until verified.** - -Transform tasks into verifiable goals: -- "Add validation" → "Write tests for invalid inputs, then make them pass" -- "Fix the bug" → "Write a test that reproduces it, then make it pass" -- "Refactor X" → "Ensure tests pass before and after" - -For multi-step tasks, state a brief plan: -``` -1. [Step] → verify: [check] -2. [Step] → verify: [check] -3. [Step] → verify: [check] -``` - -Strong success criteria let you loop independently. Weak criteria ("make it work") require constant clarification. - ---- - -**These guidelines are working if:** fewer unnecessary changes in diffs, fewer rewrites due to overcomplication, and clarifying questions come before implementation rather than after mistakes. \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/README.md b/README.md index 02b6155..7ef5a5c 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ On a **reproducible 10-incident suite** (real AWS + Azure resources, scored agai - **AWS Configuration settings tab** — admin-only editable tab in Settings for SQS Queue URL and AWS Region; shared org-wide via database-backed app config; includes an inline IAM permission checker per service - **Web UI** — React + Vite SPA served by FastAPI: - **Chat page** — streaming responses, collapsible tool call inspector, cost/latency card, stop button; supports `?prompt=` deeplink for pre-seeded investigations from the Monitoring dashboard + - **Replayable evidence pack** — an Evidence button opens the investigation's ranked hypotheses, each with cited evidence linked to the supporting tool call, the exact query/command that ran, and a deterministic AWS-console deeplink; copy-to-clipboard and JSON export. Served read-only from `GET /api/sessions/{id}/evidence` — see [apps/documentation/evidence_pack.md](apps/documentation/evidence_pack.md) - **Session history sidebar** — lists all past conversations; click any to resume with full tool call inspector and cost card restored; new chat and delete (soft) buttons - **Monitoring page** — live incident feed from event-driven detection; alert detail with investigate deeplink - **Dashboard** — session counts, tool call stats, cost/latency, context saved, activity chart, service breakdown, root cause distribution, recent sessions diff --git a/apps/backend/src/api/app.py b/apps/backend/src/api/app.py index eb82e12..7687826 100644 --- a/apps/backend/src/api/app.py +++ b/apps/backend/src/api/app.py @@ -20,6 +20,7 @@ auth, chat, dashboard, + evidence, history, integrations, monitoring, @@ -185,6 +186,7 @@ async def lifespan(_app: FastAPI): app.include_router(chat.router) app.include_router(sessions.router) +app.include_router(evidence.router) app.include_router(dashboard.router) app.include_router(history.router) app.include_router(auth.router) diff --git a/apps/backend/src/api/routers/evidence.py b/apps/backend/src/api/routers/evidence.py new file mode 100644 index 0000000..ffe9013 --- /dev/null +++ b/apps/backend/src/api/routers/evidence.py @@ -0,0 +1,20 @@ +"""Replayable evidence pack — read-only view over a session's persisted tool calls. + +Joins the investigation conclusion's hypotheses to the supporting tool calls that produced +them, surfacing the exact query/command that ran plus a deterministic console deeplink. +Uses the `/api/sessions` prefix so the SPA fallback never intercepts it. +""" + +from __future__ import annotations + +from fastapi import APIRouter +from opendevops_core.agent.db import db +from opendevops_core.agent.evidence import build_evidence_pack + +router = APIRouter(prefix="/api/sessions", tags=["evidence"]) + + +@router.get("/{session_id}/evidence") +async def get_evidence(session_id: str) -> dict: + raw = await db.get_evidence(session_id) + return build_evidence_pack(session_id, raw["aws_region"], raw["tool_calls"]) diff --git a/apps/backend/tests/test_api/test_evidence.py b/apps/backend/tests/test_api/test_evidence.py new file mode 100644 index 0000000..1e1d5ea --- /dev/null +++ b/apps/backend/tests/test_api/test_evidence.py @@ -0,0 +1,152 @@ +"""Tests for GET /api/sessions/{id}/evidence — the replayable evidence pack endpoint.""" + +from __future__ import annotations + +import os + +import pytest + +os.environ.setdefault("CHECKPOINT_BACKEND", "memory") +os.environ.setdefault("LLM_MODEL", "openrouter/anthropic/claude-3.5-sonnet") +os.environ.setdefault("LLM_API_KEY", "test-key") + + +async def _seed_investigation(session_id: str) -> None: + """Persist a session with supporting tool calls + a submit_investigation conclusion.""" + from opendevops_core.agent.db import db + + await db.upsert_session(session_id, "test-model", "us-east-1", title="Lambda throttling") + msg_id = await db.save_message(session_id, "assistant", "Investigation complete.") + + await db.save_tool_call( + session_id, + msg_id, + "get_metric_data", + { + "namespace": "AWS/Lambda", + "metric": "Throttles", + "dimensions": [{"Name": "FunctionName", "Value": "payment-fn"}], + }, + {"count": 1, "datapoints": [{"timestamp": "t", "value": 120}]}, + ) + await db.save_tool_call( + session_id, + msg_id, + "query_logs_insights", + { + "log_group": "/aws/lambda/payment-fn", + "query": "fields @timestamp, @message | filter @message like /Throttl/", + }, + {"results": []}, + ) + await db.save_tool_call( + session_id, + msg_id, + "run_bash_command", + {"command": "az monitor metrics list --resource payment-fn"}, + {"stdout": "ok"}, + ) + await db.save_tool_call( + session_id, + msg_id, + "submit_investigation", + { + "root_cause_category": "RESOURCE_LIMIT", + "root_cause_summary": "payment-fn hit its concurrency limit", + "hypotheses": [ + { + "hypothesis": "Concurrency limit reached on payment-fn", + "evidence": ["Throttles metric on payment-fn spiked to 120"], + "confidence": "HIGH", + }, + { + "hypothesis": "Downstream dependency slow", + "evidence": ["No corroborating evidence found"], + "confidence": "LOW", + }, + ], + "evidence": ["Throttles metric on payment-fn spiked to 120"], + "confidence": "HIGH", + }, + {}, + ) + + +@pytest.mark.asyncio +async def test_evidence_grouped_per_hypothesis_and_linked(): + from httpx import ASGITransport, AsyncClient + + from api.app import app + + session_id = "evid-test-grouped-1" + await _seed_investigation(session_id) + + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + r = await client.get(f"/api/sessions/{session_id}/evidence") + + assert r.status_code == 200 + pack = r.json() + + assert pack["has_conclusion"] is True + assert pack["aws_region"] == "us-east-1" + assert pack["root_cause_category"] == "RESOURCE_LIMIT" + + # Two ranked hypotheses, most likely first. + assert [h["hypothesis"] for h in pack["hypotheses"]] == [ + "Concurrency limit reached on payment-fn", + "Downstream dependency slow", + ] + + # submit_investigation is the conclusion, never a replay entry. + assert all(tc["tool"] != "submit_investigation" for tc in pack["tool_calls"]) + assert len(pack["tool_calls"]) == 3 + + # The top hypothesis's evidence links to the get_metric_data call that produced it. + top_ev = pack["hypotheses"][0]["evidence"][0] + linked_id = top_ev["tool_call_id"] + assert linked_id is not None + linked = next(tc for tc in pack["tool_calls"] if tc["id"] == linked_id) + assert linked["tool"] == "get_metric_data" + + +@pytest.mark.asyncio +async def test_evidence_exposes_command_and_console_deeplink(): + from httpx import ASGITransport, AsyncClient + + from api.app import app + + session_id = "evid-test-deeplink-2" + await _seed_investigation(session_id) + + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + r = await client.get(f"/api/sessions/{session_id}/evidence") + + pack = r.json() + by_tool = {tc["tool"]: tc for tc in pack["tool_calls"]} + + # Logs Insights: exact query verbatim + a deterministic console deeplink. + insights = by_tool["query_logs_insights"] + assert insights["command"] == "fields @timestamp, @message | filter @message like /Throttl/" + assert insights["console_url"].startswith("https://us-east-1.console.aws.amazon.com/cloudwatch") + assert "logs-insights" in insights["console_url"] + + # Azure / bash: the literal command is surfaced, no console deeplink. + bash = by_tool["run_bash_command"] + assert bash["command"] == "az monitor metrics list --resource payment-fn" + assert bash["console_url"] is None + + +@pytest.mark.asyncio +async def test_evidence_unknown_session_is_empty(): + from httpx import ASGITransport, AsyncClient + + from api.app import app + + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + r = await client.get("/api/sessions/no-such-session/evidence") + + assert r.status_code == 200 + pack = r.json() + assert pack["has_conclusion"] is False + assert pack["hypotheses"] == [] + assert pack["tool_calls"] == [] diff --git a/apps/backend/tests/test_tools/test_evidence_pack.py b/apps/backend/tests/test_tools/test_evidence_pack.py new file mode 100644 index 0000000..77f9bd7 --- /dev/null +++ b/apps/backend/tests/test_tools/test_evidence_pack.py @@ -0,0 +1,111 @@ +"""Unit tests for the evidence-pack builder, console deeplinks, and the ranked-hypotheses +schema extension to submit_investigation.""" + +from __future__ import annotations + +import inspect + +from opendevops_core.agent.evidence import ( + build_evidence_pack, + console_deeplink, + exact_command, +) + + +def test_console_deeplink_log_group_encoding(): + url = console_deeplink("get_log_events", {"log_group": "/aws/lambda/fn"}, "us-east-1") + assert url == ( + "https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1" + "#logsV2:log-groups/log-group/$252Faws$252Flambda$252Ffn" + ) + + +def test_console_deeplink_logs_insights_query_detail(): + url = console_deeplink( + "query_logs_insights", + {"log_group": "/aws/lambda/fn", "query": "fields @timestamp", "hours": 2}, + "eu-west-1", + ) + assert url.startswith( + "https://eu-west-1.console.aws.amazon.com/cloudwatch/home?region=eu-west-1" + "#logsV2:logs-insights$3FqueryDetail$3D" + ) + # Relative window encoded as seconds; query string escaped (space -> *20, @ -> *40). + assert "start~-7200" in url + assert "editorString~'fields*20*40timestamp" in url + assert "source~(~'*2faws*2flambda*2ffn)" in url + + +def test_console_deeplink_none_without_region(): + assert console_deeplink("get_log_events", {"log_group": "/x"}, None) is None + + +def test_exact_command_only_for_query_and_bash(): + assert exact_command("query_logs_insights", {"query": "stats count(*)"}) == "stats count(*)" + assert exact_command("run_bash_command", {"command": "az vm list"}) == "az vm list" + assert exact_command("get_alarms", {"state": "ALARM"}) is None + + +def _conclusion(hypotheses=None, evidence=None): + args = { + "root_cause_category": "RESOURCE_LIMIT", + "root_cause_summary": "throttled", + "confidence": "HIGH", + "evidence": evidence if evidence is not None else ["flat evidence"], + } + if hypotheses is not None: + args["hypotheses"] = hypotheses + return {"tool_name": "submit_investigation", "args": args, "id": "concl"} + + +def test_build_pack_links_evidence_to_tool_call(): + tool_calls = [ + { + "id": "tc-metric", + "tool_name": "get_metric_data", + "args": { + "namespace": "AWS/Lambda", + "metric": "Throttles", + "dimensions": [{"Name": "FunctionName", "Value": "payment-fn"}], + }, + "result": {"count": 1}, + }, + _conclusion( + hypotheses=[ + { + "hypothesis": "concurrency", + "evidence": ["payment-fn throttled hard"], + "confidence": "HIGH", + } + ] + ), + ] + pack = build_evidence_pack("s1", "us-east-1", tool_calls) + + assert pack["has_conclusion"] is True + assert len(pack["tool_calls"]) == 1 # conclusion excluded from replay + linked = pack["hypotheses"][0]["evidence"][0]["tool_call_id"] + assert linked == "tc-metric" + + +def test_build_pack_falls_back_to_flat_evidence_for_legacy(): + """Old investigations without `hypotheses` still produce one grouped hypothesis.""" + pack = build_evidence_pack("s2", "us-east-1", [_conclusion(evidence=["only flat"])]) + assert len(pack["hypotheses"]) == 1 + assert pack["hypotheses"][0]["evidence"][0]["text"] == "only flat" + assert pack["hypotheses"][0]["confidence"] == "HIGH" + + +def test_build_pack_no_conclusion(): + pack = build_evidence_pack("s3", "us-east-1", []) + assert pack["has_conclusion"] is False + assert pack["hypotheses"] == [] + + +def test_submit_investigation_has_hypotheses_param(): + from opendevops_core.tools.final_answer import submit_investigation + + sig = inspect.signature(submit_investigation) + assert "hypotheses" in sig.parameters + # Must stay a primitive list type so DeepAgents can infer the schema. + assert sig.parameters["hypotheses"].annotation == list[dict] diff --git a/apps/core/src/opendevops_core/agent/db/base.py b/apps/core/src/opendevops_core/agent/db/base.py index 8d76356..0fba554 100644 --- a/apps/core/src/opendevops_core/agent/db/base.py +++ b/apps/core/src/opendevops_core/agent/db/base.py @@ -86,6 +86,13 @@ async def get_session_model(self, session_id: str) -> str | None: @abstractmethod async def get_messages(self, session_id: str, org_id: str | None = None) -> list[dict]: ... + async def get_evidence(self, session_id: str, org_id: str | None = None) -> dict: + """Return the raw material for a session's evidence pack: + ``{"aws_region": str | None, "tool_calls": [...]}`` where each tool call carries + ``id``, ``tool_name``, ``args``, ``result``, ``error`` and ``created_at`` ordered + oldest-first. Read-only. Default returns empty — every backend overrides it.""" + return {"aws_region": None, "tool_calls": []} + @abstractmethod async def delete_session(self, session_id: str) -> None: ... diff --git a/apps/core/src/opendevops_core/agent/db/memory.py b/apps/core/src/opendevops_core/agent/db/memory.py index 5c2b384..7fdb5f2 100644 --- a/apps/core/src/opendevops_core/agent/db/memory.py +++ b/apps/core/src/opendevops_core/agent/db/memory.py @@ -220,6 +220,25 @@ async def get_messages(self, session_id: str, org_id: str | None = None) -> list result.append(item) return result + async def get_evidence(self, session_id: str, org_id: str | None = None) -> dict: + session = self._sessions.get(session_id) + if session is None or session.get("is_deleted"): + return {"aws_region": None, "tool_calls": []} + if org_id is not None and session.get("org_id") != org_id: + return {"aws_region": None, "tool_calls": []} + tool_calls = [ + { + "id": tc["id"], + "tool_name": tc["tool_name"], + "args": tc["args"], + "result": tc["result"], + "error": tc["error"], + "created_at": tc["created_at"], + } + for tc in self._tool_calls.get(session_id, []) + ] + return {"aws_region": session.get("aws_region"), "tool_calls": tool_calls} + async def delete_session(self, session_id: str) -> None: if session_id in self._sessions: self._sessions[session_id]["is_deleted"] = True diff --git a/apps/core/src/opendevops_core/agent/db/postgres.py b/apps/core/src/opendevops_core/agent/db/postgres.py index e8df9b5..1b5a337 100644 --- a/apps/core/src/opendevops_core/agent/db/postgres.py +++ b/apps/core/src/opendevops_core/agent/db/postgres.py @@ -319,6 +319,33 @@ async def get_messages(self, session_id: str, org_id: str | None = None) -> list result.append(item) return result + async def get_evidence(self, session_id: str, org_id: str | None = None) -> dict: + uid = uuid.UUID(session_id) + session = await self._fetchrow( + "SELECT is_deleted, org_id, aws_region FROM sessions WHERE id = %s", uid + ) + if session is None or session.get("is_deleted"): + return {"aws_region": None, "tool_calls": []} + if org_id is not None and str(session.get("org_id")) != str(org_id): + return {"aws_region": None, "tool_calls": []} + rows = await self._fetchall( + "SELECT id, tool_name, args, result, error, created_at FROM tool_calls " + "WHERE session_id = %s ORDER BY created_at ASC", + uid, + ) + tool_calls = [ + { + "id": str(r["id"]), + "tool_name": r["tool_name"], + "args": r["args"], + "result": r["result"], + "error": r["error"], + "created_at": r["created_at"].isoformat() if r["created_at"] else None, + } + for r in rows + ] + return {"aws_region": session.get("aws_region"), "tool_calls": tool_calls} + async def delete_session(self, session_id: str) -> None: await self._exec( "UPDATE sessions SET is_deleted = TRUE, deleted_at = NOW() WHERE id = %s", diff --git a/apps/core/src/opendevops_core/agent/db/sqlite.py b/apps/core/src/opendevops_core/agent/db/sqlite.py index 1e635fb..4152eb1 100644 --- a/apps/core/src/opendevops_core/agent/db/sqlite.py +++ b/apps/core/src/opendevops_core/agent/db/sqlite.py @@ -526,6 +526,33 @@ async def get_messages(self, session_id: str, org_id: str | None = None) -> list result.append(item) return result + async def get_evidence(self, session_id: str, org_id: str | None = None) -> dict: + # org_id ignored — SQLite is single-tenant (see upsert_session note). + session = await self._fetchone( + "SELECT is_deleted, aws_region FROM sessions WHERE id = ?", session_id + ) + if session is None or session.get("is_deleted"): + return {"aws_region": None, "tool_calls": []} + rows = await self._fetchall( + "SELECT id, tool_name, args, result, error, created_at FROM tool_calls " + "WHERE session_id = ? ORDER BY created_at ASC", + session_id, + ) + tool_calls = [ + { + "id": r["id"], + "tool_name": r["tool_name"], + "args": json.loads(r["args"]) if isinstance(r["args"], str) else r["args"], + "result": ( + json.loads(r["result"]) if isinstance(r["result"], str) else r["result"] + ), + "error": r["error"], + "created_at": r["created_at"], + } + for r in rows + ] + return {"aws_region": session.get("aws_region"), "tool_calls": tool_calls} + async def delete_session(self, session_id: str) -> None: await self._exec( "UPDATE sessions SET is_deleted = 1, " diff --git a/apps/core/src/opendevops_core/agent/evidence.py b/apps/core/src/opendevops_core/agent/evidence.py new file mode 100644 index 0000000..7933d09 --- /dev/null +++ b/apps/core/src/opendevops_core/agent/evidence.py @@ -0,0 +1,288 @@ +"""Read-side evidence-pack builder. + +Turns the verbatim tool-call rows already persisted for a session into a replayable +evidence pack: hypotheses grouped from the investigation conclusion, each evidence +item tied back to the tool call that produced it, the exact query/command that ran, +and a deterministic cloud-console deeplink where one applies. + +This is pure presentation logic — it reads existing data only and never mutates state. +""" + +from __future__ import annotations + +import urllib.parse +from typing import Any + +# Characters the AWS console leaves un-escaped inside its hash-object string tokens. +_CONSOLE_SAFE = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-._") + +# Map a tool name to the AWS service it belongs to, for grouping/labels. +_SERVICE_BY_TOOL: dict[str, str] = { + "get_alarms": "CloudWatch", + "get_alarm_history": "CloudWatch", + "get_metric_data": "CloudWatch", + "get_log_events": "CloudWatch Logs", + "describe_log_groups": "CloudWatch Logs", + "query_logs_insights": "CloudWatch Logs", + "lookup_cloudtrail_events": "CloudTrail", + "list_ecs_clusters": "ECS", + "list_ecs_services": "ECS", + "describe_ecs_service": "ECS", + "get_ecs_task_logs": "ECS", + "list_lambda_functions": "Lambda", + "get_lambda_function_config": "Lambda", + "get_lambda_error_rate": "Lambda", + "describe_ec2_instances": "EC2", + "get_ec2_system_status": "EC2", + "describe_rds_instances": "RDS", + "get_rds_events": "RDS", + "get_caller_identity": "IAM", + "get_iam_role_policies": "IAM", + "run_bash_command": "CLI", +} + + +def _console_quote(value: str) -> str: + """Encode a path segment for the CloudWatch console hash (e.g. a log-group name). + + The console double-encodes: standard percent-encoding, then `%` itself becomes `$25`. + `/aws/lambda/fn` -> `$252Faws$252Flambda$252Ffn`. + """ + return urllib.parse.quote(value, safe="").replace("%", "$25") + + +def _console_str(value: str) -> str: + """Serialize a string into an AWS console hash-object token (`'` prefix, `*xx` escapes).""" + out = ["'"] + for ch in value: + if ch in _CONSOLE_SAFE: + out.append(ch) + else: + out.extend(f"*{b:02x}" for b in ch.encode("utf-8")) + return "".join(out) + + +def _console_obj(obj: Any) -> str: + """Serialize a python value to the AWS console hash-object format used in deeplinks.""" + if isinstance(obj, bool): + return "true" if obj else "false" + if isinstance(obj, (int, float)): + return str(obj) + if isinstance(obj, str): + return _console_str(obj) + if isinstance(obj, dict): + # Objects carry a leading tilde; pairs are `key~value` joined by `~`. + return "~(" + "~".join(f"{k}~{_console_obj(v)}" for k, v in obj.items()) + ")" + if isinstance(obj, (list, tuple)): + # Arrays have no leading tilde of their own — each element is tilde-prefixed. + return "(" + "".join(f"~{_console_obj(v)}" for v in obj) + ")" + return "null" + + +def _cw_base(region: str) -> str: + return f"https://{region}.console.aws.amazon.com/cloudwatch/home?region={region}#" + + +def console_deeplink(tool_name: str, args: dict, region: str | None) -> str | None: + """Build a deterministic AWS-console deeplink from a tool call's stored args. + + Returns None when the tool has no meaningful console target (or no region). Azure / + bash calls have no deeplink — their replay value is the literal command string. + """ + if not region or not isinstance(args, dict): + return None + + if tool_name in ("get_log_events", "describe_log_groups"): + lg = args.get("log_group") or args.get("prefix") + if not lg: + return f"{_cw_base(region)}logsV2:log-groups" + return f"{_cw_base(region)}logsV2:log-groups/log-group/{_console_quote(lg)}" + + if tool_name == "query_logs_insights": + lg = args.get("log_group") + query = args.get("query", "") + hours = args.get("hours", 1) + try: + start = -int(hours) * 3600 + except (TypeError, ValueError): + start = -3600 + detail = { + "end": 0, + "start": start, + "timeType": "RELATIVE", + "unit": "seconds", + "editorString": query, + "isLiveTail": False, + "source": [lg] if lg else [], + } + return f"{_cw_base(region)}logsV2:logs-insights$3FqueryDetail$3D" + _console_obj(detail) + + if tool_name in ("get_alarms", "get_alarm_history"): + name = args.get("alarm_name") + if name: + return f"{_cw_base(region)}alarmsV2:alarm/{_console_quote(name)}" + return f"{_cw_base(region)}alarmsV2:" + + if tool_name == "get_metric_data": + namespace = args.get("namespace", "") + metric = args.get("metric", "") + dims = args.get("dimensions") or [] + series: list[Any] = [namespace, metric] + for d in dims: + if isinstance(d, dict) and "Name" in d and "Value" in d: + series.extend([d["Name"], d["Value"]]) + graph = {"metrics": [series], "region": region} + return f"{_cw_base(region)}metricsV2:graph={_console_obj(graph)}" + + if tool_name in ("get_lambda_function_config", "get_lambda_error_rate"): + name = args.get("name") + if name: + enc = urllib.parse.quote(name, safe="") + return f"https://{region}.console.aws.amazon.com/lambda/home?region={region}#/functions/{enc}" + + if tool_name == "get_ec2_system_status": + iid = args.get("instance_id") + if iid: + return ( + f"https://{region}.console.aws.amazon.com/ec2/home?region={region}" + f"#InstanceDetails:instanceId={urllib.parse.quote(iid, safe='')}" + ) + + if tool_name == "get_rds_events": + dbid = args.get("db_identifier") + if dbid: + return ( + f"https://{region}.console.aws.amazon.com/rds/home?region={region}" + f"#database:id={urllib.parse.quote(dbid, safe='')};is-cluster=false" + ) + + return None + + +def exact_command(tool_name: str, args: dict) -> str | None: + """The verbatim query/command a tool ran, when it stored one (Logs Insights, CLI).""" + if not isinstance(args, dict): + return None + if tool_name == "query_logs_insights": + return args.get("query") + if tool_name == "run_bash_command": + return args.get("command") + return None + + +def _identifiers(args: dict) -> list[str]: + """Distinctive string values from a tool call's args, used to link evidence text.""" + ids: list[str] = [] + if not isinstance(args, dict): + return ids + for value in args.values(): + if isinstance(value, str) and len(value) >= 4: + ids.append(value) + elif isinstance(value, list): + for item in value: + if isinstance(item, dict): + v = item.get("Value") + if isinstance(v, str) and len(v) >= 4: + ids.append(v) + return ids + + +def _replay_entry(index: int, tc: dict, region: str | None) -> dict: + tool_name = tc.get("tool_name", "") + args = tc.get("args") or {} + return { + "id": tc.get("id") or f"tc-{index}", + "index": index, + "tool": tool_name, + "service": _SERVICE_BY_TOOL.get(tool_name, "Other"), + "args": args, + "result": tc.get("result"), + "error": tc.get("error"), + "command": exact_command(tool_name, args), + "console_url": console_deeplink(tool_name, args, region), + "created_at": tc.get("created_at"), + } + + +def _match_tool_call(evidence_text: str, replay: list[dict]) -> str | None: + """Best-effort, deterministic link from an evidence string to the tool call id + that most likely produced it — by counting how many of the call's distinctive + arg identifiers appear in the evidence text.""" + if not evidence_text: + return None + text = evidence_text.lower() + best_id: str | None = None + best_score = 0 + for entry in replay: + score = sum(1 for ident in _identifiers(entry["args"]) if ident.lower() in text) + # A bare mention of the tool name is a weak signal, used only as a tie-breaker. + if entry["tool"] and entry["tool"].lower() in text: + score += 1 + if score > best_score: + best_score = score + best_id = entry["id"] + return best_id if best_score > 0 else None + + +def build_evidence_pack( + session_id: str, + aws_region: str | None, + tool_calls: list[dict], +) -> dict: + """Assemble the replayable evidence pack for a session. + + `tool_calls` are the raw persisted rows (tool_name, args, result, error, id, created_at), + ordered oldest-first. The latest `submit_investigation` row is the conclusion; the rest + are the supporting calls that get replayed and linked to each hypothesis's evidence. + """ + conclusion: dict | None = None + supporting: list[dict] = [] + for tc in tool_calls: + if tc.get("tool_name") == "submit_investigation": + conclusion = tc.get("args") or {} + else: + supporting.append(tc) + + replay = [_replay_entry(i, tc, aws_region) for i, tc in enumerate(supporting)] + + # Prefer the ranked hypotheses (new schema); fall back to the legacy single + # root-cause + flat evidence so older investigations still render. + raw_hypotheses: list[dict] = [] + if conclusion: + hyps = conclusion.get("hypotheses") + if isinstance(hyps, list) and hyps: + raw_hypotheses = [h for h in hyps if isinstance(h, dict)] + else: + raw_hypotheses = [ + { + "hypothesis": conclusion.get("root_cause_summary", ""), + "evidence": conclusion.get("evidence", []), + "confidence": conclusion.get("confidence", "LOW"), + } + ] + + hypotheses: list[dict] = [] + for h in raw_hypotheses: + ev_items = [] + for ev in h.get("evidence", []) or []: + if not isinstance(ev, str): + continue + ev_items.append({"text": ev, "tool_call_id": _match_tool_call(ev, replay)}) + hypotheses.append( + { + "hypothesis": h.get("hypothesis", ""), + "confidence": h.get("confidence", "LOW"), + "evidence": ev_items, + } + ) + + return { + "session_id": session_id, + "aws_region": aws_region, + "has_conclusion": conclusion is not None, + "root_cause_category": (conclusion or {}).get("root_cause_category"), + "root_cause_summary": (conclusion or {}).get("root_cause_summary", ""), + "confidence": (conclusion or {}).get("confidence"), + "hypotheses": hypotheses, + "tool_calls": replay, + } diff --git a/apps/core/src/opendevops_core/agent/prompts.py b/apps/core/src/opendevops_core/agent/prompts.py index b8a88bc..0472227 100644 --- a/apps/core/src/opendevops_core/agent/prompts.py +++ b/apps/core/src/opendevops_core/agent/prompts.py @@ -73,6 +73,8 @@ When you have gathered sufficient evidence and reached a conclusion, you MUST call the `submit_investigation` tool with all fields populated. Do not write a JSON block in free text — call the tool instead. This is required to complete the investigation. +Populate `hypotheses` with your ranked candidate explanations, most likely first — do not compress ambiguity into one confident story. Each entry is `{{"hypothesis": ..., "evidence": [...], "confidence": "HIGH"|"MEDIUM"|"LOW"}}`, and each evidence string should quote the concrete finding (a metric value, a log line, a CloudTrail event, an `az`/`kubectl` output) that you used to confirm or rule it out, so it can be traced back to the tool call that produced it. Your top hypothesis should agree with `root_cause_summary`, `root_cause_category`, and `confidence`. + ## Tone Be concise. Skip obvious observations. Go straight to anomalies. If you're uncertain, say so explicitly and reflect it in the confidence level. diff --git a/apps/core/src/opendevops_core/migrations/015_findings_hypotheses.sql b/apps/core/src/opendevops_core/migrations/015_findings_hypotheses.sql new file mode 100644 index 0000000..1244701 --- /dev/null +++ b/apps/core/src/opendevops_core/migrations/015_findings_hypotheses.sql @@ -0,0 +1,3 @@ +-- Ranked hypotheses for a finding: list of {hypothesis, evidence[], confidence}. +-- Backfills empty for existing rows so older investigations still load. +ALTER TABLE findings ADD COLUMN IF NOT EXISTS hypotheses JSONB NOT NULL DEFAULT '[]'; diff --git a/apps/core/src/opendevops_core/models/agent.py b/apps/core/src/opendevops_core/models/agent.py index 367b5a4..d6cdd47 100644 --- a/apps/core/src/opendevops_core/models/agent.py +++ b/apps/core/src/opendevops_core/models/agent.py @@ -37,6 +37,9 @@ class Finding(BaseModel): class InvestigationResult(BaseModel): root_cause_category: RootCauseCategory = RootCauseCategory.UNKNOWN root_cause_summary: str = "" + # Ranked hypotheses (most likely first), each with its own cited evidence and + # confidence. The flat `evidence` below is retained for backward compatibility. + hypotheses: list[Finding] = Field(default_factory=list) evidence: list[str] = Field(default_factory=list) mitigation_steps: list[str] = Field(default_factory=list) validation_steps: list[str] = Field(default_factory=list) diff --git a/apps/core/src/opendevops_core/tools/final_answer.py b/apps/core/src/opendevops_core/tools/final_answer.py index bdf8d37..b1f2116 100644 --- a/apps/core/src/opendevops_core/tools/final_answer.py +++ b/apps/core/src/opendevops_core/tools/final_answer.py @@ -16,6 +16,7 @@ def submit_investigation( root_cause_category: VALID_CATEGORIES, root_cause_summary: str, + hypotheses: list[dict], evidence: list[str], mitigation_steps: list[str], validation_steps: list[str], @@ -28,6 +29,17 @@ def submit_investigation( gathered sufficient evidence and reached a conclusion. Do not output a JSON block in free text — call this tool instead. + hypotheses: the ranked candidate explanations, most likely first — do not compress + ambiguity into one confident story. Each item is a dict + {"hypothesis": str, "evidence": list[str], "confidence": "HIGH"|"MEDIUM"|"LOW"}. + Each evidence string should quote the concrete finding (a metric value, a log line, + a CloudTrail event, an `az`/`kubectl` command's output) that supports that hypothesis, + so it can be traced back to the tool call that produced it. The top hypothesis should + match root_cause_summary / root_cause_category / confidence. + + evidence: a flat list of the key findings overall (kept for backward compatibility; + prefer attaching evidence to the relevant hypothesis above). + follow_up_questions: 3 short drill-down questions the user might want to ask next, e.g. ["What caused the spike at 14:32?", "Are retries configured on the Lambda?", "Has this happened before this week?"]. diff --git a/apps/documentation/cli.md b/apps/documentation/cli.md index f56f9ce..b01e287 100644 --- a/apps/documentation/cli.md +++ b/apps/documentation/cli.md @@ -40,16 +40,20 @@ uv run devops-agent investigate "ECS tasks keep crashing" --json **Output (Rich panel):** - Root cause category and summary - Confidence level (HIGH / MEDIUM / LOW) +- Ranked hypotheses (most likely first, each with its own cited evidence and confidence) - Evidence list - Mitigation steps - Validation steps - Services affected - Recommended follow-up -**JSON output fields:** `root_cause_category`, `root_cause_summary`, `evidence`, -`mitigation_steps`, `validation_steps`, `confidence`, `services_affected`, +**JSON output fields:** `root_cause_category`, `root_cause_summary`, `hypotheses`, +`evidence`, `mitigation_steps`, `validation_steps`, `confidence`, `services_affected`, `recommended_follow_up`, `tool_calls_made`. +`hypotheses` is a list of `{"hypothesis", "evidence": [...], "confidence"}` objects; the +flat `evidence` field is kept for backward compatibility. + --- ## `ask` diff --git a/apps/documentation/evidence_pack.md b/apps/documentation/evidence_pack.md new file mode 100644 index 0000000..f5aaf84 --- /dev/null +++ b/apps/documentation/evidence_pack.md @@ -0,0 +1,94 @@ +# Replayable Evidence Pack + +## What it does + +Every investigation ends with the agent calling `submit_investigation`, whose arguments +are the structured conclusion. The evidence pack is a read-only view over that conclusion +and the tool calls that produced it: it groups the investigation's **ranked hypotheses**, +ties each piece of cited evidence back to the tool call that produced it, surfaces the +exact query/command that ran, and builds a deterministic AWS-console deeplink so a human +can reproduce the agent's steps. + +It is pure presentation — it reads already-persisted data and never mutates state, changes +the SSE contract, or touches the bash allowlist. + +## Endpoint + +``` +GET /api/sessions/{session_id}/evidence +``` + +The `/api/` prefix keeps the SPA fallback catch-all from intercepting the request. The +router (`src/api/routers/evidence.py`) calls `db.get_evidence(session_id)` for the region + +raw tool-call rows, then `build_evidence_pack()` (in `opendevops_core/agent/evidence.py`) +assembles the response. The conclusion is read from the `tool_calls` row whose +`tool_name = 'submit_investigation'` — **not** from the `findings` table, which is currently +an unwritten placeholder. + +### Response shape + +```json +{ + "session_id": "…", + "aws_region": "us-east-1", + "has_conclusion": true, + "root_cause_category": "RESOURCE_LIMIT", + "root_cause_summary": "…", + "confidence": "HIGH", + "hypotheses": [ + { + "hypothesis": "…", + "confidence": "HIGH", + "evidence": [ + { "text": "…concrete finding…", "tool_call_id": "tc-3" } + ] + } + ], + "tool_calls": [ + { + "id": "tc-3", + "index": 3, + "tool": "query_logs_insights", + "service": "CloudWatch Logs", + "args": { "…": "…" }, + "result": "…", + "error": null, + "command": "fields @timestamp, @message | filter …", + "console_url": "https://us-east-1.console.aws.amazon.com/cloudwatch/home#…", + "created_at": "…" + } + ] +} +``` + +## How linking, commands, and deeplinks work + +- **Evidence → tool call** — each evidence string is matched to the supporting tool call by + a deterministic best-effort substring count over the call's distinctive arg identifiers + (the tool name is only a tie-breaker). `tool_call_id` is `null` when nothing matches. +- **Exact command** — `command` is populated only for the tools that ran a verbatim string: + the Logs Insights query (`query_logs_insights`) and the bash CLI command + (`run_bash_command`, e.g. `aws`/`az`/`kubectl`). Azure has no console deeplink by design, + so the command string **is** the replay artifact. +- **Console deeplink** — `console_url` is a deterministic AWS-console URL for CloudWatch + alarms/metrics/logs, Logs Insights, Lambda, EC2, and RDS calls; `null` when the tool has + no console target or no region is known. + +## Backward compatibility + +When a conclusion has no `hypotheses` (investigations from before the ranked-hypotheses +schema), the builder synthesizes a single hypothesis from the legacy `root_cause_summary` + +flat `evidence[]` so older sessions still render. + +## UI + +A **Evidence** button in the chat header opens a slide-over `EvidencePanel` that renders the +grouped hypotheses and replay cards, with copy-to-clipboard and a full JSON export. See +[ui.md](ui.md). + +## Testing it + +```bash +uv run pytest tests/test_api/test_evidence.py # endpoint shape, grouping, linking, deeplinks +uv run pytest tests/test_tools/test_evidence_pack.py # builder, console encoder, schema +``` diff --git a/apps/documentation/schema.md b/apps/documentation/schema.md index 0d4d91c..8725e80 100644 --- a/apps/documentation/schema.md +++ b/apps/documentation/schema.md @@ -133,7 +133,7 @@ Top-level tenant for multi-org / SaaS support. Table exists in the schema but is Per-org named AWS connection configs for multi-account support. Table exists but not yet wired up. ### `findings` *(future — Phase 2)* -Structured root-cause analysis rows extracted from agent final answers. Table exists but not yet populated. +Structured root-cause analysis rows extracted from agent final answers. Table exists but not yet populated. Migration `015` adds a `hypotheses JSONB` column (default `[]`) for the ranked-hypotheses conclusion schema; the replayable evidence pack currently reads the conclusion straight from `tool_calls` (the `submit_investigation` row) rather than from this table. ### `api_keys` *(future — Phase 3)* Hashed API keys for programmatic access. Table exists but not yet implemented. diff --git a/apps/documentation/ui.md b/apps/documentation/ui.md index 5483de0..f075edc 100644 --- a/apps/documentation/ui.md +++ b/apps/documentation/ui.md @@ -34,6 +34,13 @@ and see the full arguments and result JSON. - Total cost for the turn - Latency in milliseconds +**Evidence panel** — an **Evidence** button in the chat header (shown once the session +has messages) opens a slide-over panel rendering the session's replayable evidence pack +(`GET /api/sessions/{id}/evidence`). It groups the investigation's ranked hypotheses, each +with its cited evidence linked to the supporting tool call, the verbatim query/command that +ran, and a deterministic AWS-console deeplink. Replay cards offer copy-to-clipboard and a +full JSON export. See [evidence_pack.md](evidence_pack.md). + **Session continuity** — the session ID is stored in the URL. Refreshing the page or sharing the URL resumes the same conversation from where it left off (SQLite/Postgres backends only — memory backend loses history on restart). diff --git a/apps/frontend/src/components/EvidencePanel.tsx b/apps/frontend/src/components/EvidencePanel.tsx new file mode 100644 index 0000000..4f68836 --- /dev/null +++ b/apps/frontend/src/components/EvidencePanel.tsx @@ -0,0 +1,213 @@ +import { useEffect, useState } from 'react'; +import { toast } from 'sonner'; +import { X, Copy, ExternalLink, Download, FlaskConical, Wrench, Terminal } from 'lucide-react'; +import { fetchEvidence } from '../lib/api'; +import { cn, fmtJson } from '../lib/utils'; +import type { EvidencePack, EvidenceToolCall } from '../types'; + +interface Props { + sessionId: string; + onClose: () => void; +} + +const CONFIDENCE_STYLES: Record = { + HIGH: 'bg-emerald-100 text-emerald-700 dark:bg-emerald-500/15 dark:text-emerald-400', + MEDIUM: 'bg-amber-100 text-amber-700 dark:bg-amber-500/15 dark:text-amber-400', + LOW: 'bg-gray-100 text-gray-600 dark:bg-gray-500/15 dark:text-gray-400', +}; + +function copy(text: string, label = 'Copied') { + navigator.clipboard.writeText(text).then( + () => toast.success(label), + () => toast.error('Copy failed'), + ); +} + +function ToolCallCard({ tc }: { tc: EvidenceToolCall }) { + return ( +
+
+ + {tc.tool} + {tc.service} + {tc.error && error} +
+ {tc.console_url && ( + + Console + + )} +
+
+ + {tc.command != null && ( +
+
+ + + Exact query / command + + +
+ 
+            {tc.command}
+
+
+ )} + +
+ Input + 
+          {fmtJson(tc.args)}
+
+ {tc.result != null && ( + <> + Output + 
+              {fmtJson(tc.result)}
+
+ + )} +
+
+ ); +} + +export default function EvidencePanel({ sessionId, onClose }: Props) { + const [pack, setPack] = useState(null); + const [error, setError] = useState(false); + + useEffect(() => { + let live = true; + fetchEvidence(sessionId) + .then(p => { if (live) setPack(p); }) + .catch(() => { if (live) setError(true); }); + return () => { live = false; }; + }, [sessionId]); + + const exportJson = () => { + if (!pack) return; + const blob = new Blob([JSON.stringify(pack, null, 2)], { type: 'application/json' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `evidence-${sessionId}.json`; + a.click(); + URL.revokeObjectURL(url); + }; + + const tcById = (id: string | null): EvidenceToolCall | undefined => + id ? pack?.tool_calls.find(t => t.id === id) : undefined; + + const scrollToCall = (id: string) => { + const el = document.getElementById(`evidence-tc-${id}`); + if (el) { + el.scrollIntoView({ behavior: 'smooth', block: 'center' }); + el.classList.add('ring-2', 'ring-indigo-400'); + setTimeout(() => el.classList.remove('ring-2', 'ring-indigo-400'), 1200); + } + }; + + return ( +
+
+
+
+ + Evidence pack + {pack?.aws_region && ( + {pack.aws_region} + )} +
+ + +
+
+ +
+ {error &&

Failed to load evidence.

} + {!error && !pack &&

Loading…

} + + {pack && !pack.has_conclusion && ( +

+ No completed investigation in this session yet — the evidence pack appears once the agent + submits its conclusion. +

+ )} + + {pack && pack.has_conclusion && ( + <> + {/* Hypotheses */} +
+

+ Ranked hypotheses +

+ {pack.hypotheses.map((h, i) => ( +
+
+ #{i + 1} +

{h.hypothesis}

+ + {h.confidence} + +
+ {h.evidence.length > 0 && ( +
    + {h.evidence.map((ev, j) => { + const tc = tcById(ev.tool_call_id); + return ( +
  • + • {ev.text} + {tc && ( + + )} +
    • + ); + })} +
+ )} +
+ ))} +
+ + {/* Replay */} +
+

+ Replay — {pack.tool_calls.length} tool call{pack.tool_calls.length === 1 ? '' : 's'} +

+ {pack.tool_calls.map(tc => )} +
+ + )} +
+
+
+ ); +} diff --git a/apps/frontend/src/lib/api.ts b/apps/frontend/src/lib/api.ts index 06398cf..8c5f9b3 100644 --- a/apps/frontend/src/lib/api.ts +++ b/apps/frontend/src/lib/api.ts @@ -1,4 +1,4 @@ -import type { Session, MessageRecord, HistoryStats, SearchResult, User, Alert, ServiceStatus } from '../types'; +import type { Session, MessageRecord, HistoryStats, SearchResult, User, Alert, ServiceStatus, EvidencePack } from '../types'; export function getAuthToken(): string | null { return localStorage.getItem('auth-token'); @@ -29,6 +29,12 @@ export async function deleteSession(sessionId: string): Promise { await apiFetch(`/sessions/${sessionId}`, { method: 'DELETE' }); } +export async function fetchEvidence(sessionId: string): Promise { + const res = await apiFetch(`/api/sessions/${sessionId}/evidence`); + if (!res.ok) throw new Error('Failed to load evidence'); + return res.json() as Promise; +} + export async function renameSession(sessionId: string, title: string): Promise { await apiFetch(`/sessions/${sessionId}`, { method: 'PATCH', diff --git a/apps/frontend/src/pages/ChatPage.tsx b/apps/frontend/src/pages/ChatPage.tsx index 2589d39..3b0f459 100644 --- a/apps/frontend/src/pages/ChatPage.tsx +++ b/apps/frontend/src/pages/ChatPage.tsx @@ -1,11 +1,12 @@ import { useState, useEffect, useRef } from 'react'; import { useParams, Navigate, useSearchParams } from 'react-router-dom'; import { toast } from 'sonner'; -import { Plus } from 'lucide-react'; +import { Plus, FlaskConical } from 'lucide-react'; import EmptyState from '../components/EmptyState'; import UserMessage from '../components/UserMessage'; import AgentMessage from '../components/AgentMessage'; import InputArea from '../components/InputArea'; +import EvidencePanel from '../components/EvidencePanel'; import { fetchMessages, getAuthToken } from '../lib/api'; import type { Message, AgentMessage as AgentMsg, MessageRecord } from '../types'; @@ -41,6 +42,7 @@ export default function ChatPage({ onSessionsChange, onNew }: Props) { const [messages, setMessages] = useState([]); const [busy, setBusy] = useState(false); const [followUpQuestions, setFollowUpQuestions] = useState([]); + const [showEvidence, setShowEvidence] = useState(false); const abortRef = useRef(null); const bottomRef = useRef(null); const autoPromptFired = useRef(false); @@ -189,15 +191,30 @@ export default function ChatPage({ onSessionsChange, onNew }: Props) { Active
- +
+ {messages.length > 0 && ( + + )} + +
+ {showEvidence && sessionId && ( + setShowEvidence(false)} /> + )} + {/* Messages */}
{messages.length === 0 ? ( diff --git a/apps/frontend/src/types.ts b/apps/frontend/src/types.ts index 2304c2b..b36fd2a 100644 --- a/apps/frontend/src/types.ts +++ b/apps/frontend/src/types.ts @@ -59,6 +59,43 @@ export interface AgentMessage { export type Message = UserMessage | AgentMessage; +// ── Replayable evidence pack ──────────────────────────────────────────────── + +export interface EvidenceToolCall { + id: string; + index: number; + tool: string; + service: string; + args: unknown; + result: unknown; + error: string | null; + command: string | null; + console_url: string | null; + created_at: string | null; +} + +export interface EvidenceItem { + text: string; + tool_call_id: string | null; +} + +export interface EvidenceHypothesis { + hypothesis: string; + confidence: string; + evidence: EvidenceItem[]; +} + +export interface EvidencePack { + session_id: string; + aws_region: string | null; + has_conclusion: boolean; + root_cause_category: string | null; + root_cause_summary: string; + confidence: string | null; + hypotheses: EvidenceHypothesis[]; + tool_calls: EvidenceToolCall[]; +} + export interface HistoryAlarm { alarm_name: string; session_count: number;