diff --git a/.dockerignore b/.dockerignore index 7649a13..8332d07 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,20 +1,24 @@ -.venv/ -__pycache__/ -*.pyc -*.pyo +apps/backend/.venv/ +apps/backend/__pycache__/ +apps/backend/*.pyc +apps/backend/*.pyo +apps/backend/tests/ +apps/backend/*.db +apps/backend/data/ + +apps/frontend/node_modules/ +apps/frontend/dist/ +apps/frontend/.vite/ + .env .env.* !.env.example -frontend/node_modules/ -frontend/dist/ -frontend/.vite/ - .git/ .github/ -tests/ -docs/ +apps/documentation/ design-system/ +demos/ PLAN.md *.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07a2746..8a71d20 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,9 @@ on: jobs: lint-and-test: runs-on: ubuntu-latest + defaults: + run: + working-directory: apps/backend steps: - name: Checkout diff --git a/CLAUDE.md b/CLAUDE.md index 6d4e10d..b7e1f58 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,16 +6,51 @@ OpenDevOps Agent is an open-source AWS incident investigation tool powered by an --- +## Repo Structure + +``` +apps/ + core/ Installable package `opendevops-core` — the shared agent brain: + src/opendevops_core/{agent, tools, providers, models, skills, + integrations, migrations, config.py}. Has no web/CLI layer. + Consumed by the OSS backend (and, later, the SaaS product repo). + backend/ OSS web app + CLI — src/{api, cli, config, mcp_server.py}, tests/, + scripts/, pyproject.toml. Depends on opendevops-core via a uv path source. + frontend/ React/Vite UI — src/, package.json, vite.config.ts + documentation/ Markdown docs (future hosted docs site) +deployment/ + docker-compose/ docker-compose.yml (PostgreSQL + backend + frontend) + railway/ Dockerfile.railway + railway.toml (combined single-image deploy) +design-system/ Cross-cutting design reference (colors, typography, UI kits) +demos/ Reproducible AWS incident scripts for local testing +Makefile Root convenience targets — wraps `cd apps/backend && uv run ...` +``` + +All Python commands run from `apps/backend/` (or via `make ` at repo root). +`uv sync` from `apps/backend/` installs `opendevops-core` editable from `../core`, so edits +to core are live without republishing. + +### Core vs app boundary +- **Reusable agent logic lives in `apps/core` (`opendevops_core`)** — the DeepAgents loop, + tools, providers, models, skills, integrations, DB backends, and baseline migrations. +- **Web/CLI-only code stays in `apps/backend`** — FastAPI routers, auth, the CLI, `mcp_server.py`. +- **Config injection:** core reads settings through the `settings` proxy in + `opendevops_core/config.py`, which delegates to whatever instance the host app registers via + `configure()`. The OSS app's `Settings(CoreSettings)` (in `apps/backend/src/config/appsettings.py`) + adds web/auth-only fields (e.g. `jwt_*`) and calls `configure(settings)` at startup. + +--- + ## Automatic Behavior Rules **Always do these when making changes:** -- **New env var:** add to both `src/config/appsettings.py` (source of truth, Pydantic field with default) AND `.env.example` (with a comment). Never read env vars directly — always go through `settings`. -- **New DB column or table:** create a new numbered migration in `migrations/` (e.g. `006_name.sql`). Never add columns inline in Python code. -- **New tool:** add it to `ALL_TOOLS` in `src/agent/core.py`. Tool functions must be plain synchronous Python functions — DeepAgents infers the JSON schema from type hints and docstrings. -- **New API route that matches a React Router path:** prefix it with `/api/` to avoid the SPA fallback conflict. The `/{full_path:path}` catch-all in `app.py` intercepts any GET that matches a registered FastAPI route first. -- **New skill:** drop a `SKILL.md` file into `src/skills//SKILL.md`. It is picked up automatically at startup — no code changes needed. Use the frontmatter format (`name`, `description`) from the existing `lambda-throttling` skill. -- **Docs sync:** if a feature has a corresponding file in `docs/`, update it when the feature changes. The `docs/` folder is the public documentation source. +- **New env var:** if core reads it, add the Pydantic field to `CoreSettings` in `apps/core/src/opendevops_core/config.py`; if it's web/auth-only, add it to `Settings(CoreSettings)` in `apps/backend/src/config/appsettings.py`. Either way mirror it in `.env.example` (with a comment). Never read env vars directly — always go through `settings`. +- **New DB column or table:** add a new numbered migration. Core-domain schema (tables core code reads/writes) goes in `apps/core/src/opendevops_core/migrations/` (e.g. `014_name.sql`); OSS-app-only schema goes in `apps/backend/migrations/`. The runner applies core-then-app, tracked in the `schema_migrations(source, version)` ledger. Never add columns inline in Python code. +- **New tool:** add it to `ALL_TOOLS` in `apps/core/src/opendevops_core/agent/core.py`. Tool functions must be plain synchronous Python functions — DeepAgents infers the JSON schema from type hints and docstrings. +- **New API route that matches a React Router path:** prefix it with `/api/` to avoid the SPA fallback conflict. The `/{full_path:path}` catch-all in `apps/backend/src/api/app.py` intercepts any GET that matches a registered FastAPI route first. +- **New skill:** drop a `SKILL.md` file into `apps/core/src/opendevops_core/skills//SKILL.md`. It is picked up automatically at startup (and bundled into the core wheel) — no code changes needed. Use the frontmatter format (`name`, `description`) from the existing `lambda-throttling` skill. +- **Docs sync:** if a feature has a corresponding file in `apps/documentation/`, update it when the feature changes. The `apps/documentation/` folder is the public documentation source. --- @@ -23,41 +58,41 @@ OpenDevOps Agent is an open-source AWS incident investigation tool powered by an ```bash # Install / update dependencies -uv sync +cd apps/backend && uv sync # or: make install -# Development server — FastAPI with hot reload (equivalent to uv run python src/run.py) -uv run dev +# Development server — FastAPI with hot reload +cd apps/backend && uv run dev # or: make dev # Production web UI (FastAPI backend + serves built frontend, no reload) -uv run devops-agent ui +cd apps/backend && uv run devops-agent ui # or: make ui # Apply SQL migrations to PostgreSQL (requires CHECKPOINT_BACKEND=postgres + DATABASE_URL) -uv run migrate +cd apps/backend && uv run migrate # or: make migrate # CLI investigation -uv run devops-agent investigate "Lambda high error rate on payment service" -uv run devops-agent ask "Why would an ECS task OOM?" -uv run devops-agent report --days 7 +cd apps/backend && uv run devops-agent investigate "Lambda high error rate on payment service" +cd apps/backend && uv run devops-agent ask "Why would an ECS task OOM?" +cd apps/backend && uv run devops-agent report --days 7 # MCP server -uv run devops-agent mcp # stdio transport (Claude Desktop, Cursor) -uv run devops-agent mcp --http # HTTP+SSE transport, port 8001 +cd apps/backend && uv run devops-agent mcp # stdio transport (Claude Desktop, Cursor) +cd apps/backend && uv run devops-agent mcp --http # HTTP+SSE transport, port 8001 # Tests -uv run pytest +cd apps/backend && uv run pytest # or: make test -# Lint / format -uv run ruff check src/ -uv run ruff format src/ +# Lint / format (covers both the app and the core package) +cd apps/backend && uv run ruff check src/ ../core/src +cd apps/backend && uv run ruff format src/ ../core/src # or: make lint / make lint-fix # Full stack with PostgreSQL (Docker Compose) -docker compose up +docker compose -f deployment/docker-compose/docker-compose.yml up --build # or: make compose-up # Frontend dev server (port 5173, proxies API to localhost:8000) -cd frontend && npm install && npm run dev +cd apps/frontend && npm install && npm run dev # or: make frontend-dev -# Frontend production build (output to frontend/dist/ — served by FastAPI) -cd frontend && npm run build +# Frontend production build (output to apps/frontend/dist/ — served by FastAPI) +cd apps/frontend && npm run build # or: make frontend-build ``` --- @@ -68,7 +103,7 @@ Everything below is built and working in the codebase: ### Agent & Tools - **Framework:** DeepAgents (`create_deep_agent`) wrapping a LangGraph ReAct loop. `ChatLiteLLM` as the model interface — supports OpenRouter, Anthropic, OpenAI, Groq, Ollama, and any OpenAI-compatible endpoint via a single `LLM_MODEL` env var. -- **27 tools total** registered in `ALL_TOOLS` in `src/agent/core.py`: +- **27 tools total** registered in `ALL_TOOLS` in `apps/core/src/opendevops_core/agent/core.py`: - CloudWatch (6): `get_alarms`, `get_alarm_history`, `get_metric_data`, `get_log_events`, `describe_log_groups`, `query_logs_insights` - CloudTrail (2): trail events + event lookup - ECS (4): clusters, services, service detail, tasks @@ -82,19 +117,20 @@ Everything below is built and working in the codebase: - Final answer (1): `submit_investigation` — structured output required to end every investigation - **Tool response capping:** `with_cap()` wraps every tool at startup when `TOOL_RESPONSE_MAX_CHARS > 0`; truncates oversized responses and appends a notice to the LLM. - **Tool caching:** `@tool_cached` — in-process TTL LRU cache (2-min TTL, 256 entries max); cache key includes function name + AWS profile + region. -- **Skills system:** one skill ships (`lambda-throttling`). The system prompt is built at import time by scanning `src/skills/*/SKILL.md` — skill names are injected; full content is loaded lazily when the agent calls `use_skill(name)`. +- **Skills system:** several skills ship (e.g. `lambda-throttling`). The system prompt is built at import time by scanning `apps/core/src/opendevops_core/skills/*/SKILL.md` — skill names are injected; full content is loaded lazily when the agent calls `use_skill(name)`. - **Summarization:** `maybe_summarize()` runs before each agent call; compacts old messages when total chars exceed `SUMMARIZATION_THRESHOLD_CHARS`; tracks the event in `usage_events` with `metadata.summarization=True`. - **Cancellation:** `DELETE /chat/{session_id}` sets an `asyncio.Event` that stops the streaming loop at the next chunk boundary. ### Storage - Three backends all implementing `DatabaseBackend` ABC: `memory` (default, zero config), `sqlite` (aiosqlite + LangGraph SQLite checkpointer), `postgres` (psycopg3 async + `AsyncPostgresSaver`). -- LangGraph checkpointer tables are created automatically by `AsyncPostgresSaver.setup()`. Application tables come from `migrations/001–005_*.sql`. +- LangGraph checkpointer tables are created automatically by `AsyncPostgresSaver.setup()`. Application tables come from the bundled core migrations in `apps/core/src/opendevops_core/migrations/*.sql`. - Schema tables: `organizations`, `users`, `aws_profiles`, `sessions`, `messages`, `tool_calls`, `usage_events`, `findings`, `api_keys`, `alerts`. - Soft delete is in place on sessions (`is_deleted`, `deleted_at` from migration 002). +- Multi-tenant scoping: `upsert_session`, `list_sessions`, and `get_messages` accept an optional `org_id` (default `None` = unscoped). Postgres enforces it (filters lists, denies cross-org `get_messages`); memory scopes it; sqlite accepts-but-ignores it (single-tenant). `None` preserves single-tenant OSS behavior — the scoping exists for downstream multi-tenant consumers (the SaaS product). ### API - FastAPI SSE endpoint at `POST /chat`; streams `token`, `tool_status`, `tool_call`, `error`, `done`, `cancelled` events. -- SPA fallback: `GET /{full_path:path}` returns `frontend/dist/index.html` so React Router works on refresh. +- SPA fallback: `GET /{full_path:path}` returns `apps/frontend/dist/index.html` so React Router works on refresh. - All routes that could conflict with React Router paths use the `/api/` prefix: `/api/settings`, `/api/users`, `/api/history`, `/api/monitoring`, `/api/init`. - Auth: optional JWT (HS256 via python-jose + bcrypt). Disabled when `JWT_SECRET` is unset — `get_current_user()` returns `None` in that case, meaning all routes are public. @@ -125,12 +161,12 @@ Everything below is built and working in the codebase: Incomplete items from the README roadmap (do not mark complete here — update README when done): - **Custom tools via URL** — register external tools by OpenAPI endpoint; agent discovers them alongside built-in tools -- **Bash sandbox Phase 2** — throwaway Docker container per command: `--network none`, read-only FS, non-root, `--memory 256m`, killed immediately after; current Phase 1 (subprocess allowlist) is in `src/tools/bash_tool.py` +- **Bash sandbox Phase 2** — throwaway Docker container per command: `--network none`, read-only FS, non-root, `--memory 256m`, killed immediately after; current Phase 1 (subprocess allowlist) is in `apps/core/src/opendevops_core/tools/bash_tool.py` - **Optimize tool loading** — pass only contextually relevant tools instead of the full 27-tool set per invocation - **OpenTelemetry traces** — spans for agent steps, tool call latency, token usage; OTLP export - **Follow-up question suggestions** — add `follow_up_questions: list[str]` to `submit_investigation` schema (same call, no extra LLM cost); surface as chips in the chat UI after investigation completes - **Session / user feedback loop** — thumbs up/down on investigations; `feedback` column in `usage_events` (needs migration 006) -- **Slack Integration UI** — Slack backend is fully implemented (`src/integrations/slack_webhook.py`); Settings → Integrations "Connect" button is currently a non-functional stub +- **Slack Integration UI** — Slack backend is fully implemented (`apps/core/src/opendevops_core/integrations/slack_webhook.py`); Settings → Integrations "Connect" button is currently a non-functional stub - **Session rename** — `PATCH /sessions/{id}` + inline edit in sidebar three-dot menu - **Multi-account AWS** — `aws_profiles` table already in schema; needs Settings UI + per-session profile selector - **Knowledge base** — attach runbooks, post-mortems, architecture docs beyond the skills system @@ -143,7 +179,7 @@ Incomplete items from the README roadmap (do not mark complete here — update R |---|---| | **SSE event types:** `token`, `tool_status`, `tool_call`, `error`, `done`, `cancelled` | Frontend `ChatPage.tsx` switches on these exact strings. Renaming or adding new required fields is a breaking change. | | **Tool function signatures** | DeepAgents infers JSON schema from Python type hints + docstrings. Adding `*args`, `**kwargs`, removing type hints, or making parameters non-primitive breaks schema inference silently. | -| **`DatabaseBackend` ABC** (`src/agent/db/base.py`) | All three backends must implement the same interface. Adding a method requires implementing it in all three backends plus `memory.py` defaults. | +| **`DatabaseBackend` ABC** (`apps/core/src/opendevops_core/agent/db/base.py`) | All three backends must implement the same interface. Adding a method requires implementing it in all three backends plus `memory.py` defaults. | | **LangGraph checkpointer wiring** | The checkpointer is passed into `create_deep_agent()` and drives session continuity via `thread_id = session_id`. Do not write messages to the DB outside `save_*` calls or bypass the checkpointer. | | **Agent framework (DeepAgents + LangGraph)** | The ReAct loop, tool dispatch, checkpointing, and `recursion_limit` contract all depend on this. Do not swap. | | **DB schema migrations** | Tables have a defined shape. New columns need a new file in `migrations/`. Never add columns inline in Python or modify existing migration files. | @@ -173,13 +209,13 @@ Incomplete items from the README roadmap (do not mark complete here — update R | Linting | `ruff>=0.4.0` (line-length 100, Python 3.11 target, `asyncio_mode = "auto"`) | | Package manager | **uv** — always `uv run` and `uv add`, never bare `pip` | | Frontend | React 18, TypeScript, Vite, Tailwind CSS 3, `@tailwindcss/typography` | -| Font | Inter Variable (Google Fonts) — full system fallback stack in `tailwind.config.js` | +| Font | Inter Variable (Google Fonts) — full system fallback stack in `apps/frontend/tailwind.config.js` | --- ## Environment Variables -All read from `src/config/appsettings.py` (Pydantic Settings — single source of truth). `.env.example` mirrors every variable. +Agent/core variables are defined on `CoreSettings` in `apps/core/src/opendevops_core/config.py`; web/auth-only variables (e.g. `JWT_SECRET`) live on `Settings(CoreSettings)` in `apps/backend/src/config/appsettings.py`. `.env.example` mirrors every variable. The OSS app instantiates `Settings` and registers it via `configure()` so core sees it at runtime. ```bash # LLM — LiteLLM model string format diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c0d79c6 --- /dev/null +++ b/Makefile @@ -0,0 +1,46 @@ +.PHONY: dev ui migrate test lint frontend-dev frontend-build help + +BACKEND := apps/backend +FRONTEND := apps/frontend + +# ── Backend ──────────────────────────────────────────────────────────────────── +dev: + cd $(BACKEND) && uv run dev + +ui: + cd $(BACKEND) && uv run devops-agent ui + +migrate: + cd $(BACKEND) && uv run migrate + +test: + cd $(BACKEND) && uv run pytest -q + +lint: + cd $(BACKEND) && uv run ruff check src/ && uv run ruff format --check src/ + +lint-fix: + cd $(BACKEND) && uv run ruff check --fix src/ && uv run ruff format src/ + +install: + cd $(BACKEND) && uv sync + +# ── Frontend ─────────────────────────────────────────────────────────────────── +frontend-dev: + cd $(FRONTEND) && npm install && npm run dev + +frontend-build: + cd $(FRONTEND) && npm install && npm run build + +# ── Full stack (Docker Compose) ──────────────────────────────────────────────── +compose-up: + docker compose -f deployment/docker-compose/docker-compose.yml up --build + +compose-down: + docker compose -f deployment/docker-compose/docker-compose.yml down + +# ── Help ─────────────────────────────────────────────────────────────────────── +help: + @echo "Backend: make dev | ui | migrate | test | lint | lint-fix | install" + @echo "Frontend: make frontend-dev | frontend-build" + @echo "Docker: make compose-up | compose-down" diff --git a/README.md b/README.md index a4b0cbe..1f0b78e 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,9 @@ and gives actionable mitigation plans — without the AWS DevOps Agent price tag - **Sandboxed bash execution tool** — agent can run whitelisted read-only AWS CLI, kubectl, and docker commands as a last resort when the structured tools fall short; every command validated against an allowlist before execution; never uses `shell=True`; hard 30-second timeout - Includes **CloudWatch Logs Insights** (`query_logs_insights`) — full query language support: `fields`, `filter`, `stats`, `sort`, `limit`; results include scanned MB - **Streaming responses** — FastAPI SSE endpoint streams agent tokens in real time as the LLM reasons; tool calls appear as they complete -- **Event-driven incident detection** — EventBridge → SQS → long-poll consumer; 9 EventBridge rules cover CloudWatch alarms, ECS task failures, Lambda async errors, RDS events, EC2 state changes, CodePipeline failures, and AWS Health events; uses a DLQ plus database-backed incident claims to avoid duplicate investigations; runs alongside the metric poller — see [docs/event_detection.md](docs/event_detection.md) +- **Event-driven incident detection** — EventBridge → SQS → long-poll consumer; 9 EventBridge rules cover CloudWatch alarms, ECS task failures, Lambda async errors, RDS events, EC2 state changes, CodePipeline failures, and AWS Health events; uses a DLQ plus database-backed incident claims to avoid duplicate investigations; runs alongside the metric poller — see [apps/documentation/event_detection.md](apps/documentation/event_detection.md) - **Context enrichment** — before the LLM runs, deterministic boto3 calls fetch facts about the affected resource (alarm details, recent logs, function config, etc.) to reduce tool call count and speed up investigations -- **Monitoring dashboard** — live incident feed showing all event-driven investigations: confidence level (or FAILED badge), affected service, root cause summary; each alert links back to its original investigation session via **View investigation** so you can follow up without losing context; real-time SSE push keeps the page live without polling — see [docs/monitoring.md](docs/monitoring.md) +- **Monitoring dashboard** — live incident feed showing all event-driven investigations: confidence level (or FAILED badge), affected service, root cause summary; each alert links back to its original investigation session via **View investigation** so you can follow up without losing context; real-time SSE push keeps the page live without polling — see [apps/documentation/monitoring.md](apps/documentation/monitoring.md) - **AWS Configuration settings tab** — admin-only editable tab in Settings for SQS Queue URL and AWS Region; shared org-wide via database-backed app config; includes an inline IAM permission checker per service - **Web UI** — React + Vite SPA served by FastAPI: - **Chat page** — streaming responses, collapsible tool call inspector, cost/latency card, stop button; supports `?prompt=` deeplink for pre-seeded investigations from the Monitoring dashboard @@ -32,12 +32,12 @@ and gives actionable mitigation plans — without the AWS DevOps Agent price tag - **History page** — keyword search across all past sessions - **Settings page** — AWS Configuration (editable, admin-only), Environment (read-only env vars), Agent config, Integrations - **Team page** — admin-only user management: add, remove, and change roles -- **Auth & RBAC** — optional password-based auth with `admin` and `user` roles; JWT tokens; first registered user auto-becomes admin; disabled by default (set `JWT_SECRET` to enable) — see [docs/auth.md](docs/auth.md) -- **Three storage backends** — pick one via `CHECKPOINT_BACKEND` in `.env`; see [`docs/databases.md`](docs/databases.md) +- **Auth & RBAC** — optional password-based auth with `admin` and `user` roles; JWT tokens; first registered user auto-becomes admin; disabled by default (set `JWT_SECRET` to enable) — see [apps/documentation/auth.md](apps/documentation/auth.md) +- **Three storage backends** — pick one via `CHECKPOINT_BACKEND` in `.env`; see [`apps/documentation/databases.md`](apps/documentation/databases.md) - `memory` — zero config, no persistence; great for CI and quick testing; autonomous polling/event monitoring is disabled in this mode - `sqlite` — local file, no external services; recommended for single-server and personal use - `postgres` — full production persistence via psycopg3 + `AsyncPostgresSaver` - - Schema: `users`, `sessions`, `messages`, `tool_calls`, `usage_events` — see [`docs/schema.md`](docs/schema.md) + - Schema: `users`, `sessions`, `messages`, `tool_calls`, `usage_events` — see [`apps/documentation/schema.md`](apps/documentation/schema.md) - Soft delete — deleted sessions are hidden immediately but data is preserved for the 30-day cleanup job - **Structured logging** via Loguru — used consistently across all modules (tools, agent, API, CLI); every request shows agent reasoning, tool calls with args/results, and a done summary with latency + token counts - **CLI** — `devops-agent investigate`, `ask`, and `report` commands powered by the same agent @@ -48,7 +48,7 @@ and gives actionable mitigation plans — without the AWS DevOps Agent price tag ### 1. Install dependencies ```bash -uv sync +cd apps/backend && uv sync ``` ### 2. Configure environment @@ -73,7 +73,7 @@ aws sts get-caller-identity --profile devops-agent-readonly ### 4. Choose a storage backend -Three options — pick one and add it to `.env`. Full details in [`docs/databases.md`](docs/databases.md). +Three options — pick one and add it to `.env`. Full details in [`apps/documentation/databases.md`](apps/documentation/databases.md). **Memory** (default — zero config, nothing persists on restart) ```bash @@ -101,7 +101,7 @@ CHECKPOINT_BACKEND=postgres DATABASE_URL=postgresql://dev:dev@localhost:5433/opendevops # Create app tables (safe to re-run) -uv run migrate +cd apps/backend && uv run migrate ``` ### 5. Run @@ -109,7 +109,7 @@ uv run migrate **Option A — Docker Compose (recommended, AWS CLI included)** ```bash -docker compose up --build +docker compose -f deployment/docker-compose/docker-compose.yml up --build # Backend: http://localhost:8000 # Frontend: http://localhost:80 # Postgres (host): localhost:5433 @@ -124,13 +124,12 @@ an IAM role to the instance/task instead. ```bash # Terminal 1 — FastAPI backend with hot reload -uv run dev +cd apps/backend && uv run dev ``` ```bash # Terminal 2 — React frontend (Vite dev server with HMR) -cd frontend -npm run dev +cd apps/frontend && npm run dev # Open http://localhost:5173 ``` @@ -140,6 +139,8 @@ npm run dev **CLI** ```bash +cd apps/backend + # Investigate an incident uv run devops-agent investigate "high error rate on my payment Lambda" @@ -158,53 +159,54 @@ uv run devops-agent report The agent needs read access across your AWS account, plus optional write access scoped to `opendevops-*` resources if you use the event-driven monitoring setup wizard. Two least-privilege policies (Operational + Setup) and full step-by-step instructions are in -**[docs/iam_setup.md](docs/iam_setup.md)**. +**[apps/documentation/iam_setup.md](apps/documentation/iam_setup.md)**. ## Project Structure ``` -src/ -├── agent/ # DeepAgents setup, prompts, models, config, DB layer -├── tools/ # 19 read-only AWS tool functions -├── api/ -│ ├── app.py # FastAPI app factory — mounts routers, serves frontend -│ ├── auth.py # JWT helpers + FastAPI auth dependencies -│ └── routers/ -│ ├── auth.py # POST /auth/register|login · GET /auth/status|me -│ ├── chat.py # POST /chat — SSE streaming endpoint -│ ├── sessions.py# GET/DELETE /sessions — session history -│ ├── users.py # GET/POST/PATCH/DELETE /users (admin only) -│ ├── settings.py# GET /settings — read-only config view -│ ├── history.py # GET /history/* — cross-session analytics -│ └── dashboard.py# GET /stats -├── cli/ # Typer CLI commands -├── config/ -│ └── appsettings.py # Pydantic Settings — single source of truth for all env vars -├── models/ # Pydantic models: agent, chat, sessions, users -├── skills/ # Markdown runbooks (lambda-throttling + add your own) -└── integrations/ - └── slack_webhook.py -frontend/ -└── src/ - ├── pages/ # ChatPage, DashboardPage, HistoryPage, SettingsPage, UsersPage, LoginPage - └── components/ # Sidebar, Header, ProtectedRoute, AgentMessage, ... -migrations/ -├── 001_initial.sql # Base schema -├── 002_soft_delete.sql -├── 003_usage_events_metadata.sql -└── 004_users_rbac.sql # password_hash + role on users -docs/ # Feature reference — auth, schema, skills, databases, UI, ... +apps/ +├── core/ # Installable package `opendevops-core` — the shared agent brain +│ └── src/opendevops_core/ +│ ├── agent/ # DeepAgents setup, prompts, LLM wiring, DB layer (backends + ABC) +│ ├── tools/ # bash, history, skills, final-answer + response cap/cache +│ ├── providers/ # AWS provider — tools, context, poller, event consumer +│ ├── models/ # Pydantic models: agent, chat, sessions, users +│ ├── skills/ # Markdown runbooks (lambda-throttling + add your own) +│ ├── integrations/ # slack_webhook.py, telegram.py +│ ├── migrations/ # Numbered baseline SQL migrations (001–013) — bundled with the wheel +│ └── config.py # CoreSettings + get_settings()/configure() injection hook +├── backend/ # OSS web app + CLI — depends on opendevops-core via uv path source +│ ├── src/ +│ │ ├── api/ +│ │ │ ├── app.py # FastAPI app factory — mounts routers, serves frontend +│ │ │ ├── auth.py # JWT helpers + FastAPI auth dependencies +│ │ │ └── routers/ # chat, sessions, users, settings, history, dashboard, monitoring +│ │ ├── cli/ # Typer CLI commands +│ │ ├── config/ +│ │ │ └── appsettings.py # Settings(CoreSettings) — adds web/auth-only fields, calls configure() +│ │ └── mcp_server.py # MCP server (stdio / HTTP+SSE) +│ ├── migrations/ # OSS-app-only migrations (currently none; all schema is core baseline) +│ ├── tests/ +│ └── pyproject.toml +├── frontend/ +│ └── src/ +│ ├── pages/ # ChatPage, DashboardPage, HistoryPage, SettingsPage, UsersPage, LoginPage +│ └── components/ # Sidebar, Header, ProtectedRoute, AgentMessage, ... +└── documentation/ # Feature reference — auth, schema, skills, databases, UI, ... +deployment/ +├── docker-compose/ # docker-compose.yml (PostgreSQL + backend + frontend) +└── railway/ # Dockerfile.railway + railway.toml (combined single-image deploy) ``` ## Configuration | Variable | Default | Description | |---|---|---| -| `LLM_MODEL` | `openrouter/openai/gpt-4o` | LiteLLM model string — `provider/model` format; see [docs/llm_providers.md](docs/llm_providers.md) | +| `LLM_MODEL` | `openrouter/openai/gpt-4o` | LiteLLM model string — `provider/model` format; see [apps/documentation/llm_providers.md](apps/documentation/llm_providers.md) | | `LLM_API_BASE` | none | Custom base URL for OpenAI-compatible endpoints (e.g. Ollama, vLLM) | | `LLM_API_KEY` | none | API key for custom endpoints; standard provider keys (e.g. `ANTHROPIC_API_KEY`) are read automatically | | `OPENROUTER_API_KEY` | none | Required when using any `openrouter/` model | -| `CHECKPOINT_BACKEND` | `memory` | Storage backend: `memory` · `sqlite` · `postgres` — see [docs/databases.md](docs/databases.md) | +| `CHECKPOINT_BACKEND` | `memory` | Storage backend: `memory` · `sqlite` · `postgres` — see [apps/documentation/databases.md](apps/documentation/databases.md) | | `SQLITE_PATH` | `./data/agent.db` | SQLite file path — only used when `CHECKPOINT_BACKEND=sqlite` | | `DATABASE_URL` | none | PostgreSQL connection string — only used when `CHECKPOINT_BACKEND=postgres` | | `AWS_REGION` | `us-east-1` | AWS region | @@ -235,14 +237,14 @@ docs/ # Feature reference — auth, schema, skills, databases, - [x] **Schema / models layer** — centralized `src/models/` package for all Pydantic models: agent domain, memory state, and API request/response schemas - [ ] **Soft-deleted session cleanup job** — product version only; OSS users manage their own DB - [x] **Investigation history skill** — cross-session analysis: recurring errors, most-triggered alarms, patterns across all past sessions for a user -- [x] **User roles** — `admin` / `user` roles with JWT auth, first-user bootstrap, admin-only user management UI; optional (disabled when `JWT_SECRET` unset) — see [docs/auth.md](docs/auth.md) +- [x] **User roles** — `admin` / `user` roles with JWT auth, first-user bootstrap, admin-only user management UI; optional (disabled when `JWT_SECRET` unset) — see [apps/documentation/auth.md](apps/documentation/auth.md) ### Medium-term - [x] **React frontend** — rewrite the single-file HTML UI in React; component-based architecture, proper state management, hot reload - [x] **Dashboard** — summarized view of troubleshooting activity, recurring incidents, query breakdown by service -- [x] **Multi-provider LLM support** — 100+ providers via LiteLLM; swap models with a single `LLM_MODEL` env var change; supports OpenRouter, Anthropic, OpenAI, Groq, Ollama, and any OpenAI-compatible endpoint; see [docs/llm_providers.md](docs/llm_providers.md) -- [x] **MCP integration** — expose the agent as an MCP server (`devops-agent mcp`); `investigate`, `ask`, and `list_sessions` tools available in Claude Desktop, Cursor, or any MCP-compatible client; stdio and HTTP+SSE transports; see [docs/mcp_server.md](docs/mcp_server.md) -- [x] **Multi-backend storage** — `memory` (zero config), `sqlite` (local file, no external service), `postgres` (production); switch with one env var; see [docs/databases.md](docs/databases.md) +- [x] **Multi-provider LLM support** — 100+ providers via LiteLLM; swap models with a single `LLM_MODEL` env var change; supports OpenRouter, Anthropic, OpenAI, Groq, Ollama, and any OpenAI-compatible endpoint; see [apps/documentation/llm_providers.md](apps/documentation/llm_providers.md) +- [x] **MCP integration** — expose the agent as an MCP server (`devops-agent mcp`); `investigate`, `ask`, and `list_sessions` tools available in Claude Desktop, Cursor, or any MCP-compatible client; stdio and HTTP+SSE transports; see [apps/documentation/mcp_server.md](apps/documentation/mcp_server.md) +- [x] **Multi-backend storage** — `memory` (zero config), `sqlite` (local file, no external service), `postgres` (production); switch with one env var; see [apps/documentation/databases.md](apps/documentation/databases.md) - [x] **Skills system** — on-demand investigation skills loaded from `src/skills/*/SKILL.md`; skill names injected into system prompt at startup, full content loaded only when agent calls `use_skill(name)`; ships with `lambda-throttling` skill; add your own by dropping a `SKILL.md` into `src/skills//` - [ ] **Custom tools via URL** — register external tools by pointing at an OpenAPI/HTTP endpoint; agent discovers and calls them alongside built-in AWS tools - [x] **Bash CLI escape hatch (Phase 1)** — `run_bash_command` is implemented for read-only AWS CLI, kubectl, and docker commands with strict allowlist validation and timeout. @@ -254,10 +256,10 @@ docs/ # Feature reference — auth, schema, skills, databases, - [ ] **Guardrails** — input/output validation, PII scrubbing, query scope enforcement - [ ] **Multi-model escalation** — route simple queries to cheaper/smaller models, escalate hard investigations to larger ones - [x] **Fun streaming labels** — contextual loading copy ("Digging through CloudTrail…", "Lemonizing metrics…", "Cooking up a root cause…") -- [x] **Slack & Telegram notifications** — reactive: posts after every investigation to Slack (Block Kit) and/or Telegram (HTML bot message); proactive: background poller checks CloudWatch alarms and Lambda error rates, auto-investigates, and delivers to both channels; set `SLACK_WEBHOOK_URL` and/or `TELEGRAM_BOT_TOKEN`+`TELEGRAM_CHAT_ID` in `.env`; see [docs/telegram.md](docs/telegram.md) -- [x] **Event-driven incident detection** — EventBridge → SQS → long-poll consumer; 9 EventBridge rules covering CloudWatch alarms, ECS, Lambda, RDS, EC2, CodePipeline, and AWS Health; runs in parallel with the metric poller; see [docs/event_detection.md](docs/event_detection.md) +- [x] **Slack & Telegram notifications** — reactive: posts after every investigation to Slack (Block Kit) and/or Telegram (HTML bot message); proactive: background poller checks CloudWatch alarms and Lambda error rates, auto-investigates, and delivers to both channels; set `SLACK_WEBHOOK_URL` and/or `TELEGRAM_BOT_TOKEN`+`TELEGRAM_CHAT_ID` in `.env`; see [apps/documentation/telegram.md](apps/documentation/telegram.md) +- [x] **Event-driven incident detection** — EventBridge → SQS → long-poll consumer; 9 EventBridge rules covering CloudWatch alarms, ECS, Lambda, RDS, EC2, CodePipeline, and AWS Health; runs in parallel with the metric poller; see [apps/documentation/event_detection.md](apps/documentation/event_detection.md) - [x] **Context enrichment** — deterministic boto3 calls per event type before LLM runs; reduces tool call count by front-loading relevant resource facts -- [x] **Monitoring dashboard** — live incident feed with real-time SSE push, per-service health summary (DB-backed, survives restarts), alert detail page; "View investigation" opens the original agent session for follow-up; failed investigations flagged separately; see [docs/monitoring.md](docs/monitoring.md) +- [x] **Monitoring dashboard** — live incident feed with real-time SSE push, per-service health summary (DB-backed, survives restarts), alert detail page; "View investigation" opens the original agent session for follow-up; failed investigations flagged separately; see [apps/documentation/monitoring.md](apps/documentation/monitoring.md) - [x] **AWS Configuration settings tab** — admin-only editable tab for SQS/region config; shared org-wide via database-backed app config; inline IAM permission checker ### Later @@ -299,11 +301,13 @@ and waits for human approval before anything changes. ## Development +All backend commands run from `apps/backend/`, or use root `make` targets. + ```bash # Run tests -uv run pytest +cd apps/backend && uv run pytest # or: make test # Lint + format -uv run ruff check src/ tests/ -uv run ruff format src/ tests/ +cd apps/backend && uv run ruff check src/ tests/ +cd apps/backend && uv run ruff format src/ tests/ # or: make lint / make lint-fix ``` diff --git a/apps/backend/.dockerignore b/apps/backend/.dockerignore new file mode 100644 index 0000000..283cc24 --- /dev/null +++ b/apps/backend/.dockerignore @@ -0,0 +1,15 @@ +.venv/ +__pycache__/ +*.pyc +*.pyo +*.pyd + +.env +.env.* +!.env.example + +.git/ +tests/ +*.md +*.db +data/ diff --git a/.env.example b/apps/backend/.env.example similarity index 100% rename from .env.example rename to apps/backend/.env.example diff --git a/Dockerfile b/apps/backend/Dockerfile similarity index 100% rename from Dockerfile rename to apps/backend/Dockerfile diff --git a/pyproject.toml b/apps/backend/pyproject.toml similarity index 62% rename from pyproject.toml rename to apps/backend/pyproject.toml index 8bfe374..56705f4 100644 --- a/pyproject.toml +++ b/apps/backend/pyproject.toml @@ -4,27 +4,11 @@ version = "0.1.0" description = "Open-source AWS DevOps Agent powered by OpenRouter LLMs" requires-python = ">=3.11" dependencies = [ - "deepagents", - "langchain-openai>=0.1.0", - "langgraph>=0.2.0", - "boto3>=1.34.0", + "opendevops-core", "typer>=0.12.0", "rich>=13.7.0", - "pydantic>=2.7.0", - "pydantic-settings>=2.3.0", - "python-dotenv>=1.0.0", - "loguru>=0.7.0", "fastapi>=0.111.0", "uvicorn>=0.30.0", - "langgraph-checkpoint-postgres>=2.0.0", - "langgraph-checkpoint-sqlite>=2.0.0", - "psycopg[binary,pool]>=3.1.0", - "aiosqlite>=0.20.0", - "cachetools>=5.3.0", - "httpx>=0.27.0", - "litellm>=1.83.0", - "langchain-community>=0.4.1", - "langchain-litellm>=0.6.4", "fastmcp>=3.2.4", "python-jose[cryptography]>=3.3.0", "bcrypt>=5.0.0", @@ -40,11 +24,15 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] -packages = ["src/agent", "src/tools", "src/cli", "src/integrations", "src/api", "src/models"] +packages = ["src/api", "src/cli", "src/config"] +dev-mode-dirs = ["src"] [tool.uv] package = true +[tool.uv.sources] +opendevops-core = { path = "../core", editable = true } + [dependency-groups] dev = [ "pytest>=8.0.0", diff --git a/pytest.ini b/apps/backend/pytest.ini similarity index 100% rename from pytest.ini rename to apps/backend/pytest.ini diff --git a/scripts/setup_db.py b/apps/backend/scripts/setup_db.py similarity index 90% rename from scripts/setup_db.py rename to apps/backend/scripts/setup_db.py index e4b4cc1..668ce7b 100644 --- a/scripts/setup_db.py +++ b/apps/backend/scripts/setup_db.py @@ -46,18 +46,16 @@ def _info(msg: str) -> None: # ── Sync migration (psycopg3 sync API — simpler for a one-shot script) ─────── def run_migration(database_url: str) -> None: - import psycopg # type: ignore - - migrations_dir = Path(__file__).parent.parent / "migrations" - sql_files = sorted(migrations_dir.glob("*.sql")) + from opendevops_core.migrations import run_migrations + app_dir = Path(__file__).parent.parent / "migrations" _info(f"Connecting to {_mask_url(database_url)}") - with psycopg.connect(database_url, autocommit=True) as conn: - _ok("Connected") - for sql_file in sql_files: - _info(f"Running {sql_file.name} …") - conn.execute(sql_file.read_text(encoding="utf-8")) - _ok(f"{sql_file.name} applied") + applied = run_migrations(database_url, app_dir) + if applied: + for ident in applied: + _ok(f"{ident} applied") + else: + _ok("Already up to date — no pending migrations") # ── Async LangGraph checkpointer setup ─────────────────────────────────────── diff --git a/scripts/test_db_connection.py b/apps/backend/scripts/test_db_connection.py similarity index 100% rename from scripts/test_db_connection.py rename to apps/backend/scripts/test_db_connection.py diff --git a/scripts/test_dlq.py b/apps/backend/scripts/test_dlq.py similarity index 100% rename from scripts/test_dlq.py rename to apps/backend/scripts/test_dlq.py diff --git a/scripts/test_pipeline.py b/apps/backend/scripts/test_pipeline.py similarity index 100% rename from scripts/test_pipeline.py rename to apps/backend/scripts/test_pipeline.py diff --git a/src/agent/__init__.py b/apps/backend/src/api/__init__.py similarity index 100% rename from src/agent/__init__.py rename to apps/backend/src/api/__init__.py diff --git a/src/api/app.py b/apps/backend/src/api/app.py similarity index 89% rename from src/api/app.py rename to apps/backend/src/api/app.py index 8d6953d..8dbb88e 100644 --- a/src/api/app.py +++ b/apps/backend/src/api/app.py @@ -13,8 +13,9 @@ # Suppress LiteLLM's "Provider List" info spam — we don't need provider discovery hints in logs litellm.suppress_debug_info = True -from agent.core import init_agent # noqa: E402 -from agent.db import db # noqa: E402 +from opendevops_core.agent.core import init_agent # noqa: E402 +from opendevops_core.agent.db import db # noqa: E402 + from api.routers import ( # noqa: E402 auth, chat, @@ -66,7 +67,7 @@ def start_event_consumer(app_instance: "FastAPI | None" = None) -> None: existing: asyncio.Task | None = getattr(target, "_consumer_task", None) if existing and not existing.done(): return - from providers import get_active_provider + from opendevops_core.providers import get_active_provider target._consumer_task = asyncio.create_task(get_active_provider().event_consumer_loop()) # type: ignore[attr-defined] logger.info("Event consumer task started") @@ -96,7 +97,7 @@ async def lifespan(_app: FastAPI): from config import settings as _cfg checkpointer = await db.init() - from agent.init_store import refresh_init_cache_from_db + from opendevops_core.agent.init_store import refresh_init_cache_from_db await refresh_init_cache_from_db() init_agent(checkpointer) @@ -109,7 +110,7 @@ async def lifespan(_app: FastAPI): "or postgres for durable incident claims" ) else: - from providers import get_active_provider + from opendevops_core.providers import get_active_provider poller_task = asyncio.create_task(get_active_provider().polling_loop()) logger.info("Proactive poller started (interval={}s)", _cfg.poll_interval_seconds) @@ -117,7 +118,7 @@ async def lifespan(_app: FastAPI): # Event consumer — started if explicitly enabled, SQS URL is set, or init wizard completed if _cfg.checkpoint_backend == "memory": try: - from agent.init_store import is_event_infra_enabled + from opendevops_core.agent.init_store import is_event_infra_enabled init_enabled = is_event_infra_enabled() except Exception: @@ -131,7 +132,7 @@ async def lifespan(_app: FastAPI): start_event_consumer(_app) else: try: - from agent.init_store import is_event_infra_enabled + from opendevops_core.agent.init_store import is_event_infra_enabled if is_event_infra_enabled(): start_event_consumer(_app) @@ -175,7 +176,10 @@ async def lifespan(_app: FastAPI): app.include_router(init_router.router) app.include_router(monitoring.router) -_DIST = Path(__file__).parent.parent.parent / "frontend" / "dist" +_DIST = Path(__file__).resolve().parents[3] / "frontend" / "dist" +if not _DIST.is_dir(): + # Docker layout: src/ at /app/src, frontend/dist at /app/frontend/dist + _DIST = Path(__file__).resolve().parents[2] / "frontend" / "dist" # Mount compiled JS/CSS assets from the Vite build if _DIST.is_dir(): diff --git a/src/api/auth.py b/apps/backend/src/api/auth.py similarity index 100% rename from src/api/auth.py rename to apps/backend/src/api/auth.py diff --git a/src/api/__init__.py b/apps/backend/src/api/routers/__init__.py similarity index 100% rename from src/api/__init__.py rename to apps/backend/src/api/routers/__init__.py diff --git a/src/api/routers/auth.py b/apps/backend/src/api/routers/auth.py similarity index 98% rename from src/api/routers/auth.py rename to apps/backend/src/api/routers/auth.py index 71f1803..110e7d1 100644 --- a/src/api/routers/auth.py +++ b/apps/backend/src/api/routers/auth.py @@ -5,9 +5,9 @@ from typing import Annotated from fastapi import APIRouter, Depends, HTTPException +from opendevops_core.agent.db import db from pydantic import BaseModel, EmailStr -from agent.db import db from api.auth import create_access_token, get_current_user, hash_password, verify_password from config import settings diff --git a/src/api/routers/chat.py b/apps/backend/src/api/routers/chat.py similarity index 97% rename from src/api/routers/chat.py rename to apps/backend/src/api/routers/chat.py index f07e9bc..2fa6c99 100644 --- a/src/api/routers/chat.py +++ b/apps/backend/src/api/routers/chat.py @@ -12,13 +12,13 @@ from fastapi import APIRouter, HTTPException from fastapi.responses import StreamingResponse from loguru import logger +from opendevops_core.agent.core import get_active_model, get_agent +from opendevops_core.agent.summarizer import maybe_summarize +from opendevops_core.agent.turns import calc_cost, notify_slack, save_turn +from opendevops_core.models.chat import ChatRequest -from config import settings -from agent.core import get_agent, get_active_model -from agent.summarizer import maybe_summarize -from agent.turns import calc_cost, save_turn, notify_slack -from models.chat import ChatRequest from api.streaming_labels import STREAMING_LABELS +from config import settings router = APIRouter(tags=["chat"]) diff --git a/src/api/routers/dashboard.py b/apps/backend/src/api/routers/dashboard.py similarity index 87% rename from src/api/routers/dashboard.py rename to apps/backend/src/api/routers/dashboard.py index bd32b2f..a661db1 100644 --- a/src/api/routers/dashboard.py +++ b/apps/backend/src/api/routers/dashboard.py @@ -3,8 +3,7 @@ from __future__ import annotations from fastapi import APIRouter - -from agent.db import db +from opendevops_core.agent.db import db router = APIRouter(tags=["dashboard"]) diff --git a/src/api/routers/history.py b/apps/backend/src/api/routers/history.py similarity index 93% rename from src/api/routers/history.py rename to apps/backend/src/api/routers/history.py index e15f2e7..2565616 100644 --- a/src/api/routers/history.py +++ b/apps/backend/src/api/routers/history.py @@ -3,8 +3,7 @@ from __future__ import annotations from fastapi import APIRouter, Query - -from agent.db import db +from opendevops_core.agent.db import db router = APIRouter(prefix="/api/history", tags=["history"]) diff --git a/src/api/routers/init.py b/apps/backend/src/api/routers/init.py similarity index 96% rename from src/api/routers/init.py rename to apps/backend/src/api/routers/init.py index 3255512..4da45b9 100644 --- a/src/api/routers/init.py +++ b/apps/backend/src/api/routers/init.py @@ -10,16 +10,16 @@ import boto3 from fastapi import APIRouter, Depends, HTTPException from loguru import logger -from pydantic import BaseModel, EmailStr, Field - -from agent.db import db -from agent.llm import get_backend_info -from agent.init_store import ( +from opendevops_core.agent.db import db +from opendevops_core.agent.init_store import ( get_runtime_aws_region, get_runtime_sqs_queue_url, load_init_async, save_init_async, ) +from opendevops_core.agent.llm import get_backend_info +from pydantic import BaseModel, EmailStr, Field + from api.auth import hash_password, require_admin from config import settings @@ -46,7 +46,7 @@ async def reset_init_state( _: Annotated[dict | None, Depends(require_admin)], ): """Clear persisted init/setup state. Useful for re-running the wizard without touching AWS.""" - from agent.init_store import reset_init_async + from opendevops_core.agent.init_store import reset_init_async await reset_init_async() return {"reset": True} @@ -126,7 +126,7 @@ async def setup( async def check_permissions_endpoint( _: Annotated[dict | None, Depends(require_admin)], ): - from providers import get_active_provider + from opendevops_core.providers import get_active_provider provider = get_active_provider() data = await load_init_async() @@ -165,7 +165,7 @@ async def complete( region = data.get("aws_region") or settings.aws_region try: - from providers.aws.event_infra import setup_event_infra + from opendevops_core.providers.aws.event_infra import setup_event_infra result = await asyncio.get_event_loop().run_in_executor(None, setup_event_infra, region) data["sqs_queue_url"] = result["queue_url"] @@ -240,7 +240,7 @@ async def teardown_infra( return {"torn_down": True, "warnings": ["No event infrastructure was configured"]} try: - from providers.aws.event_infra import teardown_event_infra + from opendevops_core.providers.aws.event_infra import teardown_event_infra result = await asyncio.get_event_loop().run_in_executor( None, diff --git a/src/api/routers/integrations.py b/apps/backend/src/api/routers/integrations.py similarity index 94% rename from src/api/routers/integrations.py rename to apps/backend/src/api/routers/integrations.py index 2774498..6eca472 100644 --- a/src/api/routers/integrations.py +++ b/apps/backend/src/api/routers/integrations.py @@ -21,7 +21,7 @@ async def test_slack( if not settings.slack_webhook_url: raise HTTPException(status_code=400, detail="SLACK_WEBHOOK_URL is not configured") - from integrations.slack_webhook import post_investigation + from opendevops_core.integrations.slack_webhook import post_investigation test_result = { "root_cause_category": "SYSTEM_CHANGE", @@ -55,7 +55,7 @@ async def test_telegram( detail="TELEGRAM_BOT_TOKEN and TELEGRAM_CHAT_ID must both be configured", ) - from integrations.telegram import _build_message, _post + from opendevops_core.integrations.telegram import _build_message, _post test_result = { "root_cause_category": "SYSTEM_CHANGE", diff --git a/src/api/routers/monitoring.py b/apps/backend/src/api/routers/monitoring.py similarity index 97% rename from src/api/routers/monitoring.py rename to apps/backend/src/api/routers/monitoring.py index ef59c9f..3c34cf3 100644 --- a/src/api/routers/monitoring.py +++ b/apps/backend/src/api/routers/monitoring.py @@ -8,14 +8,14 @@ from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import StreamingResponse - -from agent.monitor_store import ( +from opendevops_core.agent.monitor_store import ( get_alert_async, get_alerts_async, get_services, # async — returns in-memory or falls back to DB subscribe_alerts, unsubscribe_alerts, ) + from api.auth import get_current_user router = APIRouter(prefix="/api/monitoring", tags=["monitoring"]) diff --git a/src/api/routers/sessions.py b/apps/backend/src/api/routers/sessions.py similarity index 91% rename from src/api/routers/sessions.py rename to apps/backend/src/api/routers/sessions.py index 0158e5c..3bdff34 100644 --- a/src/api/routers/sessions.py +++ b/apps/backend/src/api/routers/sessions.py @@ -1,11 +1,10 @@ """Session management endpoints.""" from fastapi import APIRouter, HTTPException, Query +from opendevops_core.agent.db import db +from opendevops_core.models.sessions import MessageRecord, SessionSummary from pydantic import BaseModel -from agent.db import db -from models.sessions import MessageRecord, SessionSummary - router = APIRouter(prefix="/sessions", tags=["sessions"]) diff --git a/src/api/routers/settings.py b/apps/backend/src/api/routers/settings.py similarity index 97% rename from src/api/routers/settings.py rename to apps/backend/src/api/routers/settings.py index 67c85fd..5ced6fa 100644 --- a/src/api/routers/settings.py +++ b/apps/backend/src/api/routers/settings.py @@ -2,14 +2,13 @@ from __future__ import annotations -from typing import Annotated - import os +from typing import Annotated from fastapi import APIRouter, Depends +from opendevops_core.agent.init_store import get_runtime_aws_region +from opendevops_core.agent.llm import get_backend_info -from agent.llm import get_backend_info -from agent.init_store import get_runtime_aws_region from api.auth import get_current_user from config import settings diff --git a/src/api/routers/users.py b/apps/backend/src/api/routers/users.py similarity index 94% rename from src/api/routers/users.py rename to apps/backend/src/api/routers/users.py index 49d0017..dc221b7 100644 --- a/src/api/routers/users.py +++ b/apps/backend/src/api/routers/users.py @@ -5,10 +5,10 @@ from typing import Annotated from fastapi import APIRouter, Depends, HTTPException +from opendevops_core.agent.db import db +from opendevops_core.models.users import UserCreate, UserOut, UserUpdate -from agent.db import db from api.auth import hash_password, require_admin -from models.users import UserCreate, UserOut, UserUpdate router = APIRouter(prefix="/api/users", tags=["users"]) diff --git a/src/api/sessions.py b/apps/backend/src/api/sessions.py similarity index 100% rename from src/api/sessions.py rename to apps/backend/src/api/sessions.py diff --git a/src/api/streaming_labels.py b/apps/backend/src/api/streaming_labels.py similarity index 100% rename from src/api/streaming_labels.py rename to apps/backend/src/api/streaming_labels.py diff --git a/src/api/routers/__init__.py b/apps/backend/src/cli/__init__.py similarity index 100% rename from src/api/routers/__init__.py rename to apps/backend/src/cli/__init__.py diff --git a/src/cli/ask.py b/apps/backend/src/cli/ask.py similarity index 89% rename from src/cli/ask.py rename to apps/backend/src/cli/ask.py index 26bf24e..f7b09a7 100644 --- a/src/cli/ask.py +++ b/apps/backend/src/cli/ask.py @@ -1,9 +1,10 @@ -from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import TimeoutError as FuturesTimeoutError from typing import Annotated import typer -from langchain_litellm import ChatLiteLLM from langchain_core.messages import HumanMessage, SystemMessage +from langchain_litellm import ChatLiteLLM from rich.console import Console from rich.markdown import Markdown from rich.progress import Progress, SpinnerColumn, TextColumn @@ -17,7 +18,7 @@ def ask_cmd( question: Annotated[str, typer.Argument(help="Freeform question about your AWS environment.")], ) -> None: """Ask a freeform question about your AWS environment.""" - from agent.llm import resolve_model_and_key, shape_system_content + from opendevops_core.agent.llm import resolve_model_and_key, shape_system_content model_name, api_key = resolve_model_and_key() model = ChatLiteLLM( model=model_name, diff --git a/src/cli/dev.py b/apps/backend/src/cli/dev.py similarity index 100% rename from src/cli/dev.py rename to apps/backend/src/cli/dev.py diff --git a/src/cli/investigate.py b/apps/backend/src/cli/investigate.py similarity index 96% rename from src/cli/investigate.py rename to apps/backend/src/cli/investigate.py index cddf3cc..654d901 100644 --- a/src/cli/investigate.py +++ b/apps/backend/src/cli/investigate.py @@ -2,14 +2,13 @@ from typing import Annotated import typer +from opendevops_core.agent.core import InvestigationAgent +from opendevops_core.models.agent import Confidence, Investigation +from rich import print as rprint from rich.console import Console from rich.panel import Panel from rich.progress import Progress, SpinnerColumn, TextColumn from rich.table import Table -from rich import print as rprint - -from agent.core import InvestigationAgent -from models.agent import Confidence, Investigation console = Console() diff --git a/src/cli/main.py b/apps/backend/src/cli/main.py similarity index 100% rename from src/cli/main.py rename to apps/backend/src/cli/main.py index 62f2dd3..609a888 100644 --- a/src/cli/main.py +++ b/apps/backend/src/cli/main.py @@ -30,13 +30,13 @@ def main() -> None: # Import sub-commands so they register on the app -from cli.investigate import investigate_cmd # noqa: E402 from cli.ask import ask_cmd # noqa: E402 -from cli.report import report_cmd # noqa: E402 -from cli.ui import ui_cmd # noqa: E402 -from cli.mcp import mcp_cmd # noqa: E402 from cli.dev import dev_cmd # noqa: E402 +from cli.investigate import investigate_cmd # noqa: E402 +from cli.mcp import mcp_cmd # noqa: E402 from cli.migrate import migrate_cmd # noqa: E402 +from cli.report import report_cmd # noqa: E402 +from cli.ui import ui_cmd # noqa: E402 app.command("investigate")(investigate_cmd) app.command("ask")(ask_cmd) diff --git a/src/cli/mcp.py b/apps/backend/src/cli/mcp.py similarity index 99% rename from src/cli/mcp.py rename to apps/backend/src/cli/mcp.py index 964fabb..682408d 100644 --- a/src/cli/mcp.py +++ b/apps/backend/src/cli/mcp.py @@ -1,6 +1,7 @@ -import typer from typing import Annotated +import typer + def mcp_cmd( http: Annotated[bool, typer.Option("--http", help="Use HTTP+SSE transport instead of stdio.")] = False, diff --git a/apps/backend/src/cli/migrate.py b/apps/backend/src/cli/migrate.py new file mode 100644 index 0000000..37afada --- /dev/null +++ b/apps/backend/src/cli/migrate.py @@ -0,0 +1,43 @@ +"""Apply core + app SQL migrations against the configured PostgreSQL database.""" + +from __future__ import annotations + +from pathlib import Path + +import typer +from rich.console import Console + +console = Console() + +# App-specific migrations live here (none yet — all current schema is core baseline). +# New OSS-app-only migrations go in this directory; core migrations ship with the package. +_APP_MIGRATIONS_DIR = Path(__file__).parent.parent.parent / "migrations" + + +def migrate_cmd() -> None: + """Apply all pending SQL migrations to the configured database.""" + from config import settings + + if settings.checkpoint_backend != "postgres" or not settings.database_url: + console.print( + "[bold red]migrate requires CHECKPOINT_BACKEND=postgres and " + "DATABASE_URL to be set.[/bold red]" + ) + raise typer.Exit(1) + + from opendevops_core.migrations import run_migrations + + try: + applied = run_migrations(settings.database_url, _APP_MIGRATIONS_DIR) + except Exception as e: + console.print(f"[bold red]Migration failed: {e}[/bold red]") + raise typer.Exit(1) + + if applied: + for ident in applied: + console.print(f" [green]✓[/green] {ident}") + console.print(f"[bold green]Applied {len(applied)} migration(s).[/bold green]") + else: + console.print( + "[bold green]Database already up to date — no pending migrations.[/bold green]" + ) diff --git a/src/cli/report.py b/apps/backend/src/cli/report.py similarity index 96% rename from src/cli/report.py rename to apps/backend/src/cli/report.py index 7a0c5c9..6dab7da 100644 --- a/src/cli/report.py +++ b/apps/backend/src/cli/report.py @@ -1,13 +1,12 @@ from datetime import UTC, datetime import typer +from opendevops_core.providers.aws.tools.cloudwatch import get_alarms from rich.console import Console from rich.panel import Panel from rich.progress import Progress, SpinnerColumn, TextColumn from rich.table import Table -from providers.aws.tools.cloudwatch import get_alarms - console = Console() diff --git a/src/cli/ui.py b/apps/backend/src/cli/ui.py similarity index 100% rename from src/cli/ui.py rename to apps/backend/src/cli/ui.py diff --git a/src/config/__init__.py b/apps/backend/src/config/__init__.py similarity index 100% rename from src/config/__init__.py rename to apps/backend/src/config/__init__.py diff --git a/apps/backend/src/config/appsettings.py b/apps/backend/src/config/appsettings.py new file mode 100644 index 0000000..865ce8e --- /dev/null +++ b/apps/backend/src/config/appsettings.py @@ -0,0 +1,15 @@ +from opendevops_core.config import CoreSettings, configure + + +class Settings(CoreSettings): + """OSS web-app settings — core fields plus web/auth-only fields.""" + + # Auth — leave jwt_secret unset to disable auth (dev / memory-backend mode) + jwt_secret: str | None = None + jwt_expire_minutes: int = 1440 # 24 hours + + +settings = Settings() + +# Register this instance so opendevops_core reads the app's settings at runtime. +configure(settings) diff --git a/src/mcp_server.py b/apps/backend/src/mcp_server.py similarity index 96% rename from src/mcp_server.py rename to apps/backend/src/mcp_server.py index b182a09..3589288 100644 --- a/src/mcp_server.py +++ b/apps/backend/src/mcp_server.py @@ -34,8 +34,8 @@ async def _init() -> None: """Initialise DB + agent once on first use.""" - from agent.core import get_agent, init_agent - from agent.db import db + from opendevops_core.agent.core import get_agent, init_agent + from opendevops_core.agent.db import db try: get_agent() except RuntimeError: @@ -46,8 +46,8 @@ async def _init() -> None: async def _run_agent(prompt: str) -> dict[str, Any]: """Run the agent, persist the turn to DB, notify Slack, and return the result.""" await _init() - from agent.core import ainvoke_with_timeout - from agent.turns import save_turn, notify_slack + from opendevops_core.agent.core import ainvoke_with_timeout + from opendevops_core.agent.turns import notify_slack, save_turn thread_id = str(uuid.uuid4()) config = { @@ -187,7 +187,7 @@ async def list_sessions(limit: int = 10) -> str: limit: Number of sessions to return (max 50). """ await _init() - from agent.db import db + from opendevops_core.agent.db import db limit = min(limit, 50) sessions = (await db.list_sessions())[:limit] diff --git a/src/run.py b/apps/backend/src/run.py similarity index 100% rename from src/run.py rename to apps/backend/src/run.py diff --git a/src/cli/__init__.py b/apps/backend/tests/__init__.py similarity index 100% rename from src/cli/__init__.py rename to apps/backend/tests/__init__.py diff --git a/tests/conftest.py b/apps/backend/tests/conftest.py similarity index 92% rename from tests/conftest.py rename to apps/backend/tests/conftest.py index a6ce713..d5d3d71 100644 --- a/tests/conftest.py +++ b/apps/backend/tests/conftest.py @@ -16,7 +16,7 @@ def aws_credentials(): @pytest.fixture(autouse=True) def clear_tool_cache(): """Ensure tool-level TTL cache never leaks state between tests.""" - from tools._cache import _cache + from opendevops_core.tools._cache import _cache _cache.clear() yield diff --git a/src/integrations/__init__.py b/apps/backend/tests/integration/__init__.py similarity index 100% rename from src/integrations/__init__.py rename to apps/backend/tests/integration/__init__.py diff --git a/tests/integration/test_dlq_logic.py b/apps/backend/tests/integration/test_dlq_logic.py similarity index 87% rename from tests/integration/test_dlq_logic.py rename to apps/backend/tests/integration/test_dlq_logic.py index 87aacdd..2a659bc 100644 --- a/tests/integration/test_dlq_logic.py +++ b/apps/backend/tests/integration/test_dlq_logic.py @@ -89,7 +89,7 @@ def test_processing_exception_leaves_message_in_queue(): def test_max_receive_count_constant(): """MAX_RECEIVE_COUNT must match what we verified in AWS.""" - from providers.aws.event_infra import MAX_RECEIVE_COUNT + from opendevops_core.providers.aws.event_infra import MAX_RECEIVE_COUNT assert MAX_RECEIVE_COUNT == "5", ( "maxReceiveCount changed — update the DLQ in AWS to match " "(aws sqs set-queue-attributes --attribute-names RedrivePolicy)" @@ -97,7 +97,7 @@ def test_max_receive_count_constant(): def test_dlq_queue_name_constant(): - from providers.aws.event_infra import DLQ_QUEUE_NAME, QUEUE_NAME + from opendevops_core.providers.aws.event_infra import DLQ_QUEUE_NAME, QUEUE_NAME assert DLQ_QUEUE_NAME == f"{QUEUE_NAME}-dlq", ( "DLQ name must be the main queue name with '-dlq' suffix" ) @@ -132,12 +132,12 @@ def fake_delete(QueueUrl, ReceiptHandle): # noqa: N803 mock_sqs.delete_message.side_effect = fake_delete with ( - patch("providers.aws.event_consumer.get_runtime_sqs_queue_url", return_value="https://fake-url"), - patch("providers.aws.event_consumer.get_runtime_aws_region", return_value="us-east-1"), + patch("opendevops_core.providers.aws.event_consumer.get_runtime_sqs_queue_url", return_value="https://fake-url"), + patch("opendevops_core.providers.aws.event_consumer.get_runtime_aws_region", return_value="us-east-1"), patch("boto3.Session") as mock_session, ): mock_session.return_value.client.return_value = mock_sqs - from providers.aws.event_consumer import event_consumer_loop + from opendevops_core.providers.aws.event_consumer import event_consumer_loop try: await event_consumer_loop() except asyncio.CancelledError: @@ -180,13 +180,13 @@ async def fake_process_event(_event): mock_sqs.delete_message.side_effect = fake_delete with ( - patch("providers.aws.event_consumer.get_runtime_sqs_queue_url", return_value="https://fake-url"), - patch("providers.aws.event_consumer.get_runtime_aws_region", return_value="us-east-1"), - patch("providers.aws.event_consumer._process_event", side_effect=fake_process_event), + patch("opendevops_core.providers.aws.event_consumer.get_runtime_sqs_queue_url", return_value="https://fake-url"), + patch("opendevops_core.providers.aws.event_consumer.get_runtime_aws_region", return_value="us-east-1"), + patch("opendevops_core.providers.aws.event_consumer._process_event", side_effect=fake_process_event), patch("boto3.Session") as mock_session, ): mock_session.return_value.client.return_value = mock_sqs - from providers.aws.event_consumer import event_consumer_loop + from opendevops_core.providers.aws.event_consumer import event_consumer_loop try: await event_consumer_loop() except asyncio.CancelledError: diff --git a/tests/integration/test_poller.py b/apps/backend/tests/integration/test_poller.py similarity index 77% rename from tests/integration/test_poller.py rename to apps/backend/tests/integration/test_poller.py index 77714d8..baf457d 100644 --- a/tests/integration/test_poller.py +++ b/apps/backend/tests/integration/test_poller.py @@ -4,8 +4,8 @@ import asyncio -from providers.aws import poller -from agent.incident_keys import alarm_incident_key, lambda_metric_incident_key +from opendevops_core.agent.incident_keys import alarm_incident_key, lambda_metric_incident_key +from opendevops_core.providers.aws import poller def test_claim_window_uses_reinvestigate_hours(monkeypatch): @@ -64,11 +64,11 @@ async def fake_complete(_key, _status="completed", _session_id=None): async def fake_release(_key): raise AssertionError("successful poller investigation should not release the claim") - monkeypatch.setattr("providers.aws.tools.cloudwatch.get_alarms", fake_get_alarms) - monkeypatch.setattr("agent.monitor_store.is_recent_alert", fake_recent) - monkeypatch.setattr("agent.monitor_store.claim_incident", fake_claim) - monkeypatch.setattr("agent.monitor_store.complete_incident", fake_complete) - monkeypatch.setattr("agent.monitor_store.release_incident", fake_release) + monkeypatch.setattr("opendevops_core.providers.aws.tools.cloudwatch.get_alarms", fake_get_alarms) + monkeypatch.setattr("opendevops_core.agent.monitor_store.is_recent_alert", fake_recent) + monkeypatch.setattr("opendevops_core.agent.monitor_store.claim_incident", fake_claim) + monkeypatch.setattr("opendevops_core.agent.monitor_store.complete_incident", fake_complete) + monkeypatch.setattr("opendevops_core.agent.monitor_store.release_incident", fake_release) monkeypatch.setattr(poller, "_run_investigation", fake_run) monkeypatch.setattr(poller, "_persist_and_notify", fake_persist) @@ -123,12 +123,12 @@ async def fake_release(_key): async def fake_is_claimed(_key, _within_minutes=3): return False - monkeypatch.setattr("providers.aws.tools.lambda_.list_lambda_functions", fake_list_lambda_functions) - monkeypatch.setattr("providers.aws.tools.lambda_.get_lambda_error_rate", fake_get_lambda_error_rate) - monkeypatch.setattr("agent.monitor_store.claim_incident", fake_claim) - monkeypatch.setattr("agent.monitor_store.complete_incident", fake_complete) - monkeypatch.setattr("agent.monitor_store.release_incident", fake_release) - monkeypatch.setattr("agent.monitor_store.is_incident_claimed", fake_is_claimed) + monkeypatch.setattr("opendevops_core.providers.aws.tools.lambda_.list_lambda_functions", fake_list_lambda_functions) + monkeypatch.setattr("opendevops_core.providers.aws.tools.lambda_.get_lambda_error_rate", fake_get_lambda_error_rate) + monkeypatch.setattr("opendevops_core.agent.monitor_store.claim_incident", fake_claim) + monkeypatch.setattr("opendevops_core.agent.monitor_store.complete_incident", fake_complete) + monkeypatch.setattr("opendevops_core.agent.monitor_store.release_incident", fake_release) + monkeypatch.setattr("opendevops_core.agent.monitor_store.is_incident_claimed", fake_is_claimed) monkeypatch.setattr(poller, "_run_investigation", fake_run) monkeypatch.setattr(poller, "_persist_and_notify", fake_persist) monkeypatch.setattr(poller.settings, "poll_error_threshold", 5.0, raising=False) diff --git a/tests/integration/test_slack.py b/apps/backend/tests/integration/test_slack.py similarity index 95% rename from tests/integration/test_slack.py rename to apps/backend/tests/integration/test_slack.py index 5482a6b..3ef2b55 100644 --- a/tests/integration/test_slack.py +++ b/apps/backend/tests/integration/test_slack.py @@ -4,7 +4,7 @@ import asyncio -from integrations.slack_webhook import _build_payload, post_investigation +from opendevops_core.integrations.slack_webhook import _build_payload, post_investigation SAMPLE_RESULT = { "root_cause_category": "RESOURCE_LIMIT", diff --git a/tests/integration/test_telegram.py b/apps/backend/tests/integration/test_telegram.py similarity index 96% rename from tests/integration/test_telegram.py rename to apps/backend/tests/integration/test_telegram.py index d9509cb..c6d7ea2 100644 --- a/tests/integration/test_telegram.py +++ b/apps/backend/tests/integration/test_telegram.py @@ -4,7 +4,11 @@ import asyncio -from integrations.telegram import _build_message, post_failed_investigation, post_investigation +from opendevops_core.integrations.telegram import ( + _build_message, + post_failed_investigation, + post_investigation, +) SAMPLE_RESULT = { "root_cause_category": "RESOURCE_LIMIT", diff --git a/src/tools/__init__.py b/apps/backend/tests/test_agent/__init__.py similarity index 100% rename from src/tools/__init__.py rename to apps/backend/tests/test_agent/__init__.py diff --git a/tests/test_agent/test_core.py b/apps/backend/tests/test_agent/test_core.py similarity index 96% rename from tests/test_agent/test_core.py rename to apps/backend/tests/test_agent/test_core.py index a0380de..4907228 100644 --- a/tests/test_agent/test_core.py +++ b/apps/backend/tests/test_agent/test_core.py @@ -6,9 +6,8 @@ from types import SimpleNamespace import pytest - -from agent import core -from models.agent import Investigation +from opendevops_core.agent import core +from opendevops_core.models.agent import Investigation class _FastAgent: diff --git a/tests/test_agent/test_incident_keys.py b/apps/backend/tests/test_agent/test_incident_keys.py similarity index 93% rename from tests/test_agent/test_incident_keys.py rename to apps/backend/tests/test_agent/test_incident_keys.py index 7d0605c..7e8f8040 100644 --- a/tests/test_agent/test_incident_keys.py +++ b/apps/backend/tests/test_agent/test_incident_keys.py @@ -1,6 +1,6 @@ from __future__ import annotations -from agent.incident_keys import event_incident_key, lambda_metric_incident_key +from opendevops_core.agent.incident_keys import event_incident_key, lambda_metric_incident_key def test_lambda_event_keys_split_different_error_signatures(): diff --git a/tests/__init__.py b/apps/backend/tests/test_api/__init__.py similarity index 100% rename from tests/__init__.py rename to apps/backend/tests/test_api/__init__.py diff --git a/tests/test_api/test_cancel.py b/apps/backend/tests/test_api/test_cancel.py similarity index 94% rename from tests/test_api/test_cancel.py rename to apps/backend/tests/test_api/test_cancel.py index c2f8835..01e9898 100644 --- a/tests/test_api/test_cancel.py +++ b/apps/backend/tests/test_api/test_cancel.py @@ -15,7 +15,8 @@ @pytest.mark.asyncio async def test_cancel_unknown_session(): """DELETE on a session with no active stream returns cancelled=False.""" - from httpx import AsyncClient, ASGITransport + from httpx import ASGITransport, AsyncClient + from api.app import app async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: @@ -28,7 +29,8 @@ async def test_cancel_unknown_session(): @pytest.mark.asyncio async def test_cancel_active_session(): """DELETE on a session with a registered cancel event sets it and returns cancelled=True.""" - from httpx import AsyncClient, ASGITransport + from httpx import ASGITransport, AsyncClient + from api.app import app from api.routers.chat import _cancel_events diff --git a/tests/test_backends/__init__.py b/apps/backend/tests/test_backends/__init__.py similarity index 100% rename from tests/test_backends/__init__.py rename to apps/backend/tests/test_backends/__init__.py diff --git a/tests/test_backends/test_backends.py b/apps/backend/tests/test_backends/test_backends.py similarity index 98% rename from tests/test_backends/test_backends.py rename to apps/backend/tests/test_backends/test_backends.py index dcaeffa..d5ce945 100644 --- a/tests/test_backends/test_backends.py +++ b/apps/backend/tests/test_backends/test_backends.py @@ -11,8 +11,7 @@ import pytest import pytest_asyncio - -from agent.db.base import DatabaseBackend +from opendevops_core.agent.db.base import DatabaseBackend # ── Fixtures ────────────────────────────────────────────────────────────────── @@ -21,10 +20,10 @@ async def backend(request, tmp_path) -> AsyncGenerator[DatabaseBackend, None]: """Parametrised fixture: runs every test twice — once per local backend.""" if request.param == "memory": - from agent.db.memory import MemoryBackend + from opendevops_core.agent.db.memory import MemoryBackend b = MemoryBackend() else: - from agent.db.sqlite import SQLiteBackend + from opendevops_core.agent.db.sqlite import SQLiteBackend b = SQLiteBackend() b._path = str(tmp_path / "agent.db") # isolated per test @@ -40,7 +39,8 @@ async def pg_backend() -> AsyncGenerator[DatabaseBackend, None]: if not url: pytest.skip("DATABASE_URL not set — skipping postgres backend tests") - from agent.db.postgres import PostgresBackend + from opendevops_core.agent.db.postgres import PostgresBackend + from config import settings settings.database_url = url diff --git a/tests/integration/__init__.py b/apps/backend/tests/test_llm/__init__.py similarity index 100% rename from tests/integration/__init__.py rename to apps/backend/tests/test_llm/__init__.py diff --git a/tests/test_llm/test_backend.py b/apps/backend/tests/test_llm/test_backend.py similarity index 98% rename from tests/test_llm/test_backend.py rename to apps/backend/tests/test_llm/test_backend.py index 848193e..7f86366 100644 --- a/tests/test_llm/test_backend.py +++ b/apps/backend/tests/test_llm/test_backend.py @@ -6,8 +6,7 @@ from __future__ import annotations import pytest - -from agent.llm import backend +from opendevops_core.agent.llm import backend class FakeDetector: diff --git a/tests/test_llm/test_claude_code_detector.py b/apps/backend/tests/test_llm/test_claude_code_detector.py similarity index 96% rename from tests/test_llm/test_claude_code_detector.py rename to apps/backend/tests/test_llm/test_claude_code_detector.py index a99d0a8..7da5680 100644 --- a/tests/test_llm/test_claude_code_detector.py +++ b/apps/backend/tests/test_llm/test_claude_code_detector.py @@ -6,9 +6,8 @@ import time import pytest - -from agent.llm.detectors import claude_code -from agent.llm.detectors.claude_code import ClaudeCodeDetector +from opendevops_core.agent.llm.detectors import claude_code +from opendevops_core.agent.llm.detectors.claude_code import ClaudeCodeDetector @pytest.fixture diff --git a/tests/test_llm/test_identity.py b/apps/backend/tests/test_llm/test_identity.py similarity index 87% rename from tests/test_llm/test_identity.py rename to apps/backend/tests/test_llm/test_identity.py index 6132ea5..554d202 100644 --- a/tests/test_llm/test_identity.py +++ b/apps/backend/tests/test_llm/test_identity.py @@ -2,7 +2,11 @@ from __future__ import annotations -from agent.llm.identity import CLAUDE_CODE_IDENTITY, is_subscription_token, shape_system_content +from opendevops_core.agent.llm.identity import ( + CLAUDE_CODE_IDENTITY, + is_subscription_token, + shape_system_content, +) def test_is_subscription_token(): diff --git a/tests/test_agent/__init__.py b/apps/backend/tests/test_tools/__init__.py similarity index 100% rename from tests/test_agent/__init__.py rename to apps/backend/tests/test_tools/__init__.py diff --git a/tests/test_tools/test_bash_tool.py b/apps/backend/tests/test_tools/test_bash_tool.py similarity index 90% rename from tests/test_tools/test_bash_tool.py rename to apps/backend/tests/test_tools/test_bash_tool.py index a20c45e..6cf634c 100644 --- a/tests/test_tools/test_bash_tool.py +++ b/apps/backend/tests/test_tools/test_bash_tool.py @@ -6,8 +6,7 @@ from unittest.mock import MagicMock import pytest - -from tools.bash_tool import run_bash_command +from opendevops_core.tools.bash_tool import run_bash_command # ── Helpers ────────────────────────────────────────────────────────────────── @@ -67,7 +66,7 @@ def _ok_proc(stdout: str = "output", stderr: str = "") -> MagicMock: "docker inspect my-container", ]) def test_allowlisted_commands_pass(mocker, cmd): - mocker.patch("tools.bash_tool.subprocess.run", return_value=_ok_proc()) + mocker.patch("opendevops_core.tools.bash_tool.subprocess.run", return_value=_ok_proc()) result = run_bash_command(cmd) assert result["blocked"] is False assert result["success"] is True @@ -128,7 +127,7 @@ def test_blocked_commands_return_blocked_true(cmd): def test_blocked_command_never_calls_subprocess(mocker): - mock_run = mocker.patch("tools.bash_tool.subprocess.run") + mock_run = mocker.patch("opendevops_core.tools.bash_tool.subprocess.run") run_bash_command("aws s3 ls") mock_run.assert_not_called() @@ -137,7 +136,7 @@ def test_blocked_command_never_calls_subprocess(mocker): def test_timeout_returns_structured_error(mocker): mocker.patch( - "tools.bash_tool.subprocess.run", + "opendevops_core.tools.bash_tool.subprocess.run", side_effect=subprocess.TimeoutExpired(cmd="aws logs describe-log-groups", timeout=30), ) result = run_bash_command("aws logs describe-log-groups") @@ -149,7 +148,7 @@ def test_timeout_returns_structured_error(mocker): def test_timeout_does_not_raise(mocker): mocker.patch( - "tools.bash_tool.subprocess.run", + "opendevops_core.tools.bash_tool.subprocess.run", side_effect=subprocess.TimeoutExpired(cmd="aws cloudwatch describe-alarms", timeout=30), ) # must not raise — tool always returns a dict @@ -161,7 +160,7 @@ def test_timeout_does_not_raise(mocker): def test_file_not_found_is_caught(mocker): mocker.patch( - "tools.bash_tool.subprocess.run", + "opendevops_core.tools.bash_tool.subprocess.run", side_effect=FileNotFoundError("aws: command not found"), ) result = run_bash_command("aws logs describe-log-groups") @@ -172,7 +171,7 @@ def test_file_not_found_is_caught(mocker): def test_unexpected_exception_never_raises(mocker): mocker.patch( - "tools.bash_tool.subprocess.run", + "opendevops_core.tools.bash_tool.subprocess.run", side_effect=RuntimeError("unexpected failure"), ) result = run_bash_command("aws ec2 describe-instances") @@ -191,14 +190,14 @@ def test_blocked_result_has_all_keys(): def test_success_result_has_all_keys(mocker): - mocker.patch("tools.bash_tool.subprocess.run", return_value=_ok_proc()) + mocker.patch("opendevops_core.tools.bash_tool.subprocess.run", return_value=_ok_proc()) result = run_bash_command("aws logs describe-log-groups") assert set(result.keys()) == _EXPECTED_KEYS def test_timeout_result_has_all_keys(mocker): mocker.patch( - "tools.bash_tool.subprocess.run", + "opendevops_core.tools.bash_tool.subprocess.run", side_effect=subprocess.TimeoutExpired(cmd="x", timeout=30), ) result = run_bash_command("aws cloudwatch describe-alarms") @@ -209,7 +208,7 @@ def test_timeout_result_has_all_keys(mocker): def test_stdout_is_capped(mocker): mocker.patch( - "tools.bash_tool.subprocess.run", + "opendevops_core.tools.bash_tool.subprocess.run", return_value=_ok_proc(stdout="x" * 10_000), ) result = run_bash_command("aws logs describe-log-groups") @@ -218,7 +217,7 @@ def test_stdout_is_capped(mocker): def test_stderr_is_capped(mocker): mocker.patch( - "tools.bash_tool.subprocess.run", + "opendevops_core.tools.bash_tool.subprocess.run", return_value=MagicMock(returncode=1, stdout="", stderr="e" * 5_000), ) result = run_bash_command("aws logs describe-log-groups") @@ -229,12 +228,12 @@ def test_stderr_is_capped(mocker): def test_command_field_echoes_input(mocker): cmd = "aws ec2 describe-instances --region us-east-1" - mocker.patch("tools.bash_tool.subprocess.run", return_value=_ok_proc()) + mocker.patch("opendevops_core.tools.bash_tool.subprocess.run", return_value=_ok_proc()) result = run_bash_command(cmd) assert result["command"] == cmd def test_command_is_stripped(mocker): - mocker.patch("tools.bash_tool.subprocess.run", return_value=_ok_proc()) + mocker.patch("opendevops_core.tools.bash_tool.subprocess.run", return_value=_ok_proc()) result = run_bash_command(" aws logs describe-log-groups ") assert result["command"] == "aws logs describe-log-groups" diff --git a/tests/test_tools/test_cap.py b/apps/backend/tests/test_tools/test_cap.py similarity index 96% rename from tests/test_tools/test_cap.py rename to apps/backend/tests/test_tools/test_cap.py index 689de9a..53d00ef 100644 --- a/tests/test_tools/test_cap.py +++ b/apps/backend/tests/test_tools/test_cap.py @@ -4,7 +4,7 @@ import json -from tools._cap import cap_tool_result, with_cap +from opendevops_core.tools._cap import cap_tool_result, with_cap def test_small_result_passes_through(): diff --git a/tests/test_tools/test_cloudwatch.py b/apps/backend/tests/test_tools/test_cloudwatch.py similarity index 98% rename from tests/test_tools/test_cloudwatch.py rename to apps/backend/tests/test_tools/test_cloudwatch.py index ad972b1..7d44693 100644 --- a/tests/test_tools/test_cloudwatch.py +++ b/apps/backend/tests/test_tools/test_cloudwatch.py @@ -1,7 +1,6 @@ import boto3 from moto import mock_aws - -from providers.aws.tools.cloudwatch import ( +from opendevops_core.providers.aws.tools.cloudwatch import ( describe_log_groups, get_alarm_history, get_alarms, diff --git a/tests/test_tools/test_ecs.py b/apps/backend/tests/test_tools/test_ecs.py similarity index 88% rename from tests/test_tools/test_ecs.py rename to apps/backend/tests/test_tools/test_ecs.py index e9664b4..ad33d86 100644 --- a/tests/test_tools/test_ecs.py +++ b/apps/backend/tests/test_tools/test_ecs.py @@ -1,7 +1,10 @@ import boto3 from moto import mock_aws - -from providers.aws.tools.ecs import describe_ecs_service, list_ecs_clusters, list_ecs_services +from opendevops_core.providers.aws.tools.ecs import ( + describe_ecs_service, + list_ecs_clusters, + list_ecs_services, +) @mock_aws diff --git a/tests/test_tools/test_lambda.py b/apps/backend/tests/test_tools/test_lambda.py similarity index 93% rename from tests/test_tools/test_lambda.py rename to apps/backend/tests/test_tools/test_lambda.py index 8d854fb..0f60302 100644 --- a/tests/test_tools/test_lambda.py +++ b/apps/backend/tests/test_tools/test_lambda.py @@ -1,7 +1,10 @@ import boto3 from moto import mock_aws - -from providers.aws.tools.lambda_ import get_lambda_error_rate, get_lambda_function_config, list_lambda_functions +from opendevops_core.providers.aws.tools.lambda_ import ( + get_lambda_error_rate, + get_lambda_function_config, + list_lambda_functions, +) def _create_lambda_role(iam_client: boto3.client) -> str: diff --git a/tests/test_tools/test_rds.py b/apps/backend/tests/test_tools/test_rds.py similarity index 89% rename from tests/test_tools/test_rds.py rename to apps/backend/tests/test_tools/test_rds.py index 2f05281..77e63d5 100644 --- a/tests/test_tools/test_rds.py +++ b/apps/backend/tests/test_tools/test_rds.py @@ -1,7 +1,6 @@ import boto3 from moto import mock_aws - -from providers.aws.tools.rds import describe_rds_instances, get_rds_events +from opendevops_core.providers.aws.tools.rds import describe_rds_instances, get_rds_events @mock_aws diff --git a/uv.lock b/apps/backend/uv.lock similarity index 99% rename from uv.lock rename to apps/backend/uv.lock index 8eb9f42..e574b67 100644 --- a/uv.lock +++ b/apps/backend/uv.lock @@ -2148,13 +2148,55 @@ name = "opendevops-agent" version = "0.1.0" source = { editable = "." } dependencies = [ - { name = "aiosqlite" }, { name = "bcrypt" }, + { name = "fastapi" }, + { name = "fastmcp" }, + { name = "opendevops-core" }, + { name = "python-jose", extra = ["cryptography"] }, + { name = "rich" }, + { name = "typer" }, + { name = "uvicorn" }, +] + +[package.dev-dependencies] +dev = [ + { name = "moto" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "pytest-mock" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "bcrypt", specifier = ">=5.0.0" }, + { name = "fastapi", specifier = ">=0.111.0" }, + { name = "fastmcp", specifier = ">=3.2.4" }, + { name = "opendevops-core", editable = "../core" }, + { name = "python-jose", extras = ["cryptography"], specifier = ">=3.3.0" }, + { name = "rich", specifier = ">=13.7.0" }, + { name = "typer", specifier = ">=0.12.0" }, + { name = "uvicorn", specifier = ">=0.30.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "moto", extras = ["cloudwatch", "logs", "ec2", "ecs", "lambda", "rds", "iam", "cloudtrail"], specifier = ">=5.0.0" }, + { name = "pytest", specifier = ">=8.0.0" }, + { name = "pytest-asyncio", specifier = ">=0.23.0" }, + { name = "pytest-mock", specifier = ">=3.14.0" }, + { name = "ruff", specifier = ">=0.4.0" }, +] + +[[package]] +name = "opendevops-core" +version = "0.1.0" +source = { editable = "../core" } +dependencies = [ + { name = "aiosqlite" }, { name = "boto3" }, { name = "cachetools" }, { name = "deepagents" }, - { name = "fastapi" }, - { name = "fastmcp" }, { name = "httpx" }, { name = "langchain-community" }, { name = "langchain-litellm" }, @@ -2168,30 +2210,14 @@ dependencies = [ { name = "pydantic" }, { name = "pydantic-settings" }, { name = "python-dotenv" }, - { name = "python-jose", extra = ["cryptography"] }, - { name = "rich" }, - { name = "typer" }, - { name = "uvicorn" }, -] - -[package.dev-dependencies] -dev = [ - { name = "moto" }, - { name = "pytest" }, - { name = "pytest-asyncio" }, - { name = "pytest-mock" }, - { name = "ruff" }, ] [package.metadata] requires-dist = [ { name = "aiosqlite", specifier = ">=0.20.0" }, - { name = "bcrypt", specifier = ">=5.0.0" }, { name = "boto3", specifier = ">=1.34.0" }, { name = "cachetools", specifier = ">=5.3.0" }, { name = "deepagents" }, - { name = "fastapi", specifier = ">=0.111.0" }, - { name = "fastmcp", specifier = ">=3.2.4" }, { name = "httpx", specifier = ">=0.27.0" }, { name = "langchain-community", specifier = ">=0.4.1" }, { name = "langchain-litellm", specifier = ">=0.6.4" }, @@ -2205,19 +2231,6 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.7.0" }, { name = "pydantic-settings", specifier = ">=2.3.0" }, { name = "python-dotenv", specifier = ">=1.0.0" }, - { name = "python-jose", extras = ["cryptography"], specifier = ">=3.3.0" }, - { name = "rich", specifier = ">=13.7.0" }, - { name = "typer", specifier = ">=0.12.0" }, - { name = "uvicorn", specifier = ">=0.30.0" }, -] - -[package.metadata.requires-dev] -dev = [ - { name = "moto", extras = ["cloudwatch", "logs", "ec2", "ecs", "lambda", "rds", "iam", "cloudtrail"], specifier = ">=5.0.0" }, - { name = "pytest", specifier = ">=8.0.0" }, - { name = "pytest-asyncio", specifier = ">=0.23.0" }, - { name = "pytest-mock", specifier = ">=3.14.0" }, - { name = "ruff", specifier = ">=0.4.0" }, ] [[package]] diff --git a/apps/core/pyproject.toml b/apps/core/pyproject.toml new file mode 100644 index 0000000..415c699 --- /dev/null +++ b/apps/core/pyproject.toml @@ -0,0 +1,41 @@ +[project] +name = "opendevops-core" +version = "0.1.0" +description = "Shared agent core for OpenDevOps — DeepAgents loop, AWS tools, providers, and LLM wiring" +requires-python = ">=3.11" +dependencies = [ + "deepagents", + "langchain-openai>=0.1.0", + "langgraph>=0.2.0", + "boto3>=1.34.0", + "pydantic>=2.7.0", + "pydantic-settings>=2.3.0", + "python-dotenv>=1.0.0", + "loguru>=0.7.0", + "langgraph-checkpoint-postgres>=2.0.0", + "langgraph-checkpoint-sqlite>=2.0.0", + "psycopg[binary,pool]>=3.1.0", + "aiosqlite>=0.20.0", + "cachetools>=5.3.0", + "httpx>=0.27.0", + "litellm>=1.83.0", + "langchain-community>=0.4.1", + "langchain-litellm>=0.6.4", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/opendevops_core"] + +[tool.uv] +package = true + +[tool.ruff] +line-length = 100 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "I", "UP"] diff --git a/apps/core/src/opendevops_core/__init__.py b/apps/core/src/opendevops_core/__init__.py new file mode 100644 index 0000000..ce4ff53 --- /dev/null +++ b/apps/core/src/opendevops_core/__init__.py @@ -0,0 +1,6 @@ +"""opendevops_core — shared agent core (DeepAgents loop, AWS tools, providers, LLM wiring). + +Host applications import from this package and supply their own config + API/CLI layer. +""" + +__version__ = "0.1.0" diff --git a/tests/test_api/__init__.py b/apps/core/src/opendevops_core/agent/__init__.py similarity index 100% rename from tests/test_api/__init__.py rename to apps/core/src/opendevops_core/agent/__init__.py diff --git a/src/agent/core.py b/apps/core/src/opendevops_core/agent/core.py similarity index 86% rename from src/agent/core.py rename to apps/core/src/opendevops_core/agent/core.py index c95eaa1..926ec48 100644 --- a/src/agent/core.py +++ b/apps/core/src/opendevops_core/agent/core.py @@ -8,16 +8,22 @@ from deepagents import create_deep_agent from langchain_litellm import ChatLiteLLM -from config import settings -from models.agent import Confidence, Investigation, InvestigationResult, RootCauseCategory -from agent.prompts import SYSTEM_PROMPT from loguru import logger -from providers import get_active_provider -from tools._cap import with_cap -from tools.bash_tool import ALL_BASH_TOOLS -from tools.final_answer import ALL_FINAL_ANSWER_TOOLS -from tools.history import ALL_HISTORY_TOOLS -from tools.skills import ALL_SKILL_TOOLS + +from opendevops_core.agent.prompts import SYSTEM_PROMPT +from opendevops_core.config import settings +from opendevops_core.models.agent import ( + Confidence, + Investigation, + InvestigationResult, + RootCauseCategory, +) +from opendevops_core.providers import get_active_provider +from opendevops_core.tools._cap import with_cap +from opendevops_core.tools.bash_tool import ALL_BASH_TOOLS +from opendevops_core.tools.final_answer import ALL_FINAL_ANSWER_TOOLS +from opendevops_core.tools.history import ALL_HISTORY_TOOLS +from opendevops_core.tools.skills import ALL_SKILL_TOOLS # Cloud-agnostic tools available regardless of provider. SHARED_TOOLS = ALL_HISTORY_TOOLS + ALL_BASH_TOOLS + ALL_SKILL_TOOLS + ALL_FINAL_ANSWER_TOOLS @@ -51,10 +57,12 @@ def _run_async(coro: Any) -> Any: def _build_system_prompt(api_key: str | None) -> Any: """Return the system prompt for create_deep_agent — a SystemMessage with the Claude Code identity block for subscription OAuth tokens, else a plain string.""" - from agent.llm import shape_system_content + from opendevops_core.agent.llm import shape_system_content + content = shape_system_content(SYSTEM_PROMPT, api_key) if isinstance(content, list): from langchain_core.messages import SystemMessage + return SystemMessage(content=content) return content @@ -62,7 +70,8 @@ def _build_system_prompt(api_key: str | None) -> Any: def init_agent(checkpointer: Any) -> None: """Create the agent with the given checkpointer. Called once during app startup.""" global _agent, _active_model - from agent.llm import resolve_model_and_key + from opendevops_core.agent.llm import resolve_model_and_key + model_name, api_key = resolve_model_and_key() _active_model = model_name model = ChatLiteLLM( @@ -90,13 +99,15 @@ def ensure_agent_initialized() -> None: global _agent if _agent is not None: return - from agent.db import db + from opendevops_core.agent.db import db checkpointer = _run_async(db.init()) init_agent(checkpointer) -async def ainvoke_with_timeout(agent_input: dict[str, Any], config: dict[str, Any]) -> dict[str, Any]: +async def ainvoke_with_timeout( + agent_input: dict[str, Any], config: dict[str, Any] +) -> dict[str, Any]: """Run agent invocation with a global investigation timeout.""" return await asyncio.wait_for( get_agent().ainvoke(agent_input, config=config), diff --git a/src/agent/db/__init__.py b/apps/core/src/opendevops_core/agent/db/__init__.py similarity index 60% rename from src/agent/db/__init__.py rename to apps/core/src/opendevops_core/agent/db/__init__.py index da68bb8..811cdf7 100644 --- a/src/agent/db/__init__.py +++ b/apps/core/src/opendevops_core/agent/db/__init__.py @@ -1,29 +1,32 @@ """Database package — selects and exposes the right backend via the `db` singleton. Import pattern (unchanged from the old db.py): - from agent.db import db + from opendevops_core.agent.db import db """ from __future__ import annotations -from agent.db.base import DatabaseBackend +from opendevops_core.agent.db.base import DatabaseBackend def _create_backend() -> DatabaseBackend: - from config import settings + from opendevops_core.config import settings backend = settings.checkpoint_backend if backend == "postgres": - from agent.db.postgres import PostgresBackend + from opendevops_core.agent.db.postgres import PostgresBackend + return PostgresBackend() if backend == "sqlite": - from agent.db.sqlite import SQLiteBackend + from opendevops_core.agent.db.sqlite import SQLiteBackend + return SQLiteBackend() # memory (default — zero config, no persistence) - from agent.db.memory import MemoryBackend + from opendevops_core.agent.db.memory import MemoryBackend + return MemoryBackend() diff --git a/src/agent/db/base.py b/apps/core/src/opendevops_core/agent/db/base.py similarity index 93% rename from src/agent/db/base.py rename to apps/core/src/opendevops_core/agent/db/base.py index 7d01296..07014ae 100644 --- a/src/agent/db/base.py +++ b/apps/core/src/opendevops_core/agent/db/base.py @@ -32,6 +32,8 @@ async def upsert_session( aws_region: str, title: str | None = None, source: str = "chat", + org_id: str | None = None, + user_id: str | None = None, ) -> None: ... @abstractmethod @@ -69,10 +71,12 @@ async def save_usage_event( ) -> None: ... @abstractmethod - async def list_sessions(self, limit: int = 15, offset: int = 0) -> list[dict]: ... + async def list_sessions( + self, limit: int = 15, offset: int = 0, org_id: str | None = None + ) -> list[dict]: ... @abstractmethod - async def get_messages(self, session_id: str) -> list[dict]: ... + async def get_messages(self, session_id: str, org_id: str | None = None) -> list[dict]: ... @abstractmethod async def delete_session(self, session_id: str) -> None: ... @@ -110,7 +114,11 @@ async def get_first_org(self) -> dict | None: return None async def create_user( - self, email: str, name: str, password_hash: str, role: str, + self, + email: str, + name: str, + password_hash: str, + role: str, org_id: str | None = None, ) -> dict | None: return None diff --git a/src/agent/db/memory.py b/apps/core/src/opendevops_core/agent/db/memory.py similarity index 75% rename from src/agent/db/memory.py rename to apps/core/src/opendevops_core/agent/db/memory.py index 32b089e..bad9d4f 100644 --- a/src/agent/db/memory.py +++ b/apps/core/src/opendevops_core/agent/db/memory.py @@ -9,7 +9,7 @@ from loguru import logger -from agent.db.base import DatabaseBackend +from opendevops_core.agent.db.base import DatabaseBackend class MemoryBackend(DatabaseBackend): @@ -29,6 +29,7 @@ def __init__(self) -> None: async def init(self) -> Any: from langgraph.checkpoint.memory import MemorySaver + self._checkpointer = MemorySaver() logger.warning( "In-memory backend active — all data is lost on restart and memory grows " @@ -57,12 +58,16 @@ async def upsert_session( aws_region: str, title: str | None = None, source: str = "chat", + org_id: str | None = None, + user_id: str | None = None, ) -> None: if session_id in self._sessions: self._sessions[session_id]["last_active_at"] = self._now() self._sessions[session_id]["model"] = model if source == "chat": self._sessions[session_id]["user_interacted"] = True + self._sessions[session_id].setdefault("org_id", org_id) + self._sessions[session_id].setdefault("user_id", user_id) else: self._sessions[session_id] = { "id": session_id, @@ -70,6 +75,8 @@ async def upsert_session( "model": model, "aws_region": aws_region, "source": source, + "org_id": org_id, + "user_id": user_id, "user_interacted": source == "chat", "created_at": self._now(), "last_active_at": self._now(), @@ -84,14 +91,16 @@ async def save_message( metadata: dict | None = None, ) -> str: msg_id = str(uuid.uuid4()) - self._messages[session_id].append({ - "id": msg_id, - "session_id": session_id, - "role": role, - "content": content, - "metadata": metadata or {}, - "created_at": self._now(), - }) + self._messages[session_id].append( + { + "id": msg_id, + "session_id": session_id, + "role": role, + "content": content, + "metadata": metadata or {}, + "created_at": self._now(), + } + ) return msg_id async def save_tool_call( @@ -104,17 +113,19 @@ async def save_tool_call( duration_ms: int | None = None, ) -> None: error = result.get("error") if isinstance(result, dict) else None - self._tool_calls[session_id].append({ - "id": str(uuid.uuid4()), - "session_id": session_id, - "message_id": message_id, - "tool_name": tool_name, - "args": args, - "result": result, - "error": error, - "duration_ms": duration_ms, - "created_at": self._now(), - }) + self._tool_calls[session_id].append( + { + "id": str(uuid.uuid4()), + "session_id": session_id, + "message_id": message_id, + "tool_name": tool_name, + "args": args, + "result": result, + "error": error, + "duration_ms": duration_ms, + "created_at": self._now(), + } + ) async def save_usage_event( self, @@ -128,41 +139,51 @@ async def save_usage_event( tool_call_count: int, metadata: dict | None = None, ) -> None: - self._usage[session_id].append({ - "session_id": session_id, - "message_id": message_id, - "model": model, - "input_tokens": input_tokens, - "output_tokens": output_tokens, - "cost_usd": cost_usd, - "latency_ms": latency_ms, - "tool_call_count": tool_call_count, - "metadata": metadata or {}, - }) - - async def list_sessions(self, limit: int = 15, offset: int = 0) -> list[dict]: + self._usage[session_id].append( + { + "session_id": session_id, + "message_id": message_id, + "model": model, + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "cost_usd": cost_usd, + "latency_ms": latency_ms, + "tool_call_count": tool_call_count, + "metadata": metadata or {}, + } + ) + + async def list_sessions( + self, limit: int = 15, offset: int = 0, org_id: str | None = None + ) -> list[dict]: active = [ - s for s in self._sessions.values() + s + for s in self._sessions.values() if not s.get("is_deleted") and (s.get("source", "chat") == "chat" or s.get("user_interacted", False)) + and (org_id is None or s.get("org_id") == org_id) ] sorted_sessions = sorted(active, key=lambda s: s["last_active_at"], reverse=True) - return sorted_sessions[offset: offset + limit] + return sorted_sessions[offset : offset + limit] - async def get_messages(self, session_id: str) -> list[dict]: + async def get_messages(self, session_id: str, org_id: str | None = None) -> list[dict]: session = self._sessions.get(session_id) if session is None or session.get("is_deleted"): return [] + if org_id is not None and session.get("org_id") != org_id: + return [] tc_by_msg: dict[str, list] = defaultdict(list) for tc in self._tool_calls.get(session_id, []): if tc["message_id"]: - tc_by_msg[tc["message_id"]].append({ - "tool_name": tc["tool_name"], - "args": tc["args"], - "result": tc["result"], - "error": tc["error"], - }) + tc_by_msg[tc["message_id"]].append( + { + "tool_name": tc["tool_name"], + "args": tc["args"], + "result": tc["result"], + "error": tc["error"], + } + ) usage_by_msg = { u["message_id"]: { @@ -208,27 +229,25 @@ async def get_dashboard_stats(self) -> dict: all_usage = [u for us in self._usage.values() for u in us] all_msgs = [m for ms in self._messages.values() for m in ms] - total_cost = sum(u["cost_usd"] or 0 for u in all_usage) - avg_latency = ( - sum(u["latency_ms"] for u in all_usage) // len(all_usage) - if all_usage else 0 - ) + total_cost = sum(u["cost_usd"] or 0 for u in all_usage) + avg_latency = sum(u["latency_ms"] for u in all_usage) // len(all_usage) if all_usage else 0 summ_events = [ - u for u in all_usage + u + for u in all_usage if isinstance(u.get("metadata"), dict) and u["metadata"].get("summarization") ] return { "summary": { - "total_sessions": len(active), - "total_queries": sum(1 for m in all_msgs if m["role"] == "user"), - "total_tool_calls": len(all_tc), + "total_sessions": len(active), + "total_queries": sum(1 for m in all_msgs if m["role"] == "user"), + "total_tool_calls": len(all_tc), "total_tool_errors": sum(1 for tc in all_tc if tc.get("error")), - "total_input_tokens": sum(u["input_tokens"] for u in all_usage), + "total_input_tokens": sum(u["input_tokens"] for u in all_usage), "total_output_tokens": sum(u["output_tokens"] for u in all_usage), - "total_cost_usd": total_cost, - "avg_latency_ms": avg_latency, - "total_summarizations": len(summ_events), + "total_cost_usd": total_cost, + "avg_latency_ms": avg_latency, + "total_summarizations": len(summ_events), "total_chars_compacted": sum( e["metadata"].get("chars_removed", 0) for e in summ_events ), @@ -305,10 +324,7 @@ async def create_user( async def list_users(self) -> list[dict]: users = sorted(self._users.values(), key=lambda item: item["created_at"]) - return [ - {k: v for k, v in user.items() if k != "password_hash"} - for user in users - ] + return [{k: v for k, v in user.items() if k != "password_hash"} for user in users] async def update_user(self, user_id: str, **fields: Any) -> dict | None: user = self._users.get(user_id) @@ -342,20 +358,22 @@ async def add_alert( trigger_source: str | None = None, ) -> str: alert_id = str(uuid.uuid4()) - self._alerts.append({ - "id": alert_id, - "service": service, - "error": error, - "resolution": resolution, - "confidence": confidence, - "sns_sent": sns_sent, - "timestamp": self._now(), - "dedup_key": dedup_key, - "status": status, - "session_id": session_id, - "trigger_source": trigger_source, - "notifications": [], - }) + self._alerts.append( + { + "id": alert_id, + "service": service, + "error": error, + "resolution": resolution, + "confidence": confidence, + "sns_sent": sns_sent, + "timestamp": self._now(), + "dedup_key": dedup_key, + "status": status, + "session_id": session_id, + "trigger_source": trigger_source, + "notifications": [], + } + ) return alert_id async def add_notification( @@ -367,12 +385,14 @@ async def add_notification( ) -> None: for alert in self._alerts: if alert["id"] == alert_id: - alert.setdefault("notifications", []).append({ - "channel": channel, - "status": status, - "error": error, - "sent_at": self._now(), - }) + alert.setdefault("notifications", []).append( + { + "channel": channel, + "status": status, + "error": error, + "sent_at": self._now(), + } + ) return async def is_recent_alert(self, dedup_key: str, within_minutes: int = 3) -> bool: @@ -418,11 +438,13 @@ async def complete_incident( incident_key, {"trigger_source": "unknown", "claimed_at": datetime.now(UTC)}, ) - claim.update({ - "status": status, - "session_id": session_id, - "completed_at": datetime.now(UTC), - }) + claim.update( + { + "status": status, + "session_id": session_id, + "completed_at": datetime.now(UTC), + } + ) async def release_incident(self, incident_key: str) -> None: if self._incident_claims.get(incident_key, {}).get("status") == "claimed": @@ -436,7 +458,7 @@ async def is_incident_claimed(self, incident_key: str, within_minutes: int = 3) return bool(last and datetime.now(UTC) - last <= timedelta(minutes=within_minutes)) async def get_alerts(self, limit: int = 50) -> list[dict]: - return list(reversed(self._alerts))[:min(limit, 200)] + return list(reversed(self._alerts))[: min(limit, 200)] async def get_alert(self, alert_id: str) -> dict | None: for a in self._alerts: @@ -470,12 +492,14 @@ async def search_sessions(self, query: str, limit: int = 10) -> list[dict]: content_match = True break if title_match or content_match: - results.append({ - "id": s["id"], - "title": s.get("title"), - "last_active_at": s["last_active_at"], - "model": s.get("model"), - "snippet": snippet, - }) + results.append( + { + "id": s["id"], + "title": s.get("title"), + "last_active_at": s["last_active_at"], + "model": s.get("model"), + "snippet": snippet, + } + ) results.sort(key=lambda x: x["last_active_at"], reverse=True) - return results[:min(limit, 20)] + return results[: min(limit, 20)] diff --git a/src/agent/db/postgres.py b/apps/core/src/opendevops_core/agent/db/postgres.py similarity index 82% rename from src/agent/db/postgres.py rename to apps/core/src/opendevops_core/agent/db/postgres.py index d000301..ef8aa3b 100644 --- a/src/agent/db/postgres.py +++ b/apps/core/src/opendevops_core/agent/db/postgres.py @@ -7,8 +7,8 @@ from loguru import logger -from agent.db.base import DatabaseBackend -from config import settings +from opendevops_core.agent.db.base import DatabaseBackend +from opendevops_core.config import settings class PostgresBackend(DatabaseBackend): @@ -55,6 +55,7 @@ async def init(self) -> Any: def _use_memory(self) -> Any: from langgraph.checkpoint.memory import MemorySaver + self._checkpointer = MemorySaver() return self._checkpointer @@ -72,6 +73,7 @@ def checkpointer(self) -> Any: @staticmethod def _jsonb(value: Any) -> Any: from psycopg.types.json import Jsonb # type: ignore + return Jsonb(value) async def _exec(self, query: str, *params: Any) -> None: @@ -124,20 +126,30 @@ async def upsert_session( aws_region: str, title: str | None = None, source: str = "chat", + org_id: str | None = None, + user_id: str | None = None, ) -> None: await self._exec( """ - INSERT INTO sessions (id, title, model, aws_region, source) - VALUES (%s, %s, %s, %s, %s) + INSERT INTO sessions (id, title, model, aws_region, source, org_id, user_id) + VALUES (%s, %s, %s, %s, %s, %s, %s) ON CONFLICT (id) DO UPDATE SET last_active_at = NOW(), model = EXCLUDED.model, + org_id = COALESCE(sessions.org_id, EXCLUDED.org_id), + user_id = COALESCE(sessions.user_id, EXCLUDED.user_id), user_interacted = CASE WHEN EXCLUDED.source = 'chat' THEN TRUE ELSE sessions.user_interacted END """, - uuid.UUID(session_id), title, model, aws_region, source, + uuid.UUID(session_id), + title, + model, + aws_region, + source, + uuid.UUID(org_id) if org_id else None, + uuid.UUID(user_id) if user_id else None, ) async def save_message( @@ -150,7 +162,11 @@ async def save_message( msg_id = str(uuid.uuid4()) await self._exec( "INSERT INTO messages (id, session_id, role, content, metadata) VALUES (%s, %s, %s, %s, %s)", - uuid.UUID(msg_id), uuid.UUID(session_id), role, content, self._jsonb(metadata or {}), + uuid.UUID(msg_id), + uuid.UUID(session_id), + role, + content, + self._jsonb(metadata or {}), ) return msg_id @@ -173,7 +189,10 @@ async def save_tool_call( uuid.UUID(session_id), uuid.UUID(message_id) if message_id else None, tool_name, - self._jsonb(args), self._jsonb(result), error, duration_ms, + self._jsonb(args), + self._jsonb(result), + error, + duration_ms, ) async def save_usage_event( @@ -197,16 +216,26 @@ async def save_usage_event( """, uuid.UUID(session_id), uuid.UUID(message_id) if message_id else None, - model, input_tokens, output_tokens, cost_usd, latency_ms, tool_call_count, + model, + input_tokens, + output_tokens, + cost_usd, + latency_ms, + tool_call_count, self._jsonb(metadata or {}), ) - async def list_sessions(self, limit: int = 15, offset: int = 0) -> list[dict]: + async def list_sessions( + self, limit: int = 15, offset: int = 0, org_id: str | None = None + ) -> list[dict]: + org_filter = " AND org_id = %s" if org_id else "" + params: list = [uuid.UUID(org_id), limit, offset] if org_id else [limit, offset] rows = await self._fetchall( "SELECT id, title, last_active_at, model, aws_region FROM sessions" " WHERE is_deleted = FALSE AND (source = 'chat' OR user_interacted = TRUE)" - " ORDER BY last_active_at DESC LIMIT %s OFFSET %s", - limit, offset, + + org_filter + + " ORDER BY last_active_at DESC LIMIT %s OFFSET %s", + *params, ) return [ { @@ -219,11 +248,15 @@ async def list_sessions(self, limit: int = 15, offset: int = 0) -> list[dict]: for r in rows ] - async def get_messages(self, session_id: str) -> list[dict]: + async def get_messages(self, session_id: str, org_id: str | None = None) -> list[dict]: uid = uuid.UUID(session_id) - session = await self._fetchrow("SELECT is_deleted FROM sessions WHERE id = %s", uid) + session = await self._fetchrow( + "SELECT is_deleted, org_id FROM sessions WHERE id = %s", uid + ) if session is None or session.get("is_deleted"): return [] + if org_id is not None and str(session.get("org_id")) != str(org_id): + return [] messages = await self._fetchall( "SELECT id, role, content, created_at FROM messages WHERE session_id = %s ORDER BY created_at ASC", @@ -241,12 +274,14 @@ async def get_messages(self, session_id: str) -> list[dict]: tc_by_msg: dict[str, list] = {} for tc in tool_calls: mid = str(tc["message_id"]) - tc_by_msg.setdefault(mid, []).append({ - "tool_name": tc["tool_name"], - "args": tc["args"], - "result": tc["result"], - "error": tc["error"], - }) + tc_by_msg.setdefault(mid, []).append( + { + "tool_name": tc["tool_name"], + "args": tc["args"], + "result": tc["result"], + "error": tc["error"], + } + ) usage_by_msg = { str(u["message_id"]): { @@ -285,24 +320,34 @@ async def delete_session(self, session_id: str) -> None: async def rename_session(self, session_id: str, title: str) -> None: await self._exec( "UPDATE sessions SET title = %s WHERE id = %s AND is_deleted = FALSE", - title, uuid.UUID(session_id), + title, + uuid.UUID(session_id), ) # ── Analytics ───────────────────────────────────────────────────────────── async def get_dashboard_stats(self) -> dict: _SERVICE_MAP: dict[str, str] = { - "get_alarms": "CloudWatch", "get_alarm_history": "CloudWatch", - "get_metric_data": "CloudWatch", "get_log_events": "CloudWatch", - "describe_log_groups": "CloudWatch", "query_logs_insights": "CloudWatch", + "get_alarms": "CloudWatch", + "get_alarm_history": "CloudWatch", + "get_metric_data": "CloudWatch", + "get_log_events": "CloudWatch", + "describe_log_groups": "CloudWatch", + "query_logs_insights": "CloudWatch", "lookup_cloudtrail_events": "CloudTrail", - "list_ecs_clusters": "ECS", "list_ecs_services": "ECS", - "describe_ecs_service": "ECS", "get_ecs_task_logs": "ECS", - "list_lambda_functions": "Lambda", "get_lambda_function_config": "Lambda", + "list_ecs_clusters": "ECS", + "list_ecs_services": "ECS", + "describe_ecs_service": "ECS", + "get_ecs_task_logs": "ECS", + "list_lambda_functions": "Lambda", + "get_lambda_function_config": "Lambda", "get_lambda_error_rate": "Lambda", - "describe_ec2_instances": "EC2", "get_ec2_system_status": "EC2", - "describe_rds_instances": "RDS", "get_rds_events": "RDS", - "get_caller_identity": "IAM", "get_iam_role_policies": "IAM", + "describe_ec2_instances": "EC2", + "get_ec2_system_status": "EC2", + "describe_rds_instances": "RDS", + "get_rds_events": "RDS", + "get_caller_identity": "IAM", + "get_iam_role_policies": "IAM", "submit_investigation": "Agent", } @@ -347,8 +392,12 @@ async def get_dashboard_stats(self) -> dict: service_totals[svc] = service_totals.get(svc, 0) + t["count"] total_calls = sum(service_totals.values()) or 1 service_breakdown = sorted( - [{"service": s, "calls": c, "pct": round(c / total_calls * 100, 1)} for s, c in service_totals.items()], - key=lambda x: x["calls"], reverse=True, + [ + {"service": s, "calls": c, "pct": round(c / total_calls * 100, 1)} + for s, c in service_totals.items() + ], + key=lambda x: x["calls"], + reverse=True, ) recent_rows = await self._fetchall(""" @@ -387,18 +436,20 @@ async def get_dashboard_stats(self) -> dict: return { "summary": { - "total_sessions": int(summary.get("total_sessions", 0) or 0), - "total_queries": int(summary.get("total_queries", 0) or 0), - "total_tool_calls": int(summary.get("total_tool_calls", 0) or 0), + "total_sessions": int(summary.get("total_sessions", 0) or 0), + "total_queries": int(summary.get("total_queries", 0) or 0), + "total_tool_calls": int(summary.get("total_tool_calls", 0) or 0), "total_tool_errors": int(summary.get("total_tool_errors", 0) or 0), - "total_input_tokens": int(summary.get("total_input_tokens", 0) or 0), + "total_input_tokens": int(summary.get("total_input_tokens", 0) or 0), "total_output_tokens": int(summary.get("total_output_tokens", 0) or 0), - "total_cost_usd": float(summary.get("total_cost_usd", 0) or 0), - "avg_latency_ms": round(float(summary.get("avg_latency_ms", 0) or 0)), - "total_summarizations": int(summary.get("total_summarizations", 0) or 0), + "total_cost_usd": float(summary.get("total_cost_usd", 0) or 0), + "avg_latency_ms": round(float(summary.get("avg_latency_ms", 0) or 0)), + "total_summarizations": int(summary.get("total_summarizations", 0) or 0), "total_chars_compacted": int(summary.get("total_chars_compacted", 0) or 0), }, - "activity": [{"date": str(r["day"]), "sessions": int(r["sessions"])} for r in activity_rows], + "activity": [ + {"date": str(r["day"]), "sessions": int(r["sessions"])} for r in activity_rows + ], "top_tools": top_tools, "service_breakdown": service_breakdown, "recent_sessions": recent_sessions, @@ -406,7 +457,8 @@ async def get_dashboard_stats(self) -> dict: } async def get_history_stats(self, days: int = 30) -> dict: - alarm_rows = await self._fetchall(""" + alarm_rows = await self._fetchall( + """ SELECT tc.args->>'alarm_name' AS alarm_name, COUNT(DISTINCT tc.session_id) AS session_count, COUNT(*) AS total_lookups, MAX(s.last_active_at) AS last_seen @@ -415,9 +467,12 @@ async def get_history_stats(self, days: int = 30) -> dict: AND tc.args->>'alarm_name' IS NOT NULL AND s.last_active_at > NOW() - (%s * INTERVAL '1 day') GROUP BY 1 ORDER BY session_count DESC, total_lookups DESC LIMIT 10 - """, days) + """, + days, + ) - lambda_rows = await self._fetchall(""" + lambda_rows = await self._fetchall( + """ SELECT tc.args->>'function_name' AS function_name, COUNT(DISTINCT tc.session_id) AS session_count, COUNT(*) AS total_calls, MAX(s.last_active_at) AS last_seen @@ -427,43 +482,60 @@ async def get_history_stats(self, days: int = 30) -> dict: AND tc.args->>'function_name' IS NOT NULL AND s.last_active_at > NOW() - (%s * INTERVAL '1 day') GROUP BY 1 ORDER BY session_count DESC LIMIT 10 - """, days) + """, + days, + ) - error_rows = await self._fetchall(""" + error_rows = await self._fetchall( + """ SELECT tc.tool_name, LEFT(tc.error, 120) AS error_snippet, COUNT(*) AS count, MAX(tc.created_at) AS last_seen FROM tool_calls tc JOIN sessions s ON s.id = tc.session_id WHERE s.is_deleted = FALSE AND tc.error IS NOT NULL AND s.last_active_at > NOW() - (%s * INTERVAL '1 day') GROUP BY 1, 2 ORDER BY count DESC LIMIT 10 - """, days) + """, + days, + ) - trend_rows = await self._fetchall(""" + trend_rows = await self._fetchall( + """ SELECT DATE_TRUNC('day', last_active_at AT TIME ZONE 'UTC')::date AS day, COUNT(*) AS count FROM sessions WHERE is_deleted = FALSE AND last_active_at > NOW() - (%s * INTERVAL '1 day') GROUP BY 1 ORDER BY 1 - """, days) + """, + days, + ) return { "days": days, "top_alarms": [ - {"alarm_name": r["alarm_name"], "session_count": int(r["session_count"]), - "total_lookups": int(r["total_lookups"]), - "last_seen": r["last_seen"].isoformat() if r["last_seen"] else None} + { + "alarm_name": r["alarm_name"], + "session_count": int(r["session_count"]), + "total_lookups": int(r["total_lookups"]), + "last_seen": r["last_seen"].isoformat() if r["last_seen"] else None, + } for r in alarm_rows ], "top_lambdas": [ - {"function_name": r["function_name"], "session_count": int(r["session_count"]), - "total_calls": int(r["total_calls"]), - "last_seen": r["last_seen"].isoformat() if r["last_seen"] else None} + { + "function_name": r["function_name"], + "session_count": int(r["session_count"]), + "total_calls": int(r["total_calls"]), + "last_seen": r["last_seen"].isoformat() if r["last_seen"] else None, + } for r in lambda_rows ], "recurring_errors": [ - {"tool_name": r["tool_name"], "error_snippet": r["error_snippet"], - "count": int(r["count"]), - "last_seen": r["last_seen"].isoformat() if r["last_seen"] else None} + { + "tool_name": r["tool_name"], + "error_snippet": r["error_snippet"], + "count": int(r["count"]), + "last_seen": r["last_seen"].isoformat() if r["last_seen"] else None, + } for r in error_rows ], "trend": [{"date": str(r["day"]), "count": int(r["count"])} for r in trend_rows], @@ -472,7 +544,9 @@ async def get_history_stats(self, days: int = 30) -> dict: # ── User / auth ─────────────────────────────────────────────────────────── async def count_users(self) -> int: - row = await self._fetchrow("SELECT COUNT(*) AS n FROM users WHERE password_hash IS NOT NULL") + row = await self._fetchrow( + "SELECT COUNT(*) AS n FROM users WHERE password_hash IS NOT NULL" + ) return int(row["n"]) if row else 0 async def get_user_by_email(self, email: str) -> dict | None: @@ -489,7 +563,8 @@ async def create_org(self, name: str, slug: str) -> dict | None: ON CONFLICT (slug) DO UPDATE SET name = EXCLUDED.name RETURNING id, name, slug """, - name, slug, + name, + slug, ) if row: row["id"] = str(row["id"]) @@ -504,7 +579,11 @@ async def get_first_org(self) -> dict | None: return row async def create_user( - self, email: str, name: str, password_hash: str, role: str, + self, + email: str, + name: str, + password_hash: str, + role: str, org_id: str | None = None, ) -> dict | None: if org_id: @@ -514,7 +593,11 @@ async def create_user( VALUES (%s, %s, %s, %s, %s) RETURNING id, email, name, role, created_at """, - email, name, password_hash, role, uuid.UUID(org_id), + email, + name, + password_hash, + role, + uuid.UUID(org_id), ) else: row = await self._fetchrow( @@ -523,7 +606,10 @@ async def create_user( VALUES (%s, %s, %s, %s) RETURNING id, email, name, role, created_at """, - email, name, password_hash, role, + email, + name, + password_hash, + role, ) if row: row["id"] = str(row["id"]) @@ -581,12 +667,19 @@ async def add_alert( evidence: list | None = None, ) -> str: import json as _json + row = await self._fetchrow( "INSERT INTO alerts" " (service, error, resolution, confidence, sns_sent, dedup_key, status," " session_id, trigger_source, evidence)" " VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id", - service, error, resolution, confidence, sns_sent, dedup_key, status, + service, + error, + resolution, + confidence, + sns_sent, + dedup_key, + status, uuid.UUID(session_id) if session_id else None, trigger_source, _json.dumps(evidence or []), @@ -603,14 +696,18 @@ async def add_notification( await self._exec( "INSERT INTO alert_notifications (alert_id, channel, status, error)" " VALUES (%s, %s, %s, %s)", - uuid.UUID(alert_id), channel, status, error, + uuid.UUID(alert_id), + channel, + status, + error, ) async def is_recent_alert(self, dedup_key: str, within_minutes: int = 3) -> bool: row = await self._fetchrow( "SELECT 1 FROM alerts WHERE dedup_key = %s" " AND created_at > NOW() - (%s * INTERVAL '1 minute') LIMIT 1", - dedup_key, within_minutes, + dedup_key, + within_minutes, ) return row is not None @@ -630,7 +727,9 @@ async def claim_incident( " WHERE COALESCE(incident_claims.completed_at, incident_claims.claimed_at)" " < NOW() - (%s * INTERVAL '1 minute')" " RETURNING incident_key", - incident_key, trigger_source, within_minutes, + incident_key, + trigger_source, + within_minutes, ) return row is not None @@ -662,12 +761,14 @@ async def is_incident_claimed(self, incident_key: str, within_minutes: int = 3) "SELECT 1 FROM incident_claims WHERE incident_key = %s" " AND COALESCE(completed_at, claimed_at) > NOW() - (%s * INTERVAL '1 minute')" " LIMIT 1", - incident_key, within_minutes, + incident_key, + within_minutes, ) return row is not None async def get_alerts(self, limit: int = 50) -> list[dict]: import json as _json + rows = await self._fetchall( "SELECT id, service, error, resolution, confidence, sns_sent, status," " created_at, session_id, trigger_source, dedup_key, evidence" @@ -694,6 +795,7 @@ async def get_alerts(self, limit: int = 50) -> list[dict]: async def get_alert(self, alert_id: str) -> dict | None: import json as _json + row = await self._fetchrow( "SELECT id, service, error, resolution, confidence, sns_sent, status," " created_at, session_id, trigger_source, dedup_key, evidence" @@ -746,7 +848,8 @@ async def set_app_config(self, key: str, value: dict) -> None: async def search_sessions(self, query: str, limit: int = 10) -> list[dict]: if not query.strip(): return [] - rows = await self._fetchall(""" + rows = await self._fetchall( + """ SELECT id, title, last_active_at, model, snippet FROM ( SELECT DISTINCT ON (s.id) @@ -758,7 +861,11 @@ async def search_sessions(self, query: str, limit: int = 10) -> list[dict]: AND (s.title ILIKE '%%' || %s || '%%' OR m.content ILIKE '%%' || %s || '%%') ORDER BY s.id, m.created_at ASC ) sub ORDER BY last_active_at DESC LIMIT %s - """, query, query, min(limit, 20)) + """, + query, + query, + min(limit, 20), + ) return [ { "id": str(r["id"]), diff --git a/src/agent/db/sqlite.py b/apps/core/src/opendevops_core/agent/db/sqlite.py similarity index 88% rename from src/agent/db/sqlite.py rename to apps/core/src/opendevops_core/agent/db/sqlite.py index aa6dfb9..df55eea 100644 --- a/src/agent/db/sqlite.py +++ b/apps/core/src/opendevops_core/agent/db/sqlite.py @@ -9,8 +9,8 @@ from loguru import logger -from agent.db.base import DatabaseBackend -from config import settings +from opendevops_core.agent.db.base import DatabaseBackend +from opendevops_core.config import settings _DDL = """ CREATE TABLE IF NOT EXISTS sessions ( @@ -128,17 +128,26 @@ """ _SERVICE_MAP: dict[str, str] = { - "get_alarms": "CloudWatch", "get_alarm_history": "CloudWatch", - "get_metric_data": "CloudWatch", "get_log_events": "CloudWatch", - "describe_log_groups": "CloudWatch", "query_logs_insights": "CloudWatch", + "get_alarms": "CloudWatch", + "get_alarm_history": "CloudWatch", + "get_metric_data": "CloudWatch", + "get_log_events": "CloudWatch", + "describe_log_groups": "CloudWatch", + "query_logs_insights": "CloudWatch", "lookup_cloudtrail_events": "CloudTrail", - "list_ecs_clusters": "ECS", "list_ecs_services": "ECS", - "describe_ecs_service": "ECS", "get_ecs_task_logs": "ECS", - "list_lambda_functions": "Lambda", "get_lambda_function_config": "Lambda", + "list_ecs_clusters": "ECS", + "list_ecs_services": "ECS", + "describe_ecs_service": "ECS", + "get_ecs_task_logs": "ECS", + "list_lambda_functions": "Lambda", + "get_lambda_function_config": "Lambda", "get_lambda_error_rate": "Lambda", - "describe_ec2_instances": "EC2", "get_ec2_system_status": "EC2", - "describe_rds_instances": "RDS", "get_rds_events": "RDS", - "get_caller_identity": "IAM", "get_iam_role_policies": "IAM", + "describe_ec2_instances": "EC2", + "get_ec2_system_status": "EC2", + "describe_rds_instances": "RDS", + "get_rds_events": "RDS", + "get_caller_identity": "IAM", + "get_iam_role_policies": "IAM", "submit_investigation": "Agent", } @@ -319,7 +328,11 @@ async def upsert_session( aws_region: str, title: str | None = None, source: str = "chat", + org_id: str | None = None, + user_id: str | None = None, ) -> None: + # SQLite is the single-tenant/local backend — org_id/user_id are accepted for + # interface parity but not stored or scoped. Use Postgres for multi-tenant. await self._exec( """ INSERT INTO sessions (id, title, model, aws_region, source) @@ -332,7 +345,11 @@ async def upsert_session( ELSE sessions.user_interacted END """, - session_id, title, model, aws_region, source, + session_id, + title, + model, + aws_region, + source, ) async def save_message( @@ -345,7 +362,11 @@ async def save_message( msg_id = str(uuid.uuid4()) await self._exec( "INSERT INTO messages (id, session_id, role, content, metadata) VALUES (?, ?, ?, ?, ?)", - msg_id, session_id, role, content, json.dumps(metadata or {}), + msg_id, + session_id, + role, + content, + json.dumps(metadata or {}), ) return msg_id @@ -365,8 +386,14 @@ async def save_tool_call( (id, session_id, message_id, tool_name, args, result, error, duration_ms) VALUES (?, ?, ?, ?, ?, ?, ?, ?) """, - str(uuid.uuid4()), session_id, message_id, tool_name, - json.dumps(args), json.dumps(result), error, duration_ms, + str(uuid.uuid4()), + session_id, + message_id, + tool_name, + json.dumps(args), + json.dumps(result), + error, + duration_ms, ) async def save_usage_event( @@ -388,17 +415,28 @@ async def save_usage_event( cost_usd, latency_ms, tool_call_count, metadata) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, - str(uuid.uuid4()), session_id, message_id, model, - input_tokens, output_tokens, cost_usd, latency_ms, tool_call_count, + str(uuid.uuid4()), + session_id, + message_id, + model, + input_tokens, + output_tokens, + cost_usd, + latency_ms, + tool_call_count, json.dumps(metadata or {}), ) - async def list_sessions(self, limit: int = 15, offset: int = 0) -> list[dict]: + async def list_sessions( + self, limit: int = 15, offset: int = 0, org_id: str | None = None + ) -> list[dict]: + # org_id ignored — SQLite is single-tenant (see upsert_session note). rows = await self._fetchall( "SELECT id, title, last_active_at, model, aws_region FROM sessions" " WHERE is_deleted = 0 AND (source = 'chat' OR user_interacted = 1)" " ORDER BY last_active_at DESC LIMIT ? OFFSET ?", - limit, offset, + limit, + offset, ) return [ { @@ -411,10 +449,8 @@ async def list_sessions(self, limit: int = 15, offset: int = 0) -> list[dict]: for r in rows ] - async def get_messages(self, session_id: str) -> list[dict]: - session = await self._fetchone( - "SELECT is_deleted FROM sessions WHERE id = ?", session_id - ) + async def get_messages(self, session_id: str, org_id: str | None = None) -> list[dict]: + session = await self._fetchone("SELECT is_deleted FROM sessions WHERE id = ?", session_id) if session is None or session.get("is_deleted"): return [] @@ -438,16 +474,20 @@ async def get_messages(self, session_id: str) -> list[dict]: for tc in tool_calls: mid = tc["message_id"] if mid: - tc_by_msg.setdefault(mid, []).append({ - "tool_name": tc["tool_name"], - "args": json.loads(tc["args"]) if isinstance(tc["args"], str) else tc["args"], - "result": ( - json.loads(tc["result"]) - if isinstance(tc["result"], str) - else tc["result"] - ), - "error": tc["error"], - }) + tc_by_msg.setdefault(mid, []).append( + { + "tool_name": tc["tool_name"], + "args": json.loads(tc["args"]) + if isinstance(tc["args"], str) + else tc["args"], + "result": ( + json.loads(tc["result"]) + if isinstance(tc["result"], str) + else tc["result"] + ), + "error": tc["error"], + } + ) usage_by_msg = { u["message_id"]: { @@ -488,13 +528,15 @@ async def delete_session(self, session_id: str) -> None: async def rename_session(self, session_id: str, title: str) -> None: await self._exec( "UPDATE sessions SET title = ? WHERE id = ? AND is_deleted = 0", - title, session_id, + title, + session_id, ) # ── Analytics ───────────────────────────────────────────────────────────── async def get_dashboard_stats(self) -> dict: - summary = await self._fetchone(""" + summary = ( + await self._fetchone(""" SELECT (SELECT COUNT(*) FROM sessions WHERE is_deleted = 0) AS total_sessions, (SELECT COUNT(*) FROM messages m @@ -528,7 +570,9 @@ async def get_dashboard_stats(self) -> dict: FROM usage_events ue JOIN sessions s ON s.id = ue.session_id WHERE s.is_deleted = 0 AND json_extract(ue.metadata, '$.summarization') = 1) AS total_chars_compacted - """) or {} + """) + or {} + ) activity_rows = await self._fetchall(""" SELECT strftime('%Y-%m-%d', last_active_at) AS day, COUNT(*) AS sessions @@ -611,34 +655,29 @@ async def get_dashboard_stats(self) -> dict: return { "summary": { - "total_sessions": int(summary.get("total_sessions", 0) or 0), - "total_queries": int(summary.get("total_queries", 0) or 0), - "total_tool_calls": int(summary.get("total_tool_calls", 0) or 0), + "total_sessions": int(summary.get("total_sessions", 0) or 0), + "total_queries": int(summary.get("total_queries", 0) or 0), + "total_tool_calls": int(summary.get("total_tool_calls", 0) or 0), "total_tool_errors": int(summary.get("total_tool_errors", 0) or 0), - "total_input_tokens": int(summary.get("total_input_tokens", 0) or 0), + "total_input_tokens": int(summary.get("total_input_tokens", 0) or 0), "total_output_tokens": int(summary.get("total_output_tokens", 0) or 0), - "total_cost_usd": float(summary.get("total_cost_usd", 0) or 0), - "avg_latency_ms": round(float(summary.get("avg_latency_ms", 0) or 0)), - "total_summarizations": int(summary.get("total_summarizations", 0) or 0), + "total_cost_usd": float(summary.get("total_cost_usd", 0) or 0), + "avg_latency_ms": round(float(summary.get("avg_latency_ms", 0) or 0)), + "total_summarizations": int(summary.get("total_summarizations", 0) or 0), "total_chars_compacted": int(summary.get("total_chars_compacted", 0) or 0), }, - "activity": [ - {"date": r["day"], "sessions": int(r["sessions"])} - for r in activity_rows - ], + "activity": [{"date": r["day"], "sessions": int(r["sessions"])} for r in activity_rows], "top_tools": top_tools, "service_breakdown": service_breakdown, "recent_sessions": recent_sessions, - "root_causes": [ - {"category": r["category"], "count": int(r["count"])} - for r in rc_rows - ], + "root_causes": [{"category": r["category"], "count": int(r["count"])} for r in rc_rows], } async def get_history_stats(self, days: int = 30) -> dict: cutoff = f"-{days} days" - alarm_rows = await self._fetchall(""" + alarm_rows = await self._fetchall( + """ SELECT json_extract(tc.args, '$.alarm_name') AS alarm_name, COUNT(DISTINCT tc.session_id) AS session_count, @@ -653,9 +692,12 @@ async def get_history_stats(self, days: int = 30) -> dict: GROUP BY 1 ORDER BY session_count DESC, total_lookups DESC LIMIT 10 - """, cutoff) + """, + cutoff, + ) - lambda_rows = await self._fetchall(""" + lambda_rows = await self._fetchall( + """ SELECT json_extract(tc.args, '$.function_name') AS function_name, COUNT(DISTINCT tc.session_id) AS session_count, @@ -669,9 +711,12 @@ async def get_history_stats(self, days: int = 30) -> dict: AND s.last_active_at > datetime('now', ?) GROUP BY 1 ORDER BY session_count DESC LIMIT 10 - """, cutoff) + """, + cutoff, + ) - error_rows = await self._fetchall(""" + error_rows = await self._fetchall( + """ SELECT tc.tool_name, substr(tc.error, 1, 120) AS error_snippet, @@ -684,9 +729,12 @@ async def get_history_stats(self, days: int = 30) -> dict: AND s.last_active_at > datetime('now', ?) GROUP BY 1, 2 ORDER BY count DESC LIMIT 10 - """, cutoff) + """, + cutoff, + ) - trend_rows = await self._fetchall(""" + trend_rows = await self._fetchall( + """ SELECT strftime('%Y-%m-%d', last_active_at) AS day, COUNT(*) AS count @@ -694,16 +742,18 @@ async def get_history_stats(self, days: int = 30) -> dict: WHERE is_deleted = 0 AND last_active_at > datetime('now', ?) GROUP BY 1 ORDER BY 1 - """, cutoff) + """, + cutoff, + ) return { "days": days, "top_alarms": [ { - "alarm_name": r["alarm_name"], + "alarm_name": r["alarm_name"], "session_count": int(r["session_count"]), "total_lookups": int(r["total_lookups"]), - "last_seen": r["last_seen"], + "last_seen": r["last_seen"], } for r in alarm_rows ], @@ -711,24 +761,21 @@ async def get_history_stats(self, days: int = 30) -> dict: { "function_name": r["function_name"], "session_count": int(r["session_count"]), - "total_calls": int(r["total_calls"]), - "last_seen": r["last_seen"], + "total_calls": int(r["total_calls"]), + "last_seen": r["last_seen"], } for r in lambda_rows ], "recurring_errors": [ { - "tool_name": r["tool_name"], + "tool_name": r["tool_name"], "error_snippet": r["error_snippet"], - "count": int(r["count"]), - "last_seen": r["last_seen"], + "count": int(r["count"]), + "last_seen": r["last_seen"], } for r in error_rows ], - "trend": [ - {"date": r["day"], "count": int(r["count"])} - for r in trend_rows - ], + "trend": [{"date": r["day"], "count": int(r["count"])} for r in trend_rows], } # ── User / auth ───────────────────────────────────────────────────────── @@ -836,14 +883,23 @@ async def add_alert( evidence: list | None = None, ) -> str: import json as _json + alert_id = str(uuid.uuid4()) await self._exec( "INSERT INTO alerts" " (id, service, error, resolution, confidence, sns_sent, dedup_key, status," " session_id, trigger_source, evidence)" " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", - alert_id, service, error, resolution, confidence, - 1 if sns_sent else 0, dedup_key, status, session_id, trigger_source, + alert_id, + service, + error, + resolution, + confidence, + 1 if sns_sent else 0, + dedup_key, + status, + session_id, + trigger_source, _json.dumps(evidence or []), ) return alert_id @@ -858,7 +914,11 @@ async def add_notification( await self._exec( "INSERT INTO alert_notifications (id, alert_id, channel, status, error)" " VALUES (?, ?, ?, ?, ?)", - str(uuid.uuid4()), alert_id, channel, status, error, + str(uuid.uuid4()), + alert_id, + channel, + status, + error, ) async def is_recent_alert(self, dedup_key: str, within_minutes: int = 3) -> bool: @@ -916,7 +976,9 @@ async def complete_incident( "ON CONFLICT(incident_key) DO UPDATE SET " "status = excluded.status, session_id = excluded.session_id, " "completed_at = excluded.completed_at", - incident_key, status, session_id, + incident_key, + status, + session_id, ) async def release_incident(self, incident_key: str) -> None: @@ -936,6 +998,7 @@ async def is_incident_claimed(self, incident_key: str, within_minutes: int = 3) async def get_alerts(self, limit: int = 50) -> list[dict]: import json as _json + rows = await self._fetchall( "SELECT id, service, error, resolution, confidence, sns_sent, status, " " created_at, session_id, trigger_source, dedup_key, evidence " @@ -962,6 +1025,7 @@ async def get_alerts(self, limit: int = 50) -> list[dict]: async def get_alert(self, alert_id: str) -> dict | None: import json as _json + row = await self._fetchone( "SELECT id, service, error, resolution, confidence, sns_sent, status, " " created_at, session_id, trigger_source, dedup_key, evidence " @@ -1014,7 +1078,8 @@ async def search_sessions(self, query: str, limit: int = 10) -> list[dict]: if not query.strip(): return [] pattern = f"%{query}%" - rows = await self._fetchall(""" + rows = await self._fetchall( + """ SELECT id, title, last_active_at, model, snippet FROM ( SELECT @@ -1029,14 +1094,18 @@ async def search_sessions(self, query: str, limit: int = 10) -> list[dict]: WHERE rn = 1 ORDER BY last_active_at DESC LIMIT ? - """, pattern, pattern, min(limit, 20)) + """, + pattern, + pattern, + min(limit, 20), + ) return [ { - "id": r["id"], - "title": r["title"], + "id": r["id"], + "title": r["title"], "last_active_at": r["last_active_at"], - "model": r["model"], - "snippet": r["snippet"], + "model": r["model"], + "snippet": r["snippet"], } for r in rows ] diff --git a/src/agent/incident_keys.py b/apps/core/src/opendevops_core/agent/incident_keys.py similarity index 89% rename from src/agent/incident_keys.py rename to apps/core/src/opendevops_core/agent/incident_keys.py index d6c3f82..9663697 100644 --- a/src/agent/incident_keys.py +++ b/apps/core/src/opendevops_core/agent/incident_keys.py @@ -7,12 +7,20 @@ import re from typing import Any -from config import settings +from opendevops_core.config import settings _VOLATILE_KEYS = { - "time", "timestamp", "eventTime", "requestId", "eventID", - "approximateInvokeCount", "startTime", "endTime", "updatedAt", - "stateTransitionedAt", "lastUpdatedTime", + "time", + "timestamp", + "eventTime", + "requestId", + "eventID", + "approximateInvokeCount", + "startTime", + "endTime", + "updatedAt", + "stateTransitionedAt", + "lastUpdatedTime", } @@ -79,9 +87,7 @@ def event_incident_key(event: dict[str, Any]) -> str: payload = payload if isinstance(payload, dict) else {} error_signature = { "errorType": ( - payload.get("errorType") - or detail.get("errorCode") - or detail.get("condition") + payload.get("errorType") or detail.get("errorCode") or detail.get("condition") ), "errorMessage": payload.get("errorMessage") or detail.get("errorMessage"), "detailType": detail_type, @@ -92,10 +98,7 @@ def event_incident_key(event: dict[str, Any]) -> str: task_arn = detail.get("taskArn") or detail.get("taskArnStr") or "unknown-task" cluster = detail.get("clusterArn") or "unknown-cluster" reason = detail.get("stoppedReason") or detail.get("stopCode") or "stopped" - return ( - f"ecs_task:{_region(region)}:{_part(cluster)}:" - f"{_part(task_arn)}:{_digest(reason, 8)}" - ) + return f"ecs_task:{_region(region)}:{_part(cluster)}:{_part(task_arn)}:{_digest(reason, 8)}" if source == "aws.rds": resource = detail.get("SourceIdentifier") or detail.get("SourceArn") or "unknown-db" @@ -126,4 +129,4 @@ def event_incident_key(event: dict[str, Any]) -> str: "detail-type": detail_type, "detail": _strip_volatile(detail), } - return f"event:{_part(source)}:{_digest(fingerprint)}" \ No newline at end of file + return f"event:{_part(source)}:{_digest(fingerprint)}" diff --git a/src/agent/init_store.py b/apps/core/src/opendevops_core/agent/init_store.py similarity index 97% rename from src/agent/init_store.py rename to apps/core/src/opendevops_core/agent/init_store.py index 7877b6e..01feb91 100644 --- a/src/agent/init_store.py +++ b/apps/core/src/opendevops_core/agent/init_store.py @@ -18,7 +18,7 @@ from loguru import logger -from config import settings +from opendevops_core.config import settings _INIT_KEY = "init" _CACHE: dict[str, Any] | None = None @@ -26,6 +26,7 @@ def _clone(data: dict) -> dict: import json + return json.loads(json.dumps(data)) @@ -83,6 +84,7 @@ def _with_defaults(data: dict) -> dict: # ── Sync API (safe to call from boto3 thread executors) ─────────────────────── + def load_init() -> dict: """Return a copy of the in-memory cache. Always fast — no I/O.""" global _CACHE @@ -99,11 +101,12 @@ def save_init(data: dict) -> None: # ── Async API (used by FastAPI route handlers) ──────────────────────────────── + async def load_init_async() -> dict: """Load from DB into cache if not yet loaded, then return a copy.""" global _CACHE try: - from agent.db import db + from opendevops_core.agent.db import db data = await db.get_app_config(_INIT_KEY) if data is not None: @@ -125,7 +128,7 @@ async def save_init_async(data: dict) -> dict: global _CACHE _CACHE = _with_defaults(data) try: - from agent.db import db + from opendevops_core.agent.db import db await db.set_app_config(_INIT_KEY, _CACHE) except Exception as e: @@ -145,6 +148,7 @@ async def reset_init_async() -> dict: # ── Accessors (read from cache — no I/O) ───────────────────────────────────── + def is_initialized() -> bool: return load_init().get("setup_complete", False) @@ -160,5 +164,3 @@ def get_runtime_aws_region() -> str: def get_runtime_sqs_queue_url() -> str: return str(settings.sqs_queue_url or load_init().get("sqs_queue_url") or "") - - diff --git a/src/agent/investigation_runner.py b/apps/core/src/opendevops_core/agent/investigation_runner.py similarity index 93% rename from src/agent/investigation_runner.py rename to apps/core/src/opendevops_core/agent/investigation_runner.py index 32c8e7e..4533a22 100644 --- a/src/agent/investigation_runner.py +++ b/apps/core/src/opendevops_core/agent/investigation_runner.py @@ -8,7 +8,7 @@ from loguru import logger -from config import settings +from opendevops_core.config import settings async def run_investigation( @@ -21,8 +21,8 @@ async def run_investigation( On agent exception, result is returned with _status="failed" so callers can still persist a record and send notifications. """ - from agent.core import get_agent - from agent.turns import save_turn + from opendevops_core.agent.core import get_agent + from opendevops_core.agent.turns import save_turn def _f(obj: Any, key: str, default: Any = None) -> Any: return obj.get(key, default) if isinstance(obj, dict) else getattr(obj, key, default) @@ -98,11 +98,13 @@ def _f(obj: Any, key: str, default: Any = None) -> Any: except (json.JSONDecodeError, TypeError): result = {"raw": str(content)[:500]} - tool_calls_log.append({ - "tool": call_info["tool"], - "args": call_info["args"], - "result": result, - }) + tool_calls_log.append( + { + "tool": call_info["tool"], + "args": call_info["args"], + "result": result, + } + ) if call_info["tool"] == "submit_investigation": investigation_result = call_info["args"] diff --git a/src/agent/llm/__init__.py b/apps/core/src/opendevops_core/agent/llm/__init__.py similarity index 76% rename from src/agent/llm/__init__.py rename to apps/core/src/opendevops_core/agent/llm/__init__.py index 9702c92..8419776 100644 --- a/src/agent/llm/__init__.py +++ b/apps/core/src/opendevops_core/agent/llm/__init__.py @@ -6,8 +6,12 @@ - shape_system_content(): provider-specific system-prompt shaping """ -from agent.llm.backend import LlmBackendInfo, get_backend_info, resolve_model_and_key -from agent.llm.identity import ( +from opendevops_core.agent.llm.backend import ( + LlmBackendInfo, + get_backend_info, + resolve_model_and_key, +) +from opendevops_core.agent.llm.identity import ( CLAUDE_CODE_IDENTITY, is_subscription_token, shape_system_content, diff --git a/src/agent/llm/backend.py b/apps/core/src/opendevops_core/agent/llm/backend.py similarity index 75% rename from src/agent/llm/backend.py rename to apps/core/src/opendevops_core/agent/llm/backend.py index b705c06..bd13bee 100644 --- a/src/agent/llm/backend.py +++ b/apps/core/src/opendevops_core/agent/llm/backend.py @@ -15,19 +15,19 @@ import re from typing import TypedDict -from config import settings -from agent.llm.detectors import ALL_DETECTORS +from opendevops_core.agent.llm.detectors import ALL_DETECTORS +from opendevops_core.config import settings _DATE_SUFFIX = re.compile(r"-\d{8}$") class LlmBackendInfo(TypedDict): - source: str # claude_code | anthropic | openrouter | openai | groq | custom | default - model: str # active LiteLLM model string - display_name: str # human-readable source name shown in the UI - provider: str # anthropic | openai | openrouter | groq | ollama | custom + source: str # claude_code | anthropic | openrouter | openai | groq | custom | default + model: str # active LiteLLM model string + display_name: str # human-readable source name shown in the UI + provider: str # anthropic | openai | openrouter | groq | ollama | custom configured: bool # True when valid auth is present - detail: str # friendly model name or setup hint + detail: str # friendly model name or setup hint def _strip_date(model: str) -> str: @@ -85,8 +85,12 @@ def get_backend_info() -> LlmBackendInfo: # Custom endpoint always takes precedence if api_base: return LlmBackendInfo( - source="custom", model=model, display_name="Custom endpoint", - provider="custom", configured=True, detail=api_base, + source="custom", + model=model, + display_name="Custom endpoint", + provider="custom", + configured=True, + detail=api_base, ) # Anthropic direct (env key) @@ -96,15 +100,23 @@ def get_backend_info() -> LlmBackendInfo: source = _provider_of(model) label = source.title() + " (via .env)" return LlmBackendInfo( - source=source, model=model, display_name=label, - provider=_provider_of(model), configured=True, detail=_strip_date(model), + source=source, + model=model, + display_name=label, + provider=_provider_of(model), + configured=True, + detail=_strip_date(model), ) # OpenRouter key if settings.openrouter_api_key and "openrouter" in model: return LlmBackendInfo( - source="openrouter", model=model, display_name="OpenRouter", - provider="openrouter", configured=True, detail=_strip_date(model), + source="openrouter", + model=model, + display_name="OpenRouter", + provider="openrouter", + configured=True, + detail=_strip_date(model), ) # Any other explicit provider key (OPENAI_API_KEY, GROQ_API_KEY, …) @@ -116,8 +128,12 @@ def get_backend_info() -> LlmBackendInfo: ) if has_key: return LlmBackendInfo( - source=provider, model=model, display_name=provider.title() + " (via .env)", - provider=provider, configured=True, detail=_strip_date(model), + source=provider, + model=model, + display_name=provider.title() + " (via .env)", + provider=provider, + configured=True, + detail=_strip_date(model), ) # Installed CLI detector (subscription login) @@ -126,8 +142,11 @@ def get_backend_info() -> LlmBackendInfo: return detected return LlmBackendInfo( - source="default", model=model, display_name="Not configured", - provider=_provider_of(model), configured=False, + source="default", + model=model, + display_name="Not configured", + provider=_provider_of(model), + configured=False, detail="Set LLM_MODEL and an API key in .env, or install a supported CLI (e.g. Claude Code)", ) diff --git a/src/agent/llm/detectors/__init__.py b/apps/core/src/opendevops_core/agent/llm/detectors/__init__.py similarity index 69% rename from src/agent/llm/detectors/__init__.py rename to apps/core/src/opendevops_core/agent/llm/detectors/__init__.py index a9f217d..d9ab578 100644 --- a/src/agent/llm/detectors/__init__.py +++ b/apps/core/src/opendevops_core/agent/llm/detectors/__init__.py @@ -4,8 +4,8 @@ append an instance to ALL_DETECTORS. """ -from agent.llm.detectors.base import LlmDetector -from agent.llm.detectors.claude_code import ClaudeCodeDetector +from opendevops_core.agent.llm.detectors.base import LlmDetector +from opendevops_core.agent.llm.detectors.claude_code import ClaudeCodeDetector ALL_DETECTORS: list[LlmDetector] = [ ClaudeCodeDetector(), diff --git a/src/agent/llm/detectors/base.py b/apps/core/src/opendevops_core/agent/llm/detectors/base.py similarity index 87% rename from src/agent/llm/detectors/base.py rename to apps/core/src/opendevops_core/agent/llm/detectors/base.py index 835d2dd..872aece 100644 --- a/src/agent/llm/detectors/base.py +++ b/apps/core/src/opendevops_core/agent/llm/detectors/base.py @@ -12,8 +12,8 @@ @runtime_checkable class LlmDetector(Protocol): - name: str # stable id used as the backend `source`, e.g. "claude_code" - label: str # brand name shown in the UI, e.g. "Claude Code" + name: str # stable id used as the backend `source`, e.g. "claude_code" + label: str # brand name shown in the UI, e.g. "Claude Code" provider: str # litellm provider family, e.g. "anthropic" @property diff --git a/src/agent/llm/detectors/claude_code.py b/apps/core/src/opendevops_core/agent/llm/detectors/claude_code.py similarity index 80% rename from src/agent/llm/detectors/claude_code.py rename to apps/core/src/opendevops_core/agent/llm/detectors/claude_code.py index ba5d0f1..a690ed5 100644 --- a/src/agent/llm/detectors/claude_code.py +++ b/apps/core/src/opendevops_core/agent/llm/detectors/claude_code.py @@ -18,7 +18,7 @@ from loguru import logger -from config import settings +from opendevops_core.config import settings _SETTINGS_PATH = Path.home() / ".claude" / "settings.json" _CREDS_PATH = Path.home() / ".claude" / ".credentials.json" @@ -27,20 +27,20 @@ # Includes both short aliases ("sonnet") and full slot names ("claude-sonnet-4-5"). _MODEL_SLOTS: dict[str, str] = { # Short aliases used by Claude Code settings.json - "sonnet": "anthropic/claude-sonnet-4-6", - "opus": "anthropic/claude-opus-4-7", - "haiku": "anthropic/claude-haiku-4-5-20251001", + "sonnet": "anthropic/claude-sonnet-4-6", + "opus": "anthropic/claude-opus-4-7", + "haiku": "anthropic/claude-haiku-4-5-20251001", # Full slot names - "claude-opus-4-7": "anthropic/claude-opus-4-7", - "claude-sonnet-4-6": "anthropic/claude-sonnet-4-6", - "claude-haiku-4-5": "anthropic/claude-haiku-4-5-20251001", - "claude-opus-4-5": "anthropic/claude-opus-4-5-20251001", - "claude-opus-4": "anthropic/claude-opus-4-20251101", - "claude-sonnet-4-5": "anthropic/claude-sonnet-4-5-20251024", - "claude-sonnet-4": "anthropic/claude-sonnet-4-20251120", + "claude-opus-4-7": "anthropic/claude-opus-4-7", + "claude-sonnet-4-6": "anthropic/claude-sonnet-4-6", + "claude-haiku-4-5": "anthropic/claude-haiku-4-5-20251001", + "claude-opus-4-5": "anthropic/claude-opus-4-5-20251001", + "claude-opus-4": "anthropic/claude-opus-4-20251101", + "claude-sonnet-4-5": "anthropic/claude-sonnet-4-5-20251024", + "claude-sonnet-4": "anthropic/claude-sonnet-4-20251120", "claude-3-5-sonnet-20241022": "anthropic/claude-3-5-sonnet-20241022", - "claude-3-5-haiku-20241022": "anthropic/claude-3-5-haiku-20241022", - "claude-3-opus-20240229": "anthropic/claude-3-opus-20240229", + "claude-3-5-haiku-20241022": "anthropic/claude-3-5-haiku-20241022", + "claude-3-opus-20240229": "anthropic/claude-3-opus-20240229", } _DEFAULT_MODEL = "anthropic/claude-sonnet-4-6" @@ -62,7 +62,9 @@ def resolve(self) -> tuple[str, str] | None: return None api_key = self._api_key() if not api_key: - logger.info("claude_code: installed but no auth found — run `claude` to log in or set ANTHROPIC_API_KEY") + logger.info( + "claude_code: installed but no auth found — run `claude` to log in or set ANTHROPIC_API_KEY" + ) return None model = self._model() logger.info("claude_code: auto-configuring model={} via {}", model, self._auth_method()) @@ -75,7 +77,7 @@ def status(self) -> dict: return { "installed": installed, "authenticated": bool(method), - "auth_method": method, # api_key | auth_token | subscription | None + "auth_method": method, # api_key | auth_token | subscription | None "subscription_type": oauth.get("subscriptionType"), "token_expired": self._oauth_expired(oauth) if oauth else False, "model": self._model() if installed else None, diff --git a/src/agent/llm/identity.py b/apps/core/src/opendevops_core/agent/llm/identity.py similarity index 100% rename from src/agent/llm/identity.py rename to apps/core/src/opendevops_core/agent/llm/identity.py diff --git a/src/agent/monitor_store.py b/apps/core/src/opendevops_core/agent/monitor_store.py similarity index 78% rename from src/agent/monitor_store.py rename to apps/core/src/opendevops_core/agent/monitor_store.py index ef652f6..4b1690a 100644 --- a/src/agent/monitor_store.py +++ b/apps/core/src/opendevops_core/agent/monitor_store.py @@ -45,28 +45,38 @@ async def add_alert( evidence: list | None = None, ) -> str: """Persist an alert to the DB backend and broadcast to SSE subscribers.""" - from agent.db import db + from opendevops_core.agent.db import db try: alert_id = await db.add_alert( - service, error, resolution, confidence, sns_sent, - dedup_key, status, session_id, trigger_source, evidence, + service, + error, + resolution, + confidence, + sns_sent, + dedup_key, + status, + session_id, + trigger_source, + evidence, ) if alert_id and _alert_subscribers: - _broadcast_alert({ - "id": alert_id, - "service": service, - "error": error, - "resolution": resolution, - "confidence": confidence, - "sns_sent": sns_sent, - "timestamp": datetime.now(UTC).isoformat(), - "dedup_key": dedup_key, - "status": status, - "trigger_source": trigger_source, - "session_id": session_id, - "evidence": evidence or [], - }) + _broadcast_alert( + { + "id": alert_id, + "service": service, + "error": error, + "resolution": resolution, + "confidence": confidence, + "sns_sent": sns_sent, + "timestamp": datetime.now(UTC).isoformat(), + "dedup_key": dedup_key, + "status": status, + "trigger_source": trigger_source, + "session_id": session_id, + "evidence": evidence or [], + } + ) return alert_id except Exception as e: logger.error("Failed to persist alert: {}", e) @@ -80,7 +90,7 @@ async def add_notification( error: str | None = None, ) -> None: """Record a delivery attempt for a channel against an alert.""" - from agent.db import db + from opendevops_core.agent.db import db try: await db.add_notification(alert_id, channel, status, error) @@ -90,7 +100,7 @@ async def add_notification( async def is_recent_alert(dedup_key: str, within_minutes: int = 3) -> bool: """Return True if an alert with this dedup_key was saved within the last N minutes.""" - from agent.db import db + from opendevops_core.agent.db import db try: return await db.is_recent_alert(dedup_key, within_minutes) @@ -104,7 +114,7 @@ async def claim_incident( within_minutes: int = 3, ) -> bool: """Atomically claim an incident before running an investigation.""" - from agent.db import db + from opendevops_core.agent.db import db try: return await db.claim_incident(incident_key, trigger_source, within_minutes) @@ -118,7 +128,7 @@ async def complete_incident( status: str = "completed", session_id: str | None = None, ) -> None: - from agent.db import db + from opendevops_core.agent.db import db try: await db.complete_incident(incident_key, status, session_id) @@ -127,7 +137,7 @@ async def complete_incident( async def release_incident(incident_key: str) -> None: - from agent.db import db + from opendevops_core.agent.db import db try: await db.release_incident(incident_key) @@ -136,7 +146,7 @@ async def release_incident(incident_key: str) -> None: async def is_incident_claimed(incident_key: str, within_minutes: int = 3) -> bool: - from agent.db import db + from opendevops_core.agent.db import db try: return await db.is_incident_claimed(incident_key, within_minutes) @@ -154,13 +164,13 @@ def update_service(name: str, status: str, error: str | None = None) -> None: async def get_alerts_async(limit: int = 50) -> list[dict]: - from agent.db import db + from opendevops_core.agent.db import db return await db.get_alerts(limit) async def get_alert_async(alert_id: str) -> dict | None: - from agent.db import db + from opendevops_core.agent.db import db return await db.get_alert(alert_id) diff --git a/src/agent/prompts.py b/apps/core/src/opendevops_core/agent/prompts.py similarity index 98% rename from src/agent/prompts.py rename to apps/core/src/opendevops_core/agent/prompts.py index 1049de6..64284f6 100644 --- a/src/agent/prompts.py +++ b/apps/core/src/opendevops_core/agent/prompts.py @@ -68,7 +68,8 @@ def build_system_prompt() -> str: """Build the system prompt, injecting the list of available runbooks.""" try: - from tools.skills import available_skill_summaries + from opendevops_core.tools.skills import available_skill_summaries + summaries = available_skill_summaries() except Exception: summaries = [] diff --git a/src/agent/summarizer.py b/apps/core/src/opendevops_core/agent/summarizer.py similarity index 88% rename from src/agent/summarizer.py rename to apps/core/src/opendevops_core/agent/summarizer.py index 2dcef1a..d954b20 100644 --- a/src/agent/summarizer.py +++ b/apps/core/src/opendevops_core/agent/summarizer.py @@ -18,7 +18,7 @@ from langchain_litellm import ChatLiteLLM from loguru import logger -from config import settings +from opendevops_core.config import settings _SUMMARIZE_SYSTEM = """You are compressing an AWS incident investigation conversation to save context space. @@ -58,10 +58,7 @@ def _split_messages(messages: list, keep_chars: int) -> tuple[list, list]: if not messages: return [], [] - human_indices = [ - i for i, m in enumerate(messages) - if getattr(m, "type", None) == "human" - ] + human_indices = [i for i, m in enumerate(messages) if getattr(m, "type", None) == "human"] # Need at least 2 human turns — otherwise nothing useful to summarize if len(human_indices) < 2: @@ -124,24 +121,31 @@ async def maybe_summarize(agent: Any, config: dict, session_id: str) -> bool: chars_removed = _total_chars(to_summarize) logger.info( "[{}] summarizer: session={} chars, removing {} msgs ({} chars), keeping {} msgs", - sid, total, len(to_summarize), chars_removed, len(to_keep), + sid, + total, + len(to_summarize), + chars_removed, + len(to_keep), ) # ── Summarize old messages via LLM ──────────────────────────────────────── history_text = _format_for_summary(to_summarize) start = time.time() try: - from agent.llm import resolve_model_and_key, shape_system_content + from opendevops_core.agent.llm import resolve_model_and_key, shape_system_content + model_name, api_key = resolve_model_and_key() llm = ChatLiteLLM( model=model_name, api_base=settings.llm_api_base or None, api_key=api_key, ) - response = await llm.ainvoke([ - {"role": "system", "content": shape_system_content(_SUMMARIZE_SYSTEM, api_key)}, - {"role": "user", "content": f"Summarize this investigation:\n\n{history_text}"}, - ]) + response = await llm.ainvoke( + [ + {"role": "system", "content": shape_system_content(_SUMMARIZE_SYSTEM, api_key)}, + {"role": "user", "content": f"Summarize this investigation:\n\n{history_text}"}, + ] + ) summary_text = str(response.content) except Exception as exc: logger.error("[{}] summarizer: LLM call failed — {}", sid, exc) @@ -166,20 +170,24 @@ async def maybe_summarize(agent: Any, config: dict, session_id: str) -> bool: logger.info( "[{}] summarizer done: -{} chars, summary {} chars, {}ms", - sid, chars_removed, len(summary_text), elapsed_ms, + sid, + chars_removed, + len(summary_text), + elapsed_ms, ) # ── Record in usage_events ───────────────────────────────────────────────── - from agent.db import db - from agent.turns import calc_cost + from opendevops_core.agent.db import db + from opendevops_core.agent.turns import calc_cost - input_tokens = chars_removed // 4 + input_tokens = chars_removed // 4 output_tokens = len(summary_text) // 4 cost = calc_cost(settings.llm_model, input_tokens, output_tokens) try: await db.save_usage_event( - session_id, None, + session_id, + None, model=settings.llm_model, input_tokens=input_tokens, output_tokens=output_tokens, diff --git a/src/agent/turns.py b/apps/core/src/opendevops_core/agent/turns.py similarity index 86% rename from src/agent/turns.py rename to apps/core/src/opendevops_core/agent/turns.py index 641c315..e9e0fb9 100644 --- a/src/agent/turns.py +++ b/apps/core/src/opendevops_core/agent/turns.py @@ -6,9 +6,9 @@ from loguru import logger -from agent.db import db -from agent.init_store import get_runtime_aws_region -from config import settings +from opendevops_core.agent.db import db +from opendevops_core.agent.init_store import get_runtime_aws_region +from opendevops_core.config import settings # Fallback pricing ($/M tokens) for models absent from LiteLLM's database. _FALLBACK_PRICING: dict[str, dict[str, float]] = { @@ -43,12 +43,20 @@ async def save_turn( tool_calls_log: list[dict[str, Any]], usage: dict[str, Any], source: str = "chat", + org_id: str | None = None, + user_id: str | None = None, ) -> None: """Persist a completed turn to Postgres. Errors are logged, never raised.""" try: title = user_message[:80] if user_message else None await db.upsert_session( - session_id, usage.get("model", ""), get_runtime_aws_region(), title, source + session_id, + usage.get("model", ""), + get_runtime_aws_region(), + title, + source, + org_id=org_id, + user_id=user_id, ) await db.save_message(session_id, "user", user_message) @@ -96,7 +104,7 @@ async def notify_slack(session_id: str, tool_calls_log: list[dict[str, Any]]) -> return False for tc in tool_calls_log: if tc.get("tool") == "submit_investigation": - from integrations.slack_webhook import post_investigation + from opendevops_core.integrations.slack_webhook import post_investigation return await post_investigation(settings.slack_webhook_url, tc["args"], session_id) return False diff --git a/src/config/appsettings.py b/apps/core/src/opendevops_core/config.py similarity index 60% rename from src/config/appsettings.py rename to apps/core/src/opendevops_core/config.py index db0f4a7..7404e1b 100644 --- a/src/config/appsettings.py +++ b/apps/core/src/opendevops_core/config.py @@ -1,9 +1,20 @@ -from typing import Literal +"""Core configuration. + +`CoreSettings` holds every setting the shared agent core reads. Host applications +(the OSS web app, the SaaS product) subclass it to add their own fields, instantiate +it, and register the instance via `configure()`. Core modules read settings through the +module-level `settings` proxy, which always delegates to the configured instance — so a +host app's richer settings object is what core actually sees at runtime. +""" + +from __future__ import annotations + +from typing import Any, Literal from pydantic_settings import BaseSettings, SettingsConfigDict -class Settings(BaseSettings): +class CoreSettings(BaseSettings): model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") # Primary LLM settings — use LiteLLM model string format: @@ -11,8 +22,8 @@ class Settings(BaseSettings): # groq/llama3-70b-8192 | ollama/llama3 | openai/gpt-4o # For custom OpenAI-compatible endpoints set llm_api_base + llm_api_key. llm_model: str = "openrouter/openai/gpt-4o" - llm_api_base: str | None = None # custom base URL (e.g. http://localhost:11434) - llm_api_key: str | None = None # custom API key (falls back to provider env vars) + llm_api_base: str | None = None # custom base URL (e.g. http://localhost:11434) + llm_api_key: str | None = None # custom API key (falls back to provider env vars) # OpenRouter (kept for backward-compat and pricing lookup) openrouter_api_key: str = "" @@ -52,8 +63,8 @@ class Settings(BaseSettings): # Fires before each agent call when total message chars exceed the threshold. # Set summarization_enabled=false or summarization_threshold_chars=0 to disable. summarization_enabled: bool = True - summarization_threshold_chars: int = 60_000 # ~15 K tokens; trigger compaction above this - summarization_keep_chars: int = 20_000 # ~5 K tokens of recent messages to keep intact + summarization_threshold_chars: int = 60_000 # ~15 K tokens; trigger compaction above this + summarization_keep_chars: int = 20_000 # ~5 K tokens of recent messages to keep intact # Slack — leave unset to disable notifications slack_webhook_url: str | None = None @@ -63,13 +74,9 @@ class Settings(BaseSettings): telegram_chat_id: str | None = None # Proactive polling — set poll_interval_seconds=0 to disable - poll_interval_seconds: int = 0 # disabled by default - poll_error_threshold: float = 5.0 # % Lambda error rate that triggers investigation - poll_reinvestigate_hours: int = 1 # don't re-investigate the same alarm within N hours - - # Auth — leave jwt_secret unset to disable auth (dev / memory-backend mode) - jwt_secret: str | None = None - jwt_expire_minutes: int = 1440 # 24 hours + poll_interval_seconds: int = 0 # disabled by default + poll_error_threshold: float = 5.0 # % Lambda error rate that triggers investigation + poll_reinvestigate_hours: int = 1 # don't re-investigate the same alarm within N hours # Event consumer — leave unset to disable event_consumer_enabled: bool = False @@ -84,4 +91,32 @@ class Settings(BaseSettings): claude_code_autodetect: bool = True -settings = Settings() +_settings: CoreSettings | None = None + + +def configure(settings_instance: CoreSettings) -> None: + """Register the active settings instance. Host apps call this once at startup.""" + global _settings + _settings = settings_instance + + +def get_settings() -> CoreSettings: + """Return the active settings, lazily defaulting to a bare CoreSettings().""" + global _settings + if _settings is None: + _settings = CoreSettings() + return _settings + + +class _SettingsProxy: + """Delegates attribute access to the configured settings instance at runtime. + + Lets core modules keep `from opendevops_core.config import settings` and read + `settings.x` unchanged while the host app injects its own settings via configure(). + """ + + def __getattr__(self, name: str) -> Any: + return getattr(get_settings(), name) + + +settings = _SettingsProxy() diff --git a/tests/test_llm/__init__.py b/apps/core/src/opendevops_core/integrations/__init__.py similarity index 100% rename from tests/test_llm/__init__.py rename to apps/core/src/opendevops_core/integrations/__init__.py diff --git a/src/integrations/slack_webhook.py b/apps/core/src/opendevops_core/integrations/slack_webhook.py similarity index 71% rename from src/integrations/slack_webhook.py rename to apps/core/src/opendevops_core/integrations/slack_webhook.py index 819a335..1dfb7d0 100644 --- a/src/integrations/slack_webhook.py +++ b/apps/core/src/opendevops_core/integrations/slack_webhook.py @@ -8,16 +8,18 @@ # Sidebar colour per root-cause category _CATEGORY_COLORS: dict[str, str] = { - "SYSTEM_CHANGE": "#e67e22", - "INPUT_ANOMALY": "#3498db", - "RESOURCE_LIMIT": "#e74c3c", + "SYSTEM_CHANGE": "#e67e22", + "INPUT_ANOMALY": "#3498db", + "RESOURCE_LIMIT": "#e74c3c", "COMPONENT_FAILURE": "#c0392b", - "DEPENDENCY_ISSUE": "#9b59b6", - "UNKNOWN": "#95a5a6", + "DEPENDENCY_ISSUE": "#9b59b6", + "UNKNOWN": "#95a5a6", } _CONFIDENCE_EMOJI: dict[str, str] = { - "HIGH": "🟢", "MEDIUM": "🟡", "LOW": "🔴", + "HIGH": "🟢", + "MEDIUM": "🟡", + "LOW": "🔴", } @@ -27,13 +29,13 @@ def _build_payload( app_url: str | None = None, is_test: bool = False, ) -> dict: - category = result.get("root_cause_category", "UNKNOWN") - summary = result.get("root_cause_summary", "No summary provided.") + category = result.get("root_cause_category", "UNKNOWN") + summary = result.get("root_cause_summary", "No summary provided.") confidence = result.get("confidence", "LOW") - evidence = result.get("evidence", []) + evidence = result.get("evidence", []) mitigation = result.get("mitigation_steps", []) - services = result.get("services_affected", []) - color = _CATEGORY_COLORS.get(category, "#95a5a6") + services = result.get("services_affected", []) + color = _CATEGORY_COLORS.get(category, "#95a5a6") conf_emoji = _CONFIDENCE_EMOJI.get(confidence, "🔴") header_text = ( "🧪 [TEST] OpenDevOps Agent — Investigation Complete" @@ -64,43 +66,59 @@ def _bullets(items: list[str]) -> str: ] if evidence: - blocks.append({ - "type": "section", - "text": {"type": "mrkdwn", "text": f"*Evidence*\n{_bullets(evidence)}"}, - }) + blocks.append( + { + "type": "section", + "text": {"type": "mrkdwn", "text": f"*Evidence*\n{_bullets(evidence)}"}, + } + ) if mitigation: - steps = "\n".join(f"{i+1}. {s}" for i, s in enumerate(mitigation)) - blocks.append({ - "type": "section", - "text": {"type": "mrkdwn", "text": f"*Mitigation steps*\n{steps}"}, - }) + steps = "\n".join(f"{i + 1}. {s}" for i, s in enumerate(mitigation)) + blocks.append( + { + "type": "section", + "text": {"type": "mrkdwn", "text": f"*Mitigation steps*\n{steps}"}, + } + ) if services: - blocks.append({ - "type": "context", - "elements": [{"type": "mrkdwn", "text": f"*Services affected:* {', '.join(f'`{s}`' for s in services)}"}], - }) + blocks.append( + { + "type": "context", + "elements": [ + { + "type": "mrkdwn", + "text": f"*Services affected:* {', '.join(f'`{s}`' for s in services)}", + } + ], + } + ) footer_parts = [f"Session `{session_id[:8]}`"] if app_url: footer_parts.append(f"<{app_url}/chat/{session_id}|Open in app>") - blocks.append({ - "type": "context", - "elements": [{"type": "mrkdwn", "text": " · ".join(footer_parts)}], - }) + blocks.append( + { + "type": "context", + "elements": [{"type": "mrkdwn", "text": " · ".join(footer_parts)}], + } + ) return { - "attachments": [{ - "color": color, - "blocks": blocks, - }] + "attachments": [ + { + "color": color, + "blocks": blocks, + } + ] } async def _post(webhook_url: str, payload: dict) -> bool: try: import httpx + async with httpx.AsyncClient(timeout=10) as client: resp = await client.post(webhook_url, json=payload) if resp.status_code != 200: diff --git a/src/integrations/telegram.py b/apps/core/src/opendevops_core/integrations/telegram.py similarity index 70% rename from src/integrations/telegram.py rename to apps/core/src/opendevops_core/integrations/telegram.py index 8e171d1..3f1b098 100644 --- a/src/integrations/telegram.py +++ b/apps/core/src/opendevops_core/integrations/telegram.py @@ -7,7 +7,9 @@ from loguru import logger _CONFIDENCE_EMOJI: dict[str, str] = { - "HIGH": "🔴", "MEDIUM": "🟡", "LOW": "🟢", + "HIGH": "🔴", + "MEDIUM": "🟡", + "LOW": "🟢", } @@ -16,30 +18,36 @@ def _build_message( session_id: str, is_test: bool = False, ) -> str: - category = result.get("root_cause_category", "UNKNOWN") - summary = result.get("root_cause_summary", "No summary provided.") + category = result.get("root_cause_category", "UNKNOWN") + summary = result.get("root_cause_summary", "No summary provided.") confidence = result.get("confidence", "LOW") - evidence = result.get("evidence", []) + evidence = result.get("evidence", []) mitigation = result.get("mitigation_steps", []) - services = result.get("services_affected", []) + services = result.get("services_affected", []) conf_emoji = _CONFIDENCE_EMOJI.get(confidence, "🟢") - header = "🧪 [TEST] OpenDevOps — Investigation Complete" if is_test else \ - "🔍 OpenDevOps — Investigation Complete" + header = ( + "🧪 [TEST] OpenDevOps — Investigation Complete" + if is_test + else "🔍 OpenDevOps — Investigation Complete" + ) lines = [header, ""] lines += [f"{conf_emoji} {confidence} confidence · {category}", ""] lines += [f"Root Cause\n{summary}", ""] if services: - lines += [f"Services affected: {', '.join(f'{s}' for s in services)}", ""] + lines += [ + f"Services affected: {', '.join(f'{s}' for s in services)}", + "", + ] if evidence: ev_text = "\n".join(f"• {e}" for e in evidence[:5]) lines += [f"Evidence\n{ev_text}", ""] if mitigation: - mt_text = "\n".join(f"{i+1}. {s}" for i, s in enumerate(mitigation)) + mt_text = "\n".join(f"{i + 1}. {s}" for i, s in enumerate(mitigation)) lines += [f"Mitigation Steps\n{mt_text}", ""] lines.append(f"Session {session_id[:8]}") @@ -50,14 +58,18 @@ async def _post(bot_token: str, chat_id: str, text: str) -> tuple[bool, str]: """Return (success, error_detail). error_detail is empty on success.""" try: import httpx + url = f"https://api.telegram.org/bot{bot_token}/sendMessage" async with httpx.AsyncClient(timeout=10) as client: - resp = await client.post(url, json={ - "chat_id": chat_id, - "text": text, - "parse_mode": "HTML", - "disable_web_page_preview": True, - }) + resp = await client.post( + url, + json={ + "chat_id": chat_id, + "text": text, + "parse_mode": "HTML", + "disable_web_page_preview": True, + }, + ) if resp.status_code != 200: detail = resp.text logger.warning("Telegram API returned {}: {}", resp.status_code, detail) @@ -92,8 +104,11 @@ async def post_failed_investigation( is_test: bool = False, ) -> bool: """Post a failed investigation alert to Telegram. Returns True on success.""" - header = "🧪 [TEST] OpenDevOps — Investigation Failed" if is_test else \ - "⚠️ OpenDevOps — Investigation Failed" + header = ( + "🧪 [TEST] OpenDevOps — Investigation Failed" + if is_test + else "⚠️ OpenDevOps — Investigation Failed" + ) text = ( f"{header}\n\n" f"Service: {service}\n" diff --git a/migrations/001_initial.sql b/apps/core/src/opendevops_core/migrations/001_initial.sql similarity index 100% rename from migrations/001_initial.sql rename to apps/core/src/opendevops_core/migrations/001_initial.sql diff --git a/migrations/002_soft_delete.sql b/apps/core/src/opendevops_core/migrations/002_soft_delete.sql similarity index 100% rename from migrations/002_soft_delete.sql rename to apps/core/src/opendevops_core/migrations/002_soft_delete.sql diff --git a/migrations/003_usage_events_metadata.sql b/apps/core/src/opendevops_core/migrations/003_usage_events_metadata.sql similarity index 100% rename from migrations/003_usage_events_metadata.sql rename to apps/core/src/opendevops_core/migrations/003_usage_events_metadata.sql diff --git a/migrations/004_users_rbac.sql b/apps/core/src/opendevops_core/migrations/004_users_rbac.sql similarity index 100% rename from migrations/004_users_rbac.sql rename to apps/core/src/opendevops_core/migrations/004_users_rbac.sql diff --git a/migrations/005_alerts.sql b/apps/core/src/opendevops_core/migrations/005_alerts.sql similarity index 100% rename from migrations/005_alerts.sql rename to apps/core/src/opendevops_core/migrations/005_alerts.sql diff --git a/migrations/006_app_config.sql b/apps/core/src/opendevops_core/migrations/006_app_config.sql similarity index 100% rename from migrations/006_app_config.sql rename to apps/core/src/opendevops_core/migrations/006_app_config.sql diff --git a/migrations/007_alerts_dedup_key.sql b/apps/core/src/opendevops_core/migrations/007_alerts_dedup_key.sql similarity index 100% rename from migrations/007_alerts_dedup_key.sql rename to apps/core/src/opendevops_core/migrations/007_alerts_dedup_key.sql diff --git a/migrations/008_alerts_status.sql b/apps/core/src/opendevops_core/migrations/008_alerts_status.sql similarity index 100% rename from migrations/008_alerts_status.sql rename to apps/core/src/opendevops_core/migrations/008_alerts_status.sql diff --git a/migrations/009_session_source_alert_session.sql b/apps/core/src/opendevops_core/migrations/009_session_source_alert_session.sql similarity index 100% rename from migrations/009_session_source_alert_session.sql rename to apps/core/src/opendevops_core/migrations/009_session_source_alert_session.sql diff --git a/migrations/010_session_user_interacted.sql b/apps/core/src/opendevops_core/migrations/010_session_user_interacted.sql similarity index 100% rename from migrations/010_session_user_interacted.sql rename to apps/core/src/opendevops_core/migrations/010_session_user_interacted.sql diff --git a/migrations/011_trigger_source_notifications.sql b/apps/core/src/opendevops_core/migrations/011_trigger_source_notifications.sql similarity index 100% rename from migrations/011_trigger_source_notifications.sql rename to apps/core/src/opendevops_core/migrations/011_trigger_source_notifications.sql diff --git a/migrations/012_incident_claims.sql b/apps/core/src/opendevops_core/migrations/012_incident_claims.sql similarity index 100% rename from migrations/012_incident_claims.sql rename to apps/core/src/opendevops_core/migrations/012_incident_claims.sql diff --git a/migrations/013_alerts_evidence.sql b/apps/core/src/opendevops_core/migrations/013_alerts_evidence.sql similarity index 100% rename from migrations/013_alerts_evidence.sql rename to apps/core/src/opendevops_core/migrations/013_alerts_evidence.sql diff --git a/apps/core/src/opendevops_core/migrations/__init__.py b/apps/core/src/opendevops_core/migrations/__init__.py new file mode 100644 index 0000000..b43597c --- /dev/null +++ b/apps/core/src/opendevops_core/migrations/__init__.py @@ -0,0 +1,67 @@ +"""Core baseline SQL migrations + a source-aware runner. + +The `.sql` files here define the schema that core's DB code (sessions, messages, +tool_calls, usage_events, users, organizations, alerts, incident_claims, …) reads +and writes — the baseline contract every app embedding the core must provide. + +Host apps layer their own migrations on top by passing `app_migrations_dir`. Each +source ("core", "app") has an independent version sequence tracked in the +`schema_migrations(source, version)` ledger, so a core `014` and an app `014` never +collide and nothing is applied twice. Migrations are idempotent, so re-running against +a database created before the ledger existed is safe. +""" + +from __future__ import annotations + +from pathlib import Path + +_LEDGER_DDL = """ +CREATE TABLE IF NOT EXISTS schema_migrations ( + source TEXT NOT NULL, + version TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (source, version) +); +""" + + +def core_migrations_path() -> Path: + """Directory holding the bundled core `.sql` migrations.""" + return Path(__file__).parent + + +def run_migrations(database_url: str, app_migrations_dir: Path | str | None = None) -> list[str]: + """Apply pending migrations: core first, then the host app's, recording each in + the `schema_migrations` ledger. Returns the list of newly applied "source/version" + identifiers (empty when the database is already up to date). + """ + import psycopg + + sources: list[tuple[str, Path]] = [("core", core_migrations_path())] + if app_migrations_dir is not None: + app_dir = Path(app_migrations_dir) + if app_dir.exists(): + sources.append(("app", app_dir)) + + applied: list[str] = [] + conn = psycopg.connect(database_url, autocommit=True) + try: + conn.execute(_LEDGER_DDL) + for source, directory in sources: + for path in sorted(directory.glob("*.sql")): + version = path.stem + already = conn.execute( + "SELECT 1 FROM schema_migrations WHERE source = %s AND version = %s", + (source, version), + ).fetchone() + if already: + continue + conn.execute(path.read_text(encoding="utf-8")) + conn.execute( + "INSERT INTO schema_migrations (source, version) VALUES (%s, %s)", + (source, version), + ) + applied.append(f"{source}/{version}") + finally: + conn.close() + return applied diff --git a/src/models/__init__.py b/apps/core/src/opendevops_core/models/__init__.py similarity index 61% rename from src/models/__init__.py rename to apps/core/src/opendevops_core/models/__init__.py index 2089529..2d3383b 100644 --- a/src/models/__init__.py +++ b/apps/core/src/opendevops_core/models/__init__.py @@ -1,15 +1,20 @@ """Shared Pydantic models — agent domain, memory state, and API request/response schemas.""" -from models.agent import ( +from opendevops_core.models.agent import ( Confidence, Finding, Investigation, InvestigationResult, RootCauseCategory, ) -from models.memory import InvestigationState, ToolCall -from models.chat import ChatRequest -from models.sessions import MessageRecord, SessionSummary, ToolCallRecord, UsageRecord +from opendevops_core.models.chat import ChatRequest +from opendevops_core.models.memory import InvestigationState, ToolCall +from opendevops_core.models.sessions import ( + MessageRecord, + SessionSummary, + ToolCallRecord, + UsageRecord, +) __all__ = [ "Confidence", diff --git a/src/models/agent.py b/apps/core/src/opendevops_core/models/agent.py similarity index 100% rename from src/models/agent.py rename to apps/core/src/opendevops_core/models/agent.py diff --git a/src/models/chat.py b/apps/core/src/opendevops_core/models/chat.py similarity index 100% rename from src/models/chat.py rename to apps/core/src/opendevops_core/models/chat.py diff --git a/src/models/memory.py b/apps/core/src/opendevops_core/models/memory.py similarity index 88% rename from src/models/memory.py rename to apps/core/src/opendevops_core/models/memory.py index cac9ef9..406d323 100644 --- a/src/models/memory.py +++ b/apps/core/src/opendevops_core/models/memory.py @@ -17,7 +17,9 @@ class InvestigationState(BaseModel): hypotheses: list[str] = Field(default_factory=list) messages: list[dict[str, Any]] = Field(default_factory=list) - def add_tool_call(self, tool_name: str, arguments: dict[str, Any], result: dict[str, Any]) -> None: + def add_tool_call( + self, tool_name: str, arguments: dict[str, Any], result: dict[str, Any] + ) -> None: self.tool_calls.append(ToolCall(tool_name=tool_name, arguments=arguments, result=result)) def tool_call_count(self) -> int: diff --git a/src/models/sessions.py b/apps/core/src/opendevops_core/models/sessions.py similarity index 100% rename from src/models/sessions.py rename to apps/core/src/opendevops_core/models/sessions.py diff --git a/src/models/users.py b/apps/core/src/opendevops_core/models/users.py similarity index 99% rename from src/models/users.py rename to apps/core/src/opendevops_core/models/users.py index bec502f..eafd0a0 100644 --- a/src/models/users.py +++ b/apps/core/src/opendevops_core/models/users.py @@ -1,4 +1,5 @@ from datetime import datetime + from pydantic import BaseModel, EmailStr, field_validator diff --git a/src/providers/__init__.py b/apps/core/src/opendevops_core/providers/__init__.py similarity index 65% rename from src/providers/__init__.py rename to apps/core/src/opendevops_core/providers/__init__.py index e89fc14..6e64732 100644 --- a/src/providers/__init__.py +++ b/apps/core/src/opendevops_core/providers/__init__.py @@ -2,21 +2,24 @@ from __future__ import annotations -from config import settings -from providers.base import CloudProvider +from opendevops_core.config import settings +from opendevops_core.providers.base import CloudProvider def get_active_provider() -> CloudProvider: """Return the provider selected by settings.cloud_provider.""" name = settings.cloud_provider if name == "aws": - from providers.aws import AwsProvider + from opendevops_core.providers.aws import AwsProvider + return AwsProvider() if name == "azure": - from providers.azure import AzureProvider + from opendevops_core.providers.azure import AzureProvider + return AzureProvider() if name == "gcp": - from providers.gcp import GcpProvider + from opendevops_core.providers.gcp import GcpProvider + return GcpProvider() raise ValueError(f"Unknown CLOUD_PROVIDER: {name!r} (expected aws | azure | gcp)") diff --git a/apps/core/src/opendevops_core/providers/aws/__init__.py b/apps/core/src/opendevops_core/providers/aws/__init__.py new file mode 100644 index 0000000..f0c704e --- /dev/null +++ b/apps/core/src/opendevops_core/providers/aws/__init__.py @@ -0,0 +1,51 @@ +"""AWS cloud provider — CloudWatch, CloudTrail, ECS, Lambda, EC2, RDS, IAM.""" + +from __future__ import annotations + +from typing import Any + + +class AwsProvider: + name = "aws" + + def tools(self) -> list[Any]: + from opendevops_core.providers.aws.tools.cloudtrail import ALL_CLOUDTRAIL_TOOLS + from opendevops_core.providers.aws.tools.cloudwatch import ALL_CLOUDWATCH_TOOLS + from opendevops_core.providers.aws.tools.ec2 import ALL_EC2_TOOLS + from opendevops_core.providers.aws.tools.ecs import ALL_ECS_TOOLS + from opendevops_core.providers.aws.tools.iam import ALL_IAM_TOOLS + from opendevops_core.providers.aws.tools.lambda_ import ALL_LAMBDA_TOOLS + from opendevops_core.providers.aws.tools.rds import ALL_RDS_TOOLS + + return ( + ALL_CLOUDWATCH_TOOLS + + ALL_CLOUDTRAIL_TOOLS + + ALL_ECS_TOOLS + + ALL_LAMBDA_TOOLS + + ALL_EC2_TOOLS + + ALL_RDS_TOOLS + + ALL_IAM_TOOLS + ) + + def collect_context(self, event: dict) -> dict: + from opendevops_core.providers.aws.context import collect_context + + return collect_context(event) + + def check_permissions(self, region: str | None) -> dict: + from opendevops_core.providers.aws.permissions import check_permissions + + return check_permissions(region) + + async def polling_loop(self) -> None: + from opendevops_core.providers.aws.poller import polling_loop + + await polling_loop() + + async def event_consumer_loop(self) -> None: + from opendevops_core.providers.aws.event_consumer import event_consumer_loop + + await event_consumer_loop() + + +__all__ = ["AwsProvider"] diff --git a/src/providers/aws/context.py b/apps/core/src/opendevops_core/providers/aws/context.py similarity index 98% rename from src/providers/aws/context.py rename to apps/core/src/opendevops_core/providers/aws/context.py index 01b7099..5ec97ef 100644 --- a/src/providers/aws/context.py +++ b/apps/core/src/opendevops_core/providers/aws/context.py @@ -7,8 +7,8 @@ import boto3 from loguru import logger -from agent.init_store import get_runtime_aws_region -from config import settings +from opendevops_core.agent.init_store import get_runtime_aws_region +from opendevops_core.config import settings def _session() -> boto3.Session: diff --git a/src/providers/aws/event_consumer.py b/apps/core/src/opendevops_core/providers/aws/event_consumer.py similarity index 88% rename from src/providers/aws/event_consumer.py rename to apps/core/src/opendevops_core/providers/aws/event_consumer.py index df4cb39..0986222 100644 --- a/src/providers/aws/event_consumer.py +++ b/apps/core/src/opendevops_core/providers/aws/event_consumer.py @@ -10,12 +10,12 @@ import boto3 from loguru import logger -from agent.incident_keys import event_incident_key -from agent.init_store import ( +from opendevops_core.agent.incident_keys import event_incident_key +from opendevops_core.agent.init_store import ( get_runtime_aws_region, get_runtime_sqs_queue_url, ) -from agent.monitor_store import ( +from opendevops_core.agent.monitor_store import ( add_alert, add_notification, claim_incident, @@ -23,7 +23,7 @@ release_incident, update_service, ) -from config import settings +from opendevops_core.config import settings ProcessResult = Literal["processed", "ignored", "duplicate"] @@ -110,7 +110,8 @@ def _build_investigation_prompt(event: dict) -> str: async def _run_investigation(prompt: str, session_id: str) -> dict[str, Any] | None: - from agent.investigation_runner import run_investigation + from opendevops_core.agent.investigation_runner import run_investigation + result, _tool_calls_log = await run_investigation(prompt, session_id) return result @@ -146,12 +147,19 @@ async def _deliver(result: dict[str, Any], event: dict, dedup_key: str, session_ slack_sent = False if settings.slack_webhook_url: try: - from integrations.slack_webhook import post_failed_investigation, post_investigation + from opendevops_core.integrations.slack_webhook import ( + post_failed_investigation, + post_investigation, + ) if status == "failed": slack_sent = await post_failed_investigation( - settings.slack_webhook_url, service, root_cause, - session_id, aws_error=aws_error, is_test=is_test, + settings.slack_webhook_url, + service, + root_cause, + session_id, + aws_error=aws_error, + is_test=is_test, ) else: slack_sent = await post_investigation( @@ -163,18 +171,25 @@ async def _deliver(result: dict[str, Any], event: dict, dedup_key: str, session_ telegram_sent = False if settings.telegram_bot_token and settings.telegram_chat_id: try: - from integrations.telegram import post_failed_investigation as tg_fail - from integrations.telegram import post_investigation as tg_ok + from opendevops_core.integrations.telegram import post_failed_investigation as tg_fail + from opendevops_core.integrations.telegram import post_investigation as tg_ok if status == "failed": telegram_sent = await tg_fail( - settings.telegram_bot_token, settings.telegram_chat_id, - service, root_cause, session_id, is_test=is_test, + settings.telegram_bot_token, + settings.telegram_chat_id, + service, + root_cause, + session_id, + is_test=is_test, ) else: telegram_sent = await tg_ok( - settings.telegram_bot_token, settings.telegram_chat_id, - result, session_id, is_test=is_test, + settings.telegram_bot_token, + settings.telegram_chat_id, + result, + session_id, + is_test=is_test, ) except Exception as e: logger.error("Telegram delivery failed from event consumer: {}", e) @@ -236,7 +251,9 @@ async def _process_event(event: dict) -> ProcessResult: if not await claim_incident(incident_key, "event_consumer", _claim_window_minutes()): logger.info( "Dedup: skipping {} / {} (incident_key={}) — already claimed recently", - source, detail_type, incident_key, + source, + detail_type, + incident_key, ) return "duplicate" @@ -249,7 +266,7 @@ async def _process_event(event: dict) -> ProcessResult: # Enrich prompt with deterministic boto3 context (no LLM cost) try: - from providers.aws.context import collect_context + from opendevops_core.providers.aws.context import collect_context context = collect_context(event) if context and not context.get("error"): @@ -266,7 +283,9 @@ async def _process_event(event: dict) -> ProcessResult: try: logger.debug( "Event consumer: starting investigation for {} / {} (session {})", - source, detail_type, session_id[:8], + source, + detail_type, + session_id[:8], ) result = await _run_investigation(prompt, session_id) if not result: diff --git a/src/providers/aws/event_infra.py b/apps/core/src/opendevops_core/providers/aws/event_infra.py similarity index 96% rename from src/providers/aws/event_infra.py rename to apps/core/src/opendevops_core/providers/aws/event_infra.py index 1484c7b..6235c23 100644 --- a/src/providers/aws/event_infra.py +++ b/apps/core/src/opendevops_core/providers/aws/event_infra.py @@ -8,7 +8,7 @@ from botocore.exceptions import ClientError from loguru import logger -from config import settings +from opendevops_core.config import settings QUEUE_NAME = "opendevops-agent-events" DLQ_QUEUE_NAME = "opendevops-agent-events-dlq" @@ -130,10 +130,12 @@ def setup_event_infra(region: str | None = None) -> dict: "MessageRetentionPeriod": "86400", "VisibilityTimeout": visibility_timeout, "ReceiveMessageWaitTimeSeconds": "20", - "RedrivePolicy": json.dumps({ - "deadLetterTargetArn": dlq_arn, - "maxReceiveCount": MAX_RECEIVE_COUNT, - }), + "RedrivePolicy": json.dumps( + { + "deadLetterTargetArn": dlq_arn, + "maxReceiveCount": MAX_RECEIVE_COUNT, + } + ), }, ) queue_url = resp["QueueUrl"] @@ -195,7 +197,10 @@ def setup_event_infra(region: str | None = None) -> dict: logger.info( "Event infra created: queue={} dlq={} rules={} alarm={}", - queue_url, dlq_url, len(rule_arns), ALARM_NAME, + queue_url, + dlq_url, + len(rule_arns), + ALARM_NAME, ) return { "queue_url": queue_url, diff --git a/src/providers/aws/permissions.py b/apps/core/src/opendevops_core/providers/aws/permissions.py similarity index 66% rename from src/providers/aws/permissions.py rename to apps/core/src/opendevops_core/providers/aws/permissions.py index 61c31d8..5540a12 100644 --- a/src/providers/aws/permissions.py +++ b/apps/core/src/opendevops_core/providers/aws/permissions.py @@ -5,7 +5,7 @@ import boto3 from loguru import logger -from config import settings +from opendevops_core.config import settings def _session() -> boto3.Session: @@ -36,13 +36,13 @@ def check_permissions(region: str | None = None) -> dict[str, dict]: results: dict[str, dict] = { "cloudwatch": _check(lambda: _c(s, "cloudwatch", region).describe_alarms(MaxRecords=1)), "cloudtrail": _check(lambda: _c(s, "cloudtrail", region).lookup_events(MaxResults=1)), - "ecs": _check(lambda: _c(s, "ecs", region).list_clusters(maxResults=1)), - "lambda": _check(lambda: _c(s, "lambda", region).list_functions(MaxItems=1)), - "ec2": _check(lambda: _c(s, "ec2", region).describe_instances(MaxResults=5)), - "rds": _check(lambda: _c(s, "rds", region).describe_db_instances()), - "iam": _check(lambda: _c(s, "sts", region).get_caller_identity()), - "sqs": _check(lambda: _c(s, "sqs", region).list_queues(MaxResults=1)), - "events": _check(lambda: _c(s, "events", region).list_rules(Limit=1)), + "ecs": _check(lambda: _c(s, "ecs", region).list_clusters(maxResults=1)), + "lambda": _check(lambda: _c(s, "lambda", region).list_functions(MaxItems=1)), + "ec2": _check(lambda: _c(s, "ec2", region).describe_instances(MaxResults=5)), + "rds": _check(lambda: _c(s, "rds", region).describe_db_instances()), + "iam": _check(lambda: _c(s, "sts", region).get_caller_identity()), + "sqs": _check(lambda: _c(s, "sqs", region).list_queues(MaxResults=1)), + "events": _check(lambda: _c(s, "events", region).list_rules(Limit=1)), } for svc, r in results.items(): diff --git a/src/providers/aws/poller.py b/apps/core/src/opendevops_core/providers/aws/poller.py similarity index 85% rename from src/providers/aws/poller.py rename to apps/core/src/opendevops_core/providers/aws/poller.py index f2743f4..e4372ad 100644 --- a/src/providers/aws/poller.py +++ b/apps/core/src/opendevops_core/providers/aws/poller.py @@ -15,11 +15,11 @@ from loguru import logger -from agent.incident_keys import ( +from opendevops_core.agent.incident_keys import ( alarm_incident_key, lambda_error_incident_key, ) -from config import settings +from opendevops_core.config import settings # Dedicated pool so poller boto3 calls never compete with API request threads _POLLER_POOL = ThreadPoolExecutor(max_workers=4, thread_name_prefix="poller") @@ -32,15 +32,16 @@ def _claim_window_minutes() -> int: async def _run_investigation( prompt: str, session_id: str ) -> tuple[dict[str, Any] | None, list[dict[str, Any]]]: - from agent.investigation_runner import run_investigation + from opendevops_core.agent.investigation_runner import run_investigation + return await run_investigation(prompt, session_id) async def _persist_and_notify( result: dict[str, Any], tool_calls_log: list[dict[str, Any]], session_id: str, dedup_key: str ) -> None: - from agent.monitor_store import add_alert, add_notification, update_service - from agent.turns import notify_slack + from opendevops_core.agent.monitor_store import add_alert, add_notification, update_service + from opendevops_core.agent.turns import notify_slack status = result.pop("_status", "completed") services_affected = result.get("services_affected", []) @@ -53,7 +54,8 @@ async def _persist_and_notify( slack_sent = False if settings.slack_webhook_url: if status == "failed": - from integrations.slack_webhook import post_failed_investigation + from opendevops_core.integrations.slack_webhook import post_failed_investigation + slack_sent = await post_failed_investigation( settings.slack_webhook_url, service, root_cause, session_id ) @@ -63,18 +65,23 @@ async def _persist_and_notify( telegram_sent = False if settings.telegram_bot_token and settings.telegram_chat_id: try: - from integrations.telegram import post_failed_investigation as tg_fail - from integrations.telegram import post_investigation as tg_ok + from opendevops_core.integrations.telegram import post_failed_investigation as tg_fail + from opendevops_core.integrations.telegram import post_investigation as tg_ok if status == "failed": telegram_sent = await tg_fail( - settings.telegram_bot_token, settings.telegram_chat_id, - service, root_cause, session_id, + settings.telegram_bot_token, + settings.telegram_chat_id, + service, + root_cause, + session_id, ) else: telegram_sent = await tg_ok( - settings.telegram_bot_token, settings.telegram_chat_id, - result, session_id, + settings.telegram_bot_token, + settings.telegram_chat_id, + result, + session_id, ) except Exception as e: logger.error("Telegram delivery failed from poller: {}", e) @@ -103,13 +110,13 @@ async def _check_alarms() -> None: """Check CloudWatch alarms — each new ALARM state triggers an investigation.""" import uuid - from agent.monitor_store import ( + from opendevops_core.agent.monitor_store import ( claim_incident, complete_incident, is_recent_alert, release_incident, ) - from providers.aws.tools.cloudwatch import get_alarms + from opendevops_core.providers.aws.tools.cloudwatch import get_alarms data = await asyncio.get_event_loop().run_in_executor(_POLLER_POOL, get_alarms, "ALARM") alarms = data.get("alarms", []) @@ -154,14 +161,16 @@ async def _check_alarms() -> None: session_id = str(uuid.uuid4()) logger.debug( "Poller: starting investigation for alarm {} (session {})", - name, session_id[:8], + name, + session_id[:8], ) try: result, tool_calls_log = await _run_investigation(prompt, session_id) if result: logger.info( "Poller: investigation complete for alarm {} — {}", - name, result.get("confidence", "?"), + name, + result.get("confidence", "?"), ) status = result.get("_status", "completed") await _persist_and_notify(result, tool_calls_log, session_id, key) @@ -177,13 +186,16 @@ async def _check_lambda_errors() -> None: """Check Lambda functions for error rates above the configured threshold.""" import uuid - from agent.monitor_store import ( + from opendevops_core.agent.monitor_store import ( claim_incident, complete_incident, is_incident_claimed, release_incident, ) - from providers.aws.tools.lambda_ import get_lambda_error_rate, list_lambda_functions + from opendevops_core.providers.aws.tools.lambda_ import ( + get_lambda_error_rate, + list_lambda_functions, + ) funcs = await asyncio.get_event_loop().run_in_executor(_POLLER_POOL, list_lambda_functions) fn_list = funcs.get("functions", [])[:20] @@ -205,7 +217,8 @@ async def _check_lambda_errors() -> None: continue # The aggregate alarm covers all Lambda errors — if it fired recently, skip. - from providers.aws.event_infra import ALARM_NAME + from opendevops_core.providers.aws.event_infra import ALARM_NAME + aggregate_key = alarm_incident_key(ALARM_NAME) aggregate_window = max(3, (settings.investigation_timeout // 60) + 2) if await is_incident_claimed(aggregate_key, aggregate_window): @@ -234,14 +247,16 @@ async def _check_lambda_errors() -> None: session_id = str(uuid.uuid4()) logger.debug( "Poller: starting investigation for Lambda {} (session {})", - name, session_id[:8], + name, + session_id[:8], ) try: result, tool_calls_log = await _run_investigation(prompt, session_id) if result: logger.info( "Poller: investigation complete for Lambda {} — {}", - name, result.get("confidence", "?"), + name, + result.get("confidence", "?"), ) status = result.get("_status", "completed") await _persist_and_notify(result, tool_calls_log, session_id, key) @@ -266,7 +281,8 @@ async def polling_loop() -> None: await asyncio.sleep(interval) logger.debug("Poller tick — checking alarms and Lambda error rates") try: - from agent.init_store import is_event_infra_enabled + from opendevops_core.agent.init_store import is_event_infra_enabled + if not is_event_infra_enabled(): # EventBridge → SQS already covers alarm state changes when infra is active await _check_alarms() diff --git a/src/providers/aws/tools/__init__.py b/apps/core/src/opendevops_core/providers/aws/tools/__init__.py similarity index 100% rename from src/providers/aws/tools/__init__.py rename to apps/core/src/opendevops_core/providers/aws/tools/__init__.py diff --git a/src/providers/aws/tools/cloudtrail.py b/apps/core/src/opendevops_core/providers/aws/tools/cloudtrail.py similarity index 93% rename from src/providers/aws/tools/cloudtrail.py rename to apps/core/src/opendevops_core/providers/aws/tools/cloudtrail.py index f57a470..5bd7cbc 100644 --- a/src/providers/aws/tools/cloudtrail.py +++ b/apps/core/src/opendevops_core/providers/aws/tools/cloudtrail.py @@ -7,9 +7,9 @@ from botocore.exceptions import BotoCoreError, ClientError from loguru import logger -from agent.init_store import get_runtime_aws_region -from config import settings -from tools._cache import tool_cached +from opendevops_core.agent.init_store import get_runtime_aws_region +from opendevops_core.config import settings +from opendevops_core.tools._cache import tool_cached def _ct_client() -> Any: diff --git a/src/providers/aws/tools/cloudwatch.py b/apps/core/src/opendevops_core/providers/aws/tools/cloudwatch.py similarity index 98% rename from src/providers/aws/tools/cloudwatch.py rename to apps/core/src/opendevops_core/providers/aws/tools/cloudwatch.py index 12038c9..95042a5 100644 --- a/src/providers/aws/tools/cloudwatch.py +++ b/apps/core/src/opendevops_core/providers/aws/tools/cloudwatch.py @@ -8,9 +8,9 @@ from botocore.exceptions import BotoCoreError, ClientError from loguru import logger -from agent.init_store import get_runtime_aws_region -from config import settings -from tools._cache import tool_cached +from opendevops_core.agent.init_store import get_runtime_aws_region +from opendevops_core.config import settings +from opendevops_core.tools._cache import tool_cached def _cw_client() -> Any: diff --git a/src/providers/aws/tools/ec2.py b/apps/core/src/opendevops_core/providers/aws/tools/ec2.py similarity index 95% rename from src/providers/aws/tools/ec2.py rename to apps/core/src/opendevops_core/providers/aws/tools/ec2.py index 8d8e9e7..cce8c39 100644 --- a/src/providers/aws/tools/ec2.py +++ b/apps/core/src/opendevops_core/providers/aws/tools/ec2.py @@ -6,9 +6,9 @@ from botocore.exceptions import BotoCoreError, ClientError from loguru import logger -from agent.init_store import get_runtime_aws_region -from config import settings -from tools._cache import tool_cached +from opendevops_core.agent.init_store import get_runtime_aws_region +from opendevops_core.config import settings +from opendevops_core.tools._cache import tool_cached def _ec2_client() -> Any: diff --git a/src/providers/aws/tools/ecs.py b/apps/core/src/opendevops_core/providers/aws/tools/ecs.py similarity index 97% rename from src/providers/aws/tools/ecs.py rename to apps/core/src/opendevops_core/providers/aws/tools/ecs.py index d03f186..b3864cb 100644 --- a/src/providers/aws/tools/ecs.py +++ b/apps/core/src/opendevops_core/providers/aws/tools/ecs.py @@ -6,9 +6,9 @@ from botocore.exceptions import BotoCoreError, ClientError from loguru import logger -from agent.init_store import get_runtime_aws_region -from config import settings -from tools._cache import tool_cached +from opendevops_core.agent.init_store import get_runtime_aws_region +from opendevops_core.config import settings +from opendevops_core.tools._cache import tool_cached def _ecs_client() -> Any: diff --git a/src/providers/aws/tools/iam.py b/apps/core/src/opendevops_core/providers/aws/tools/iam.py similarity index 92% rename from src/providers/aws/tools/iam.py rename to apps/core/src/opendevops_core/providers/aws/tools/iam.py index 16d0a37..f0ab71a 100644 --- a/src/providers/aws/tools/iam.py +++ b/apps/core/src/opendevops_core/providers/aws/tools/iam.py @@ -6,9 +6,9 @@ from botocore.exceptions import BotoCoreError, ClientError from loguru import logger -from agent.init_store import get_runtime_aws_region -from config import settings -from tools._cache import tool_cached +from opendevops_core.agent.init_store import get_runtime_aws_region +from opendevops_core.config import settings +from opendevops_core.tools._cache import tool_cached def _iam_client() -> Any: diff --git a/src/providers/aws/tools/lambda_.py b/apps/core/src/opendevops_core/providers/aws/tools/lambda_.py similarity index 96% rename from src/providers/aws/tools/lambda_.py rename to apps/core/src/opendevops_core/providers/aws/tools/lambda_.py index 6463762..d38b7ae 100644 --- a/src/providers/aws/tools/lambda_.py +++ b/apps/core/src/opendevops_core/providers/aws/tools/lambda_.py @@ -7,9 +7,9 @@ from botocore.exceptions import BotoCoreError, ClientError from loguru import logger -from agent.init_store import get_runtime_aws_region -from config import settings -from tools._cache import tool_cached +from opendevops_core.agent.init_store import get_runtime_aws_region +from opendevops_core.config import settings +from opendevops_core.tools._cache import tool_cached def _lambda_client() -> Any: diff --git a/src/providers/aws/tools/rds.py b/apps/core/src/opendevops_core/providers/aws/tools/rds.py similarity index 94% rename from src/providers/aws/tools/rds.py rename to apps/core/src/opendevops_core/providers/aws/tools/rds.py index 626740f..a5ff4d0 100644 --- a/src/providers/aws/tools/rds.py +++ b/apps/core/src/opendevops_core/providers/aws/tools/rds.py @@ -7,9 +7,9 @@ from botocore.exceptions import BotoCoreError, ClientError from loguru import logger -from agent.init_store import get_runtime_aws_region -from config import settings -from tools._cache import tool_cached +from opendevops_core.agent.init_store import get_runtime_aws_region +from opendevops_core.config import settings +from opendevops_core.tools._cache import tool_cached def _rds_client() -> Any: diff --git a/src/providers/azure/__init__.py b/apps/core/src/opendevops_core/providers/azure/__init__.py similarity index 100% rename from src/providers/azure/__init__.py rename to apps/core/src/opendevops_core/providers/azure/__init__.py diff --git a/src/providers/base.py b/apps/core/src/opendevops_core/providers/base.py similarity index 100% rename from src/providers/base.py rename to apps/core/src/opendevops_core/providers/base.py diff --git a/src/providers/gcp/__init__.py b/apps/core/src/opendevops_core/providers/gcp/__init__.py similarity index 100% rename from src/providers/gcp/__init__.py rename to apps/core/src/opendevops_core/providers/gcp/__init__.py diff --git a/src/skills/dynamodb-throttling/SKILL.md b/apps/core/src/opendevops_core/skills/dynamodb-throttling/SKILL.md similarity index 100% rename from src/skills/dynamodb-throttling/SKILL.md rename to apps/core/src/opendevops_core/skills/dynamodb-throttling/SKILL.md diff --git a/src/skills/ecs-deployment-failure/SKILL.md b/apps/core/src/opendevops_core/skills/ecs-deployment-failure/SKILL.md similarity index 100% rename from src/skills/ecs-deployment-failure/SKILL.md rename to apps/core/src/opendevops_core/skills/ecs-deployment-failure/SKILL.md diff --git a/src/skills/iam-permission-denied/SKILL.md b/apps/core/src/opendevops_core/skills/iam-permission-denied/SKILL.md similarity index 100% rename from src/skills/iam-permission-denied/SKILL.md rename to apps/core/src/opendevops_core/skills/iam-permission-denied/SKILL.md diff --git a/src/skills/lambda-throttling/SKILL.md b/apps/core/src/opendevops_core/skills/lambda-throttling/SKILL.md similarity index 100% rename from src/skills/lambda-throttling/SKILL.md rename to apps/core/src/opendevops_core/skills/lambda-throttling/SKILL.md diff --git a/tests/test_tools/__init__.py b/apps/core/src/opendevops_core/tools/__init__.py similarity index 100% rename from tests/test_tools/__init__.py rename to apps/core/src/opendevops_core/tools/__init__.py diff --git a/src/tools/_cache.py b/apps/core/src/opendevops_core/tools/_cache.py similarity index 92% rename from src/tools/_cache.py rename to apps/core/src/opendevops_core/tools/_cache.py index 59edcd0..f356e6e 100644 --- a/src/tools/_cache.py +++ b/apps/core/src/opendevops_core/tools/_cache.py @@ -18,8 +18,8 @@ from cachetools import TTLCache, cached from cachetools.keys import hashkey -from agent.init_store import get_runtime_aws_region -from config import settings +from opendevops_core.agent.init_store import get_runtime_aws_region +from opendevops_core.config import settings F = TypeVar("F", bound=Callable[..., Any]) diff --git a/src/tools/_cap.py b/apps/core/src/opendevops_core/tools/_cap.py similarity index 97% rename from src/tools/_cap.py rename to apps/core/src/opendevops_core/tools/_cap.py index 7d7bf4d..2f1f1e2 100644 --- a/src/tools/_cap.py +++ b/apps/core/src/opendevops_core/tools/_cap.py @@ -12,7 +12,7 @@ from functools import wraps from typing import Any -from config import settings +from opendevops_core.config import settings def cap_tool_result(result: Any, max_chars: int | None = None) -> Any: @@ -46,7 +46,9 @@ def cap_tool_result(result: Any, max_chars: int | None = None) -> Any: def with_cap(fn: Any) -> Any: """Wrap a tool function so its return value is passed through cap_tool_result.""" + @wraps(fn) def wrapper(*args: Any, **kwargs: Any) -> Any: return cap_tool_result(fn(*args, **kwargs)) + return wrapper diff --git a/src/tools/base.py b/apps/core/src/opendevops_core/tools/base.py similarity index 100% rename from src/tools/base.py rename to apps/core/src/opendevops_core/tools/base.py diff --git a/src/tools/bash_tool.py b/apps/core/src/opendevops_core/tools/bash_tool.py similarity index 81% rename from src/tools/bash_tool.py rename to apps/core/src/opendevops_core/tools/bash_tool.py index 957b676..dc06d0d 100644 --- a/src/tools/bash_tool.py +++ b/apps/core/src/opendevops_core/tools/bash_tool.py @@ -16,47 +16,64 @@ # AWS CLI read-only operation verbs (the word before the first dash in the operation name). # Every read-only AWS CLI command starts with one of these — across ALL services. # e.g. "aws s3api list-buckets", "aws dynamodb describe-table", "aws sns get-topic-attributes" -_AWS_READONLY_VERBS: frozenset[str] = frozenset({ - "describe", "list", "get", "lookup", "filter", - "search", "scan", "query", "show", "view", "check", - "batch-get", -}) +_AWS_READONLY_VERBS: frozenset[str] = frozenset( + { + "describe", + "list", + "get", + "lookup", + "filter", + "search", + "scan", + "query", + "show", + "view", + "check", + "batch-get", + } +) _COMMAND_MAX_LEN = 2000 _CHAIN_TOKENS: frozenset[str] = frozenset({";", "&&", "||", "|", ">", ">>", "<"}) -_AWS_GLOBAL_FLAGS_WITH_VALUE: frozenset[str] = frozenset({ - "--profile", - "--region", - "--output", - "--query", - "--endpoint-url", - "--ca-bundle", - "--cli-connect-timeout", - "--cli-read-timeout", -}) -_AWS_GLOBAL_FLAGS_NO_VALUE: frozenset[str] = frozenset({ - "--debug", - "--no-cli-pager", - "--no-verify-ssl", - "--color", -}) +_AWS_GLOBAL_FLAGS_WITH_VALUE: frozenset[str] = frozenset( + { + "--profile", + "--region", + "--output", + "--query", + "--endpoint-url", + "--ca-bundle", + "--cli-connect-timeout", + "--cli-read-timeout", + } +) +_AWS_GLOBAL_FLAGS_NO_VALUE: frozenset[str] = frozenset( + { + "--debug", + "--no-cli-pager", + "--no-verify-ssl", + "--color", + } +) _AWS_BLOCKED_GLOBAL_FLAGS: frozenset[str] = frozenset({"--endpoint-url"}) -_KUBECTL_FLAGS_WITH_VALUE: frozenset[str] = frozenset({ - "-n", - "--namespace", - "--context", - "--cluster", - "--user", - "--kubeconfig", - "--server", - "--request-timeout", -}) +_KUBECTL_FLAGS_WITH_VALUE: frozenset[str] = frozenset( + { + "-n", + "--namespace", + "--context", + "--cluster", + "--user", + "--kubeconfig", + "--server", + "--request-timeout", + } +) _DOCKER_FLAGS_WITH_VALUE: frozenset[str] = frozenset({"-H", "--host", "--context", "--config"}) -_TIMEOUT = 30 +_TIMEOUT = 30 _MAX_OUTPUT = 4000 _MAX_STDERR = 1000 @@ -169,8 +186,8 @@ def run_bash_command(command: str) -> dict[str, Any]: logger.warning("bash_tool BLOCKED | cmd={!r}", command) return { "success": False, - "output": "", - "error": ( + "output": "", + "error": ( "Command blocked. AWS commands must use safe global flags and a " "read-only operation verb (describe-, list-, get-, lookup-, filter-, " "search-, scan-, query, batch-get-). " @@ -192,12 +209,14 @@ def run_bash_command(command: str) -> dict[str, Any]: success = proc.returncode == 0 logger.info( "bash_tool DONE | rc={} elapsed={:.2f}s cmd={!r}", - proc.returncode, elapsed, command, + proc.returncode, + elapsed, + command, ) return { "success": success, - "output": proc.stdout[:_MAX_OUTPUT], - "error": proc.stderr[:_MAX_STDERR] if proc.stderr else "", + "output": proc.stdout[:_MAX_OUTPUT], + "error": proc.stderr[:_MAX_STDERR] if proc.stderr else "", "command": command, "blocked": False, } @@ -207,8 +226,8 @@ def run_bash_command(command: str) -> dict[str, Any]: logger.warning("bash_tool TIMEOUT | elapsed={:.1f}s cmd={!r}", elapsed, command) return { "success": False, - "output": "", - "error": f"Command timed out after {_TIMEOUT}s", + "output": "", + "error": f"Command timed out after {_TIMEOUT}s", "command": command, "blocked": False, } @@ -218,8 +237,8 @@ def run_bash_command(command: str) -> dict[str, Any]: logger.error("bash_tool ERROR | elapsed={:.1f}s cmd={!r} err={}", elapsed, command, exc) return { "success": False, - "output": "", - "error": str(exc), + "output": "", + "error": str(exc), "command": command, "blocked": False, } diff --git a/src/tools/final_answer.py b/apps/core/src/opendevops_core/tools/final_answer.py similarity index 100% rename from src/tools/final_answer.py rename to apps/core/src/opendevops_core/tools/final_answer.py diff --git a/src/tools/history.py b/apps/core/src/opendevops_core/tools/history.py similarity index 96% rename from src/tools/history.py rename to apps/core/src/opendevops_core/tools/history.py index 6b6d44d..7f458af 100644 --- a/src/tools/history.py +++ b/apps/core/src/opendevops_core/tools/history.py @@ -2,7 +2,7 @@ from __future__ import annotations -from agent.db import db +from opendevops_core.agent.db import db async def get_investigation_history(days: int = 30) -> dict: diff --git a/src/tools/skills.py b/apps/core/src/opendevops_core/tools/skills.py similarity index 100% rename from src/tools/skills.py rename to apps/core/src/opendevops_core/tools/skills.py diff --git a/docs/auth.md b/apps/documentation/auth.md similarity index 100% rename from docs/auth.md rename to apps/documentation/auth.md diff --git a/docs/aws_tools.md b/apps/documentation/aws_tools.md similarity index 100% rename from docs/aws_tools.md rename to apps/documentation/aws_tools.md diff --git a/docs/caching.md b/apps/documentation/caching.md similarity index 100% rename from docs/caching.md rename to apps/documentation/caching.md diff --git a/docs/cli.md b/apps/documentation/cli.md similarity index 100% rename from docs/cli.md rename to apps/documentation/cli.md diff --git a/docs/conversation_summarization.md b/apps/documentation/conversation_summarization.md similarity index 100% rename from docs/conversation_summarization.md rename to apps/documentation/conversation_summarization.md diff --git a/docs/dashboard.md b/apps/documentation/dashboard.md similarity index 100% rename from docs/dashboard.md rename to apps/documentation/dashboard.md diff --git a/docs/databases.md b/apps/documentation/databases.md similarity index 100% rename from docs/databases.md rename to apps/documentation/databases.md diff --git a/docs/event_detection.md b/apps/documentation/event_detection.md similarity index 100% rename from docs/event_detection.md rename to apps/documentation/event_detection.md diff --git a/docs/iam_setup.md b/apps/documentation/iam_setup.md similarity index 100% rename from docs/iam_setup.md rename to apps/documentation/iam_setup.md diff --git a/docs/llm_providers.md b/apps/documentation/llm_providers.md similarity index 100% rename from docs/llm_providers.md rename to apps/documentation/llm_providers.md diff --git a/docs/mcp_server.md b/apps/documentation/mcp_server.md similarity index 100% rename from docs/mcp_server.md rename to apps/documentation/mcp_server.md diff --git a/docs/monitoring.md b/apps/documentation/monitoring.md similarity index 100% rename from docs/monitoring.md rename to apps/documentation/monitoring.md diff --git a/docs/schema.md b/apps/documentation/schema.md similarity index 100% rename from docs/schema.md rename to apps/documentation/schema.md diff --git a/docs/skills.md b/apps/documentation/skills.md similarity index 100% rename from docs/skills.md rename to apps/documentation/skills.md diff --git a/docs/slack_and_polling.md b/apps/documentation/slack_and_polling.md similarity index 100% rename from docs/slack_and_polling.md rename to apps/documentation/slack_and_polling.md diff --git a/docs/telegram.md b/apps/documentation/telegram.md similarity index 100% rename from docs/telegram.md rename to apps/documentation/telegram.md diff --git a/docs/tool_capping.md b/apps/documentation/tool_capping.md similarity index 100% rename from docs/tool_capping.md rename to apps/documentation/tool_capping.md diff --git a/docs/ui.md b/apps/documentation/ui.md similarity index 100% rename from docs/ui.md rename to apps/documentation/ui.md diff --git a/apps/frontend/.dockerignore b/apps/frontend/.dockerignore new file mode 100644 index 0000000..cda567b --- /dev/null +++ b/apps/frontend/.dockerignore @@ -0,0 +1,4 @@ +node_modules/ +dist/ +.vite/ +*.md diff --git a/frontend/.gitignore b/apps/frontend/.gitignore similarity index 100% rename from frontend/.gitignore rename to apps/frontend/.gitignore diff --git a/frontend/Dockerfile b/apps/frontend/Dockerfile similarity index 100% rename from frontend/Dockerfile rename to apps/frontend/Dockerfile diff --git a/frontend/index.html b/apps/frontend/index.html similarity index 100% rename from frontend/index.html rename to apps/frontend/index.html diff --git a/frontend/nginx.conf b/apps/frontend/nginx.conf similarity index 100% rename from frontend/nginx.conf rename to apps/frontend/nginx.conf diff --git a/frontend/old-index.html b/apps/frontend/old-index.html similarity index 100% rename from frontend/old-index.html rename to apps/frontend/old-index.html diff --git a/frontend/package-lock.json b/apps/frontend/package-lock.json similarity index 100% rename from frontend/package-lock.json rename to apps/frontend/package-lock.json diff --git a/frontend/package.json b/apps/frontend/package.json similarity index 100% rename from frontend/package.json rename to apps/frontend/package.json diff --git a/frontend/postcss.config.js b/apps/frontend/postcss.config.js similarity index 100% rename from frontend/postcss.config.js rename to apps/frontend/postcss.config.js diff --git a/frontend/src/App.tsx b/apps/frontend/src/App.tsx similarity index 100% rename from frontend/src/App.tsx rename to apps/frontend/src/App.tsx diff --git a/frontend/src/components/AgentMessage.tsx b/apps/frontend/src/components/AgentMessage.tsx similarity index 100% rename from frontend/src/components/AgentMessage.tsx rename to apps/frontend/src/components/AgentMessage.tsx diff --git a/frontend/src/components/EmptyState.tsx b/apps/frontend/src/components/EmptyState.tsx similarity index 100% rename from frontend/src/components/EmptyState.tsx rename to apps/frontend/src/components/EmptyState.tsx diff --git a/frontend/src/components/InputArea.tsx b/apps/frontend/src/components/InputArea.tsx similarity index 100% rename from frontend/src/components/InputArea.tsx rename to apps/frontend/src/components/InputArea.tsx diff --git a/frontend/src/components/LlmBackendCard.tsx b/apps/frontend/src/components/LlmBackendCard.tsx similarity index 100% rename from frontend/src/components/LlmBackendCard.tsx rename to apps/frontend/src/components/LlmBackendCard.tsx diff --git a/frontend/src/components/ProtectedRoute.tsx b/apps/frontend/src/components/ProtectedRoute.tsx similarity index 100% rename from frontend/src/components/ProtectedRoute.tsx rename to apps/frontend/src/components/ProtectedRoute.tsx diff --git a/frontend/src/components/Sidebar.tsx b/apps/frontend/src/components/Sidebar.tsx similarity index 100% rename from frontend/src/components/Sidebar.tsx rename to apps/frontend/src/components/Sidebar.tsx diff --git a/frontend/src/components/StreamStatus.tsx b/apps/frontend/src/components/StreamStatus.tsx similarity index 100% rename from frontend/src/components/StreamStatus.tsx rename to apps/frontend/src/components/StreamStatus.tsx diff --git a/frontend/src/components/ToolCallsBox.tsx b/apps/frontend/src/components/ToolCallsBox.tsx similarity index 100% rename from frontend/src/components/ToolCallsBox.tsx rename to apps/frontend/src/components/ToolCallsBox.tsx diff --git a/frontend/src/components/UsageBox.tsx b/apps/frontend/src/components/UsageBox.tsx similarity index 100% rename from frontend/src/components/UsageBox.tsx rename to apps/frontend/src/components/UsageBox.tsx diff --git a/frontend/src/components/UserMessage.tsx b/apps/frontend/src/components/UserMessage.tsx similarity index 100% rename from frontend/src/components/UserMessage.tsx rename to apps/frontend/src/components/UserMessage.tsx diff --git a/frontend/src/components/icons/IntegrationIcon.tsx b/apps/frontend/src/components/icons/IntegrationIcon.tsx similarity index 100% rename from frontend/src/components/icons/IntegrationIcon.tsx rename to apps/frontend/src/components/icons/IntegrationIcon.tsx diff --git a/frontend/src/context/AuthContext.tsx b/apps/frontend/src/context/AuthContext.tsx similarity index 100% rename from frontend/src/context/AuthContext.tsx rename to apps/frontend/src/context/AuthContext.tsx diff --git a/frontend/src/context/ThemeContext.tsx b/apps/frontend/src/context/ThemeContext.tsx similarity index 100% rename from frontend/src/context/ThemeContext.tsx rename to apps/frontend/src/context/ThemeContext.tsx diff --git a/frontend/src/index.css b/apps/frontend/src/index.css similarity index 100% rename from frontend/src/index.css rename to apps/frontend/src/index.css diff --git a/frontend/src/lib/api.ts b/apps/frontend/src/lib/api.ts similarity index 100% rename from frontend/src/lib/api.ts rename to apps/frontend/src/lib/api.ts diff --git a/frontend/src/lib/utils.ts b/apps/frontend/src/lib/utils.ts similarity index 100% rename from frontend/src/lib/utils.ts rename to apps/frontend/src/lib/utils.ts diff --git a/frontend/src/main.tsx b/apps/frontend/src/main.tsx similarity index 100% rename from frontend/src/main.tsx rename to apps/frontend/src/main.tsx diff --git a/frontend/src/pages/AlertDetailPage.tsx b/apps/frontend/src/pages/AlertDetailPage.tsx similarity index 100% rename from frontend/src/pages/AlertDetailPage.tsx rename to apps/frontend/src/pages/AlertDetailPage.tsx diff --git a/frontend/src/pages/ChatPage.tsx b/apps/frontend/src/pages/ChatPage.tsx similarity index 100% rename from frontend/src/pages/ChatPage.tsx rename to apps/frontend/src/pages/ChatPage.tsx diff --git a/frontend/src/pages/DashboardPage.tsx b/apps/frontend/src/pages/DashboardPage.tsx similarity index 100% rename from frontend/src/pages/DashboardPage.tsx rename to apps/frontend/src/pages/DashboardPage.tsx diff --git a/frontend/src/pages/HistoryPage.tsx b/apps/frontend/src/pages/HistoryPage.tsx similarity index 100% rename from frontend/src/pages/HistoryPage.tsx rename to apps/frontend/src/pages/HistoryPage.tsx diff --git a/frontend/src/pages/InitPage.tsx b/apps/frontend/src/pages/InitPage.tsx similarity index 100% rename from frontend/src/pages/InitPage.tsx rename to apps/frontend/src/pages/InitPage.tsx diff --git a/frontend/src/pages/LoginPage.tsx b/apps/frontend/src/pages/LoginPage.tsx similarity index 100% rename from frontend/src/pages/LoginPage.tsx rename to apps/frontend/src/pages/LoginPage.tsx diff --git a/frontend/src/pages/MonitoringPage.tsx b/apps/frontend/src/pages/MonitoringPage.tsx similarity index 100% rename from frontend/src/pages/MonitoringPage.tsx rename to apps/frontend/src/pages/MonitoringPage.tsx diff --git a/frontend/src/pages/SettingsPage.tsx b/apps/frontend/src/pages/SettingsPage.tsx similarity index 100% rename from frontend/src/pages/SettingsPage.tsx rename to apps/frontend/src/pages/SettingsPage.tsx diff --git a/frontend/src/pages/UsersPage.tsx b/apps/frontend/src/pages/UsersPage.tsx similarity index 100% rename from frontend/src/pages/UsersPage.tsx rename to apps/frontend/src/pages/UsersPage.tsx diff --git a/frontend/src/types.ts b/apps/frontend/src/types.ts similarity index 100% rename from frontend/src/types.ts rename to apps/frontend/src/types.ts diff --git a/frontend/tailwind.config.js b/apps/frontend/tailwind.config.js similarity index 100% rename from frontend/tailwind.config.js rename to apps/frontend/tailwind.config.js diff --git a/frontend/tsconfig.json b/apps/frontend/tsconfig.json similarity index 100% rename from frontend/tsconfig.json rename to apps/frontend/tsconfig.json diff --git a/frontend/vite.config.ts b/apps/frontend/vite.config.ts similarity index 100% rename from frontend/vite.config.ts rename to apps/frontend/vite.config.ts diff --git a/docker-compose.yml b/deployment/docker-compose/docker-compose.yml similarity index 96% rename from docker-compose.yml rename to deployment/docker-compose/docker-compose.yml index c940ffe..0f119f8 100644 --- a/docker-compose.yml +++ b/deployment/docker-compose/docker-compose.yml @@ -20,10 +20,10 @@ services: # ── FastAPI backend ───────────────────────────────────────────────────────── backend: - build: . + build: ../../apps/backend ports: - "8000:8000" - env_file: .env + env_file: ../../.env environment: # Keep one canonical variable name; container uses docker-internal host. DATABASE_URL: postgresql://dev:dev@db:5432/opendevops @@ -50,7 +50,7 @@ services: # ── React frontend (production build served by nginx) ────────────────────── frontend: - build: ./frontend + build: ../../apps/frontend ports: - "80:8080" depends_on: diff --git a/Dockerfile.railway b/deployment/railway/Dockerfile.railway similarity index 88% rename from Dockerfile.railway rename to deployment/railway/Dockerfile.railway index 8c01828..f04f542 100644 --- a/Dockerfile.railway +++ b/deployment/railway/Dockerfile.railway @@ -2,9 +2,9 @@ FROM node:20-alpine AS frontend WORKDIR /frontend -COPY frontend/package*.json ./ +COPY apps/frontend/package*.json ./ RUN npm ci --silent -COPY frontend/ ./ +COPY apps/frontend/ ./ RUN npm run build # ── Stage 2: install Python dependencies ───────────────────────────────────── @@ -13,7 +13,7 @@ FROM python:3.12.9-slim-bookworm AS deps COPY --from=ghcr.io/astral-sh/uv:0.6.17 /uv /uvx /bin/ WORKDIR /app -COPY pyproject.toml uv.lock ./ +COPY apps/backend/pyproject.toml apps/backend/uv.lock ./ RUN uv sync --frozen --no-dev --no-install-project # ── Stage 3: final image ────────────────────────────────────────────────────── @@ -36,9 +36,9 @@ WORKDIR /app COPY --from=deps /app/.venv .venv COPY --from=frontend /frontend/dist frontend/dist -COPY src/ src/ -COPY migrations/ migrations/ -COPY scripts/ scripts/ +COPY apps/backend/src/ src/ +COPY apps/backend/migrations/ migrations/ +COPY apps/backend/scripts/ scripts/ RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --create-home --shell /usr/sbin/nologin app \ && mkdir -p /app/data \ diff --git a/railway.toml b/deployment/railway/railway.toml similarity index 61% rename from railway.toml rename to deployment/railway/railway.toml index adf4b51..61d3e45 100644 --- a/railway.toml +++ b/deployment/railway/railway.toml @@ -1,5 +1,5 @@ [build] -dockerfile = "Dockerfile.railway" +dockerfile = "deployment/railway/Dockerfile.railway" [deploy] healthcheckPath = "/openapi.json" diff --git a/src/cli/migrate.py b/src/cli/migrate.py deleted file mode 100644 index 6c69305..0000000 --- a/src/cli/migrate.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Run all SQL migrations in order against the configured PostgreSQL database.""" - -from __future__ import annotations - -import sys -from pathlib import Path - -import typer -from rich.console import Console - -console = Console() - - -def migrate_cmd() -> None: - """Apply all pending SQL migrations to the configured database.""" - from config import settings - - if settings.checkpoint_backend != "postgres" or not settings.database_url: - console.print( - "[bold red]migrate requires CHECKPOINT_BACKEND=postgres and DATABASE_URL to be set.[/bold red]" - ) - raise typer.Exit(1) - - try: - import psycopg - except ImportError: - console.print("[bold red]psycopg not installed. Run: uv add psycopg[binary,pool][/bold red]") - raise typer.Exit(1) - - migrations_dir = Path(__file__).parent.parent.parent / "migrations" - sql_files = sorted(migrations_dir.glob("*.sql")) - if not sql_files: - console.print("[yellow]No migration files found in migrations/[/yellow]") - raise typer.Exit(0) - - console.print(f"[bold]Running {len(sql_files)} migration(s) against database...[/bold]") - - try: - conn = psycopg.connect(settings.database_url, autocommit=True) - except Exception as e: - console.print(f"[bold red]Could not connect to database: {e}[/bold red]") - raise typer.Exit(1) - - with conn: - for path in sql_files: - sql = path.read_text(encoding="utf-8") - try: - conn.execute(sql) - console.print(f" [green]✓[/green] {path.name}") - except Exception as e: - console.print(f" [red]✗[/red] {path.name} — {e}") - conn.close() - raise typer.Exit(1) - - conn.close() - console.print("[bold green]All migrations applied successfully.[/bold green]") diff --git a/src/providers/aws/__init__.py b/src/providers/aws/__init__.py deleted file mode 100644 index 360f067..0000000 --- a/src/providers/aws/__init__.py +++ /dev/null @@ -1,47 +0,0 @@ -"""AWS cloud provider — CloudWatch, CloudTrail, ECS, Lambda, EC2, RDS, IAM.""" - -from __future__ import annotations - -from typing import Any - - -class AwsProvider: - name = "aws" - - def tools(self) -> list[Any]: - from providers.aws.tools.cloudtrail import ALL_CLOUDTRAIL_TOOLS - from providers.aws.tools.cloudwatch import ALL_CLOUDWATCH_TOOLS - from providers.aws.tools.ec2 import ALL_EC2_TOOLS - from providers.aws.tools.ecs import ALL_ECS_TOOLS - from providers.aws.tools.iam import ALL_IAM_TOOLS - from providers.aws.tools.lambda_ import ALL_LAMBDA_TOOLS - from providers.aws.tools.rds import ALL_RDS_TOOLS - - return ( - ALL_CLOUDWATCH_TOOLS - + ALL_CLOUDTRAIL_TOOLS - + ALL_ECS_TOOLS - + ALL_LAMBDA_TOOLS - + ALL_EC2_TOOLS - + ALL_RDS_TOOLS - + ALL_IAM_TOOLS - ) - - def collect_context(self, event: dict) -> dict: - from providers.aws.context import collect_context - return collect_context(event) - - def check_permissions(self, region: str | None) -> dict: - from providers.aws.permissions import check_permissions - return check_permissions(region) - - async def polling_loop(self) -> None: - from providers.aws.poller import polling_loop - await polling_loop() - - async def event_consumer_loop(self) -> None: - from providers.aws.event_consumer import event_consumer_loop - await event_consumer_loop() - - -__all__ = ["AwsProvider"]