From 236a1244f05c6830a585c1047a45fa1630d6c264 Mon Sep 17 00:00:00 2001 From: AhmadHammad21 Date: Tue, 12 May 2026 19:59:11 +0300 Subject: [PATCH 1/9] feat: event monitoring setup wizard + test event pipeline - Rewrites InitPage as a 4-step wizard: account creation, AWS region/SNS config, IAM permission check, and infrastructure creation with skip option - Adds Event Monitoring section to Settings AWS tab with Enabled/Disabled badge, Create Infrastructure and Teardown buttons, and live queue/rule count - Adds Send test event button (admin only) to Monitoring page header; sends a synthetic Lambda alarm to SQS to verify the end-to-end pipeline - Adds DELETE /api/init/infra (teardown) and POST /api/init/test-event endpoints to the init router - Updates docs/ui.md to document the wizard, event monitoring section, and test event button --- docs/ui.md | 13 +- frontend/src/pages/InitPage.tsx | 456 +++++++++++++++++++++++--- frontend/src/pages/MonitoringPage.tsx | 59 +++- frontend/src/pages/SettingsPage.tsx | 108 +++++- src/api/routers/init.py | 104 +++++- 5 files changed, 677 insertions(+), 63 deletions(-) diff --git a/docs/ui.md b/docs/ui.md index 8310e85..0e36db2 100644 --- a/docs/ui.md +++ b/docs/ui.md @@ -82,13 +82,17 @@ Shows the live incident feed produced by the event-driven consumer. See [`monito Accessible from the sidebar for all authenticated users. Each card shows the affected service, root cause summary, confidence level, and timestamp. Click any card to open the Alert Detail page, which includes an **Investigate** button that opens a new chat session with the incident context pre-loaded. +**Send test event** *(admin only)* — appears in the header next to the refresh button. Puts a synthetic Lambda error alarm into the SQS queue. A toast confirms dispatch and the resulting investigation card appears within ~30s via auto-refresh. Shows an error toast if event monitoring is not yet configured. + --- ## Settings page Shows active configuration. Four tabs (admins see all four; regular users see the last three): -**AWS Configuration** *(admin only, default tab for admins)* — editable fields for SNS Topic ARN, SQS Queue URL, and AWS Region. Values are saved to `data/init.json` server-side and shared across all admin users. Includes a **Run checks** button that validates IAM permissions for every AWS service the agent uses. +**AWS Configuration** *(admin only, default tab for admins)* — editable fields for SNS Topic ARN, SQS Queue URL, and AWS Region. Values are saved to `data/init.json` server-side and shared across all admin users. Includes: +- **Event Monitoring** section — shows Enabled / Disabled status; **Create Infrastructure** button provisions the SQS queue and 9 EventBridge rules; **Teardown** button removes them (with confirmation). +- **Run checks** button that validates IAM permissions for every AWS service the agent uses. **Environment** — read-only view of all `.env` values. Sensitive fields (`LLM_API_KEY`, `DATABASE_URL`, `JWT_SECRET`, etc.) are masked by default; click the eye icon to reveal. SNS and SQS values are intentionally omitted here — manage them in the AWS Configuration tab. @@ -100,7 +104,12 @@ Shows active configuration. Four tabs (admins see all four; regular users see th ## First-run setup page -Accessible at `/init` before any users exist. Creates the first admin account. After login, complete AWS setup in **Settings → AWS Configuration**. +Accessible at `/init` before any users exist. A 4-step wizard guides the admin through initial setup: + +1. **Create account** — email + password for the admin user +2. **AWS Configuration** — region selector and optional SNS Topic ARN for alert delivery +3. **Permission check** — verifies IAM permissions per service; required services (CloudWatch, Lambda, SQS, EventBridge) are highlighted; "Continue anyway" is always available +4. **Enable event monitoring** — creates the SQS queue and 9 EventBridge rules; shows a success summary with queue URL and rule list; includes a **Skip** option with a disclaimer listing what won't work without it The page redirects away automatically once an admin account exists. diff --git a/frontend/src/pages/InitPage.tsx b/frontend/src/pages/InitPage.tsx index 27b6e84..15a25ba 100644 --- a/frontend/src/pages/InitPage.tsx +++ b/frontend/src/pages/InitPage.tsx @@ -1,16 +1,70 @@ import { useState, useEffect } from 'react'; import { useNavigate } from 'react-router-dom'; -import { ArrowRight, Loader2, Terminal, Eye, EyeOff } from 'lucide-react'; +import { ArrowRight, Loader2, Terminal, Eye, EyeOff, CheckCircle, XCircle, AlertTriangle, ChevronRight, Check } from 'lucide-react'; +import { cn } from '../lib/utils'; const inp = 'w-full text-base text-gray-900 dark:text-white bg-gray-50 dark:bg-[#18181B] border border-gray-200 dark:border-[#27272A] rounded-xl px-4 py-3.5 outline-none focus:border-indigo-500 dark:focus:border-indigo-400 focus:ring-2 focus:ring-indigo-500/10 transition-all placeholder:text-gray-400 dark:placeholder:text-[#52525B]'; +const AWS_REGIONS = [ + 'us-east-1', 'us-east-2', 'us-west-1', 'us-west-2', + 'eu-west-1', 'eu-west-2', 'eu-west-3', 'eu-central-1', + 'ap-southeast-1', 'ap-southeast-2', 'ap-northeast-1', 'ap-south-1', + 'ca-central-1', 'sa-east-1', +]; + +const SVC: Record = { + cloudwatch: { label: 'CloudWatch', desc: 'Alarms, metrics, logs', required: true }, + lambda: { label: 'Lambda', desc: 'Serverless functions', required: true }, + sqs: { label: 'SQS', desc: 'Event queue', required: true }, + events: { label: 'EventBridge', desc: 'Event rules', required: true }, + ecs: { label: 'ECS', desc: 'Container services' }, + rds: { label: 'RDS', desc: 'Database events' }, + ec2: { label: 'EC2', desc: 'Instance status' }, + iam: { label: 'IAM / STS', desc: 'Identity & access' }, + sns: { label: 'SNS', desc: 'Alert publishing' }, + cloudtrail: { label: 'CloudTrail', desc: 'API event history' }, +}; + +const RULES = [ + { name: 'CloudWatch Alarm State Change', src: 'aws.cloudwatch' }, + { name: 'Lambda Invocation Failure', src: 'aws.lambda' }, + { name: 'Lambda Throttling', src: 'aws.lambda' }, + { name: 'ECS Task Stopped (non-zero exit)', src: 'aws.ecs' }, + { name: 'EC2 Instance Terminated', src: 'aws.ec2' }, + { name: 'RDS DB Events (failure / failover)', src: 'aws.rds' }, + { name: 'AWS Health Event', src: 'aws.health' }, + { name: 'CodeDeploy Deployment Failure', src: 'aws.codedeploy' }, + { name: 'GuardDuty Finding', src: 'aws.guardduty' }, +]; + +interface PermResult { passed: boolean | null; error: string | null } + +type Step = 1 | 2 | 3 | 4; + export default function InitPage() { const navigate = useNavigate(); - const [loading, setLoading] = useState(false); + + const [step, setStep] = useState(1); + + // Step 1 const [username, setUsername] = useState('admin'); const [password, setPassword] = useState(''); - const [showPw, setShowPw] = useState(false); - const [error, setError] = useState(''); + const [showPw, setShowPw] = useState(false); + + // Step 2 + const [region, setRegion] = useState('us-east-1'); + const [snsArn, setSnsArn] = useState(''); + + // Step 3 + const [perms, setPerms] = useState>({}); + + // Step 4 + const [infraDone, setInfraDone] = useState(false); + const [queueUrl, setQueueUrl] = useState(''); + const [showSkipDisclaimer, setShowSkipDisclaimer] = useState(false); + + const [loading, setLoading] = useState(false); + const [error, setError] = useState(''); useEffect(() => { fetch('/api/init/status') @@ -19,20 +73,75 @@ export default function InitPage() { .catch(() => {}); }, [navigate]); + // ─── Step 1: create admin account ───────────────────────────────────────── const createUser = async () => { if (!username.trim() || !password.trim()) { setError('Both fields are required'); return; } setLoading(true); setError(''); - const r = await fetch('/api/init/create-user', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ username: username.trim(), password }), - }); - const data = await r.json(); - setLoading(false); - if (data.error) { setError(data.error); return; } - navigate('/', { replace: true }); + try { + const r = await fetch('/api/init/create-user', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ username: username.trim(), password }), + }); + const data = await r.json(); + if (data.error) { setError(data.error); return; } + setStep(2); + } finally { + setLoading(false); + } + }; + + // ─── Step 2: save AWS config ─────────────────────────────────────────────── + const saveConfig = async () => { + setLoading(true); setError(''); + try { + const r = await fetch('/api/init/setup', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ aws_region: region, sns_topic_arn: snsArn, sqs_queue_url: '' }), + }); + if (!r.ok) { setError('Failed to save configuration'); return; } + setStep(3); + } finally { + setLoading(false); + } + }; + + // ─── Step 3: check permissions ───────────────────────────────────────────── + const checkPerms = async () => { + setLoading(true); setError(''); + try { + const r = await fetch('/api/init/check-permissions', { method: 'POST' }); + const d = await r.json() as { permissions: Record }; + setPerms(d.permissions); + } catch { + setError('Permission check failed'); + } finally { + setLoading(false); + } + }; + + // ─── Step 4: create infrastructure ──────────────────────────────────────── + const createInfra = async () => { + setLoading(true); setError(''); + try { + const r = await fetch('/api/init/complete', { method: 'POST' }); + const d = await r.json() as { initialized: boolean; error: string | null }; + if (!d.initialized) { setError(d.error || 'Infrastructure setup failed'); return; } + setInfraDone(true); + // Fetch the queue URL from init status to display in summary + const s = await fetch('/api/init/status').then(res => res.json()); + setQueueUrl(s.sqs_queue_url || 'opendevops-agent-events'); + } finally { + setLoading(false); + } }; + const requiredPassed = Object.entries(perms) + .filter(([svc]) => SVC[svc]?.required) + .every(([, r]) => r.passed); + const hasPerms = Object.keys(perms).length > 0; + return (
@@ -43,46 +152,297 @@ export default function InitPage() {
-
-

Create your account

-

- Set up the admin credentials you'll use to access the dashboard. -

- -
-
- - setUsername(e.target.value)} placeholder="admin" className={inp} /> -
-
- -
- setPassword(e.target.value)} - onKeyDown={e => e.key === 'Enter' && createUser()} - placeholder="At least 6 characters" - className={inp + ' pr-11'} - /> - +
+ + {/* Step indicator */} +
+ {([1, 2, 3, 4] as Step[]).map((s, i) => ( +
+
s + ? 'bg-indigo-500 text-white' + : step === s + ? 'bg-indigo-500 text-white ring-4 ring-indigo-500/20' + : 'bg-gray-100 dark:bg-[#27272A] text-gray-400 dark:text-[#52525B]' + )}> + {step > s ? : s} +
+ {i < 3 && ( +
s ? 'bg-indigo-500' : 'bg-gray-200 dark:bg-[#27272A]' + )} /> + )}
-
+ ))} + Step {step} of 4
- {error &&

{error}

} + {/* ─── Step 1: Create Account ─────────────────────────────────── */} + {step === 1 && ( + <> +

Create your account

+

+ Set up the admin credentials you'll use to access the dashboard. +

+
+
+ + setUsername(e.target.value)} placeholder="admin" className={inp} /> +
+
+ +
+ setPassword(e.target.value)} + onKeyDown={e => e.key === 'Enter' && createUser()} + placeholder="At least 6 characters" + className={inp + ' pr-11'} + /> + +
+
+
+ {error &&

{error}

} + + + )} + + {/* ─── Step 2: AWS Configuration ──────────────────────────────── */} + {step === 2 && ( + <> +

AWS Configuration

+

+ Configure the AWS region and optional SNS topic for alert delivery. +

+
+
+ + +
+
+ + setSnsArn(e.target.value)} + placeholder="arn:aws:sns:us-east-1:123456789012:alerts" + className={inp} + /> +

+ Leave empty to skip SNS notifications — alerts will still appear in the monitoring page. +

+
+
+ {error &&

{error}

} + + + + )} + + {/* ─── Step 3: Permission Check ────────────────────────────────── */} + {step === 3 && ( + <> +

Check permissions

+

+ Verify that the IAM role has the required permissions for each AWS service. +

+ + {!hasPerms ? ( + + ) : ( + <> +
+ {Object.entries(SVC).map(([svc, info], i) => { + const r = perms[svc]; + return ( +
+
+ {!r + ?
+ : r.passed + ? + : r.passed === null + ? + : + } +
+
+
+ {info.label} + {info.desc} + {info.required && ( + Required + )} +
+ {r && !r.passed && r.error && ( +

{r.error}

+ )} +
+
+ ); + })} +
+ + {!requiredPassed && ( +
+ Some required permissions are missing. Event monitoring may not work correctly. + You can still continue — fix the IAM permissions and re-check in Settings. +
+ )} + + {error &&

{error}

} + + + + + + )} + + )} + + {/* ─── Step 4: Enable Event Monitoring ────────────────────────── */} + {step === 4 && ( + <> +

Enable event monitoring

+

+ Create the SQS queue and 9 EventBridge rules that feed incidents into the investigation pipeline. +

+ + {!infraDone ? ( + <> + {/* Rule list preview */} +
+ {RULES.map((rule, i) => ( +
+
+
+ {rule.name} + {rule.src} +
+
+ ))} +
+ + {error &&

{error}

} + + + + {/* Skip option */} + + + {showSkipDisclaimer && ( +
+

Without event monitoring:

+
    + {[ + 'Automatic incident detection', + 'Event-triggered investigations', + 'Slack / SNS alert delivery', + 'Monitoring page will be empty', + ].map(item => ( +
  • + {item} +
    • + ))} +
+

+ You can enable this anytime from Settings → AWS Configuration. +

+ +
+ )} + + ) : ( + // Success summary + <> +
+
+ + Infrastructure created successfully +
+
+

✓ SQS queue: {queueUrl || 'opendevops-agent-events'}

+

✓ {RULES.length} EventBridge rules active

+
+
+ +
+ {RULES.map((rule, i) => ( +
+ + {rule.name} +
+ ))} +
- + + + + )} + + )} -

- AWS configuration can be set up in Settings → AWS Configuration after login. -

diff --git a/frontend/src/pages/MonitoringPage.tsx b/frontend/src/pages/MonitoringPage.tsx index 4af15be..457a96b 100644 --- a/frontend/src/pages/MonitoringPage.tsx +++ b/frontend/src/pages/MonitoringPage.tsx @@ -1,8 +1,10 @@ import { useEffect, useState } from 'react'; import { useNavigate } from 'react-router-dom'; -import { Activity, CheckCircle, XCircle, AlertTriangle, RefreshCw, Radio, MessageSquare, ChevronRight } from 'lucide-react'; +import { Activity, CheckCircle, XCircle, AlertTriangle, RefreshCw, Radio, MessageSquare, ChevronRight, FlaskConical } from 'lucide-react'; +import { toast } from 'sonner'; import { cn } from '../lib/utils'; -import { fetchAlerts, fetchServices } from '../lib/api'; +import { fetchAlerts, fetchServices, apiFetch } from '../lib/api'; +import { useAuth } from '../context/AuthContext'; import type { Alert, ServiceStatus } from '../types'; function timeAgo(ts: string): string { @@ -21,9 +23,45 @@ function buildPrompt(a: Alert): string { export default function MonitoringPage() { const navigate = useNavigate(); - const [services, setServices] = useState([]); - const [alerts, setAlerts] = useState([]); - const [loading, setLoading] = useState(true); + const { isAdmin } = useAuth(); + const [services, setServices] = useState([]); + const [alerts, setAlerts] = useState([]); + const [loading, setLoading] = useState(true); + const [testSending, setTestSending] = useState(false); + const [infraEnabled, setInfraEnabled] = useState(null); + + useEffect(() => { + if (!isAdmin) return; + fetch('/api/init/status') + .then(r => r.json()) + .then(d => setInfraEnabled(!!d.initialized)) + .catch(() => setInfraEnabled(false)); + }, [isAdmin]); + + const sendTestEvent = async () => { + if (infraEnabled === false) { + toast.error('Event monitoring not set up — go to Settings → AWS Configuration to enable it'); + return; + } + setTestSending(true); + try { + const r = await apiFetch('/api/init/test-event', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ service: 'lambda' }), + }); + if (!r.ok) { + const d = await r.json(); + toast.error(d.detail || 'Failed to send test event'); + return; + } + toast.success('Test Lambda alarm sent — investigation will appear here in ~30s'); + } catch { + toast.error('Failed to send test event'); + } finally { + setTestSending(false); + } + }; const load = () => { setLoading(true); @@ -58,6 +96,17 @@ export default function MonitoringPage() { Live + {isAdmin && ( + + )} + ) : ( + + )} + +
+ {infraEnabled ? ( +
+
+ Queue URL + {sqsUrl || '—'} +
+
+ Rules + {Object.keys(infraRules).length || 9} EventBridge rules active +
+
+ ) : ( +

+ Creates an SQS queue and 9 EventBridge rules to automatically detect and investigate AWS incidents. +

+ )} +
+ + {/* Permissions card */}
diff --git a/src/api/routers/init.py b/src/api/routers/init.py index b39a79d..445583c 100644 --- a/src/api/routers/init.py +++ b/src/api/routers/init.py @@ -3,14 +3,18 @@ from __future__ import annotations import asyncio +import datetime +import json -from fastapi import APIRouter +import boto3 +from fastapi import APIRouter, HTTPException from loguru import logger from pydantic import BaseModel from agent.db import db from agent.init_store import load_init, save_init from api.auth import hash_password +from config import settings router = APIRouter(prefix="/api/init", tags=["init"]) @@ -114,3 +118,101 @@ async def complete(): logger.warning("Could not hot-start event consumer (will start on next restart): {}", e) return {"initialized": True, "error": None} + + +@router.delete("/infra") +async def teardown_infra(): + """Remove SQS queue and EventBridge rules created by /complete.""" + data = load_init() + queue_url = data.get("sqs_queue_url") or settings.sqs_queue_url + rule_arns = data.get("eventbridge_rule_arns") or {} + + if not queue_url: + raise HTTPException( + status_code=400, + detail="No SQS queue URL found — infrastructure may not be set up", + ) + + try: + from agent.event_infra import teardown_event_infra + await asyncio.get_event_loop().run_in_executor( + None, teardown_event_infra, queue_url, rule_arns + ) + except Exception as e: + logger.error("Teardown failed: {}", e) + raise HTTPException(status_code=500, detail=str(e)) + + data["initialized"] = False + data["sqs_queue_url"] = "" + data["sqs_queue_arn"] = "" + data["eventbridge_rule_arns"] = {} + save_init(data) + logger.info("Init: event infra torn down") + return {"torn_down": True} + + +class TestEventBody(BaseModel): + service: str = "lambda" + + +@router.post("/test-event") +async def send_test_event(body: TestEventBody): + """Put a synthetic event into the SQS queue to verify the pipeline end-to-end.""" + data = load_init() + queue_url = data.get("sqs_queue_url") or settings.sqs_queue_url + if not queue_url: + raise HTTPException( + status_code=400, + detail=( + "Event monitoring not configured — set up the SQS queue in" + " Settings → AWS Configuration first" + ), + ) + + now = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + event: dict = { + "source": "aws.cloudwatch", + "detail-type": "CloudWatch Alarm State Change", + "time": now, + "detail": { + "alarmName": "opendevops-test-lambda-errors", + "state": { + "value": "ALARM", + "reason": ( + "Threshold Crossed: 5 datapoints were greater than or equal to the" + " threshold (1.0). The most recent datapoints: [5.0, 3.0, 2.0, 1.0, 1.0]." + ), + }, + "configuration": { + "metrics": [{ + "metricStat": { + "metric": { + "namespace": "AWS/Lambda", + "name": "Errors", + "dimensions": {}, + }, + "period": 300, + "stat": "Sum", + } + }] + }, + }, + } + + try: + s = ( + boto3.Session(profile_name=settings.aws_profile) + if settings.aws_profile + else boto3.Session() + ) + sqs = s.client("sqs", region_name=settings.aws_region) + resp = await asyncio.get_event_loop().run_in_executor( + None, + lambda: sqs.send_message(QueueUrl=queue_url, MessageBody=json.dumps(event)), + ) + message_id = resp.get("MessageId", "") + logger.info("Test event sent to SQS: message_id={}", message_id) + return {"message_id": message_id, "queue_url": queue_url} + except Exception as e: + logger.error("Failed to send test event: {}", e) + raise HTTPException(status_code=500, detail=str(e)) From 5812b5a868d1d304ea55eae78c0b849578e0545e Mon Sep 17 00:00:00 2001 From: AhmadHammad21 Date: Tue, 12 May 2026 20:46:35 +0300 Subject: [PATCH 2/9] =?UTF-8?q?feat:=20add=20test=5Fpipeline.sh=20?= =?UTF-8?q?=E2=80=94=20real=20Lambda=20errors=20through=20monitoring=20pip?= =?UTF-8?q?eline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/test_pipeline.sh | 195 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 scripts/test_pipeline.sh diff --git a/scripts/test_pipeline.sh b/scripts/test_pipeline.sh new file mode 100644 index 0000000..3148843 --- /dev/null +++ b/scripts/test_pipeline.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash +# test_pipeline.sh — spin up a real Lambda that errors, push it through the monitoring pipeline. +# +# Usage: +# bash scripts/test_pipeline.sh # run test +# bash scripts/test_pipeline.sh --region=eu-west-1 # override region +# bash scripts/test_pipeline.sh --invocations=10 # more errors +# bash scripts/test_pipeline.sh --cleanup FUNCTION ROLE # tear down after test + +set -euo pipefail + +# ── colours ────────────────────────────────────────────────────────────────── +GREEN='\033[0;32m'; YELLOW='\033[1;33m'; RED='\033[0;31m'; NC='\033[0m' +log() { echo -e "${GREEN}[+]${NC} $*"; } +warn() { echo -e "${YELLOW}[!]${NC} $*"; } +die() { echo -e "${RED}[✗]${NC} $*" >&2; exit 1; } + +# ── cleanup mode ───────────────────────────────────────────────────────────── +if [[ "${1:-}" == "--cleanup" ]]; then + FUNCTION_NAME="${2:-}"; ROLE_NAME="${3:-}" + [[ -z "$FUNCTION_NAME" || -z "$ROLE_NAME" ]] && die "Usage: $0 --cleanup FUNCTION_NAME ROLE_NAME" + REGION="${AWS_REGION:-us-east-1}" + warn "Deleting Lambda: $FUNCTION_NAME" + aws lambda delete-function --function-name "$FUNCTION_NAME" --region "$REGION" 2>/dev/null && log "Lambda deleted" || warn "Already gone" + warn "Deleting IAM role: $ROLE_NAME" + aws iam detach-role-policy --role-name "$ROLE_NAME" \ + --policy-arn "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" 2>/dev/null || true + aws iam delete-role --role-name "$ROLE_NAME" 2>/dev/null && log "Role deleted" || warn "Already gone" + log "Cleanup done." + exit 0 +fi + +# ── config ──────────────────────────────────────────────────────────────────── +REGION="${AWS_REGION:-us-east-1}" +INVOCATIONS=5 + +for arg in "$@"; do + case $arg in + --region=*) REGION="${arg#*=}" ;; + --invocations=*) INVOCATIONS="${arg#*=}" ;; + esac +done + +SUFFIX=$(date +%s) +FUNCTION_NAME="opendevops-test-error-${SUFFIX}" +ROLE_NAME="opendevops-test-role-${SUFFIX}" + +# ── find SQS queue URL ──────────────────────────────────────────────────────── +QUEUE_URL="" +if [[ -f "data/init.json" ]]; then + QUEUE_URL=$(python3 -c "import json; d=json.load(open('data/init.json')); print(d.get('sqs_queue_url',''))" 2>/dev/null || echo "") +fi +if [[ -z "$QUEUE_URL" ]]; then + QUEUE_URL="${SQS_QUEUE_URL:-}" +fi +if [[ -z "$QUEUE_URL" ]]; then + die "No SQS queue URL found. Run 'Create Infrastructure' in Settings → AWS Configuration first." +fi + +log "Region : $REGION" +log "Queue : $QUEUE_URL" +log "Function : $FUNCTION_NAME" +echo "" + +# ── create IAM role ──────────────────────────────────────────────────────────── +log "Creating IAM execution role..." +TRUST_POLICY='{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"lambda.amazonaws.com"},"Action":"sts:AssumeRole"}]}' + +ROLE_ARN=$(aws iam create-role \ + --role-name "$ROLE_NAME" \ + --assume-role-policy-document "$TRUST_POLICY" \ + --query 'Role.Arn' \ + --output text) + +aws iam attach-role-policy \ + --role-name "$ROLE_NAME" \ + --policy-arn "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + +log "Role ARN : $ROLE_ARN" +log "Waiting 10s for IAM role to propagate..." +sleep 10 + +# ── create Lambda ZIP ───────────────────────────────────────────────────────── +TMP_DIR=$(mktemp -d) +trap 'rm -rf "$TMP_DIR"' EXIT + +cat > "$TMP_DIR/index.py" << 'PYEOF' +import random + +ERRORS = [ + "Connection pool exhausted: all 100 connections in use by active queries", + "DynamoDB ProvisionedThroughputExceededException on table 'orders' — read capacity 0/500 RCU remaining", + "Downstream service timeout: payments-service did not respond within 29000ms", + "Redis READONLY error: connected to replica — write commands not accepted", + "JWT verification failed: token expired 3601s ago (iat=1715000000)", + "Unhandled exception: Cannot read property 'userId' of undefined at processOrder:47", + "Out of memory: Lambda allocated 128 MB, peak usage 129 MB — increase memory limit", +] + +def handler(event, context): + error = random.choice(ERRORS) + raise RuntimeError(f"[TEST FAILURE] {error}") +PYEOF + +(cd "$TMP_DIR" && zip -q function.zip index.py) + +# ── deploy Lambda ───────────────────────────────────────────────────────────── +log "Deploying Lambda function..." +aws lambda create-function \ + --function-name "$FUNCTION_NAME" \ + --runtime python3.12 \ + --role "$ROLE_ARN" \ + --handler index.handler \ + --zip-file "fileb://$TMP_DIR/function.zip" \ + --region "$REGION" \ + --timeout 10 \ + --output text > /dev/null + +aws lambda wait function-active \ + --function-name "$FUNCTION_NAME" \ + --region "$REGION" + +log "Lambda deployed." + +# ── invoke it to generate real errors + CloudWatch logs ─────────────────────── +log "Invoking $INVOCATIONS times to generate real errors..." +for i in $(seq 1 "$INVOCATIONS"); do + aws lambda invoke \ + --function-name "$FUNCTION_NAME" \ + --payload '{"test": true}' \ + --invocation-type RequestResponse \ + --region "$REGION" \ + /tmp/lambda-out.json > /dev/null 2>&1 || true + printf " invocation %d/%d\n" "$i" "$INVOCATIONS" +done + +log "Errors written to CloudWatch Logs: /aws/lambda/$FUNCTION_NAME" +echo "" + +# ── push real event to SQS ──────────────────────────────────────────────────── +# Uses the real function name so the agent can look up actual CloudWatch logs. +NOW=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + +EVENT=$(cat << EOF +{ + "source": "aws.cloudwatch", + "detail-type": "CloudWatch Alarm State Change", + "time": "${NOW}", + "region": "${REGION}", + "detail": { + "alarmName": "${FUNCTION_NAME}-errors", + "state": { + "value": "ALARM", + "reason": "Threshold Crossed: ${INVOCATIONS} error datapoints in the last 1 minute (threshold: 1)." + }, + "configuration": { + "metrics": [{ + "metricStat": { + "metric": { + "namespace": "AWS/Lambda", + "name": "Errors", + "dimensions": {"FunctionName": "${FUNCTION_NAME}"} + }, + "period": 60, + "stat": "Sum" + } + }] + } + } +} +EOF +) + +log "Sending event to SQS..." +MSG_ID=$(aws sqs send-message \ + --queue-url "$QUEUE_URL" \ + --message-body "$EVENT" \ + --region "$REGION" \ + --query 'MessageId' \ + --output text) + +echo "" +echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" +echo -e "${GREEN}✓ Pipeline triggered${NC}" +echo " MessageId : $MSG_ID" +echo " Function : $FUNCTION_NAME" +echo " Real logs : CloudWatch → /aws/lambda/$FUNCTION_NAME" +echo " Monitoring: http://localhost/monitoring" +echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" +echo "" +warn "The agent will pull REAL CloudWatch logs from that function." +warn "Investigation takes ~30–90s. Watch the Monitoring page." +echo "" +echo -e "To clean up when done:" +echo -e " ${YELLOW}bash scripts/test_pipeline.sh --cleanup $FUNCTION_NAME $ROLE_NAME${NC}" From 7ae2b5f45527d8f59f43f11e455c8acdb656c365 Mon Sep 17 00:00:00 2001 From: AhmadHammad21 Date: Tue, 12 May 2026 21:20:48 +0300 Subject: [PATCH 3/9] =?UTF-8?q?feat:=20add=20test=5Fpipeline.py=20?= =?UTF-8?q?=E2=80=94=20boto3-based=20pipeline=20test=20for=20Windows=20com?= =?UTF-8?q?patibility?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/test_pipeline.py | 237 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 scripts/test_pipeline.py diff --git a/scripts/test_pipeline.py b/scripts/test_pipeline.py new file mode 100644 index 0000000..5e1fed8 --- /dev/null +++ b/scripts/test_pipeline.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +""" +test_pipeline.py — create a real Lambda that errors, fire it through the monitoring pipeline. + +Usage: + uv run python scripts/test_pipeline.py # run test + uv run python scripts/test_pipeline.py --region eu-west-1 # override region + uv run python scripts/test_pipeline.py --invocations 10 # more errors + uv run python scripts/test_pipeline.py --cleanup FUNCTION ROLE +""" + +from __future__ import annotations + +import argparse +import datetime +import io +import json +import os +import sys +import time +import zipfile + +import boto3 +from botocore.exceptions import ClientError + +GREEN = "\033[0;32m" +YELLOW = "\033[1;33m" +RED = "\033[0;31m" +NC = "\033[0m" + + +def log(msg: str) -> None: print(f"{GREEN}[+]{NC} {msg}") +def warn(msg: str) -> None: print(f"{YELLOW}[!]{NC} {msg}") +def die(msg: str) -> None: print(f"{RED}[✗]{NC} {msg}", file=sys.stderr); sys.exit(1) + + +LAMBDA_CODE = """\ +import random + +ERRORS = [ + "Connection pool exhausted: all 100 connections in use by active queries", + "DynamoDB ProvisionedThroughputExceededException on table 'orders' — read capacity 0/500 RCU remaining", + "Downstream service timeout: payments-service did not respond within 29000ms", + "Redis READONLY error: connected to replica — write commands not accepted", + "JWT verification failed: token expired 3601s ago (iat=1715000000)", + "Unhandled exception: Cannot read property 'userId' of undefined at processOrder:47", + "Out of memory: Lambda allocated 128 MB, peak usage 129 MB — increase memory limit", +] + +def handler(event, context): + error = random.choice(ERRORS) + raise RuntimeError(f"[TEST FAILURE] {error}") +""" + + +def make_zip() -> bytes: + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: + zf.writestr("index.py", LAMBDA_CODE) + return buf.getvalue() + + +def get_queue_url() -> str: + init_path = os.path.join(os.path.dirname(__file__), "..", "data", "init.json") + if os.path.exists(init_path): + with open(init_path) as f: + url = json.load(f).get("sqs_queue_url", "") + if url: + return url + return os.environ.get("SQS_QUEUE_URL", "") + + +def cleanup(function_name: str, role_name: str, region: str) -> None: + session = boto3.Session() + lam = session.client("lambda", region_name=region) + iam = session.client("iam") + + warn(f"Deleting Lambda: {function_name}") + try: + lam.delete_function(FunctionName=function_name) + log("Lambda deleted") + except ClientError: + warn("Already gone") + + warn(f"Deleting IAM role: {role_name}") + try: + iam.detach_role_policy( + RoleName=role_name, + PolicyArn="arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ) + except ClientError: + pass + try: + iam.delete_role(RoleName=role_name) + log("Role deleted") + except ClientError: + warn("Already gone") + + log("Cleanup done.") + + +def run(region: str, invocations: int) -> None: + queue_url = get_queue_url() + if not queue_url: + die("No SQS queue URL found. Run 'Create Infrastructure' in Settings → AWS Configuration first.") + + suffix = int(time.time()) + function_name = f"opendevops-test-error-{suffix}" + role_name = f"opendevops-test-role-{suffix}" + + log(f"Region : {region}") + log(f"Queue : {queue_url}") + log(f"Function : {function_name}") + print() + + session = boto3.Session() + iam = session.client("iam") + lam = session.client("lambda", region_name=region) + sqs = session.client("sqs", region_name=region) + + # ── create IAM role ────────────────────────────────────────────────────── + log("Creating IAM execution role...") + trust = json.dumps({ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": {"Service": "lambda.amazonaws.com"}, + "Action": "sts:AssumeRole", + }], + }) + role_arn = iam.create_role( + RoleName=role_name, + AssumeRolePolicyDocument=trust, + )["Role"]["Arn"] + iam.attach_role_policy( + RoleName=role_name, + PolicyArn="arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ) + log(f"Role ARN : {role_arn}") + log("Waiting 10s for IAM role to propagate...") + time.sleep(10) + + # ── deploy Lambda ──────────────────────────────────────────────────────── + log("Deploying Lambda function...") + lam.create_function( + FunctionName=function_name, + Runtime="python3.12", + Role=role_arn, + Handler="index.handler", + Code={"ZipFile": make_zip()}, + Timeout=10, + ) + + waiter = lam.get_waiter("function_active") + waiter.wait(FunctionName=function_name) + log("Lambda deployed.") + + # ── invoke to generate real errors + CloudWatch logs ──────────────────── + log(f"Invoking {invocations} times to generate real errors...") + for i in range(1, invocations + 1): + lam.invoke( + FunctionName=function_name, + InvocationType="RequestResponse", + Payload=b'{"test": true}', + ) + print(f" invocation {i}/{invocations}") + + log(f"Errors written to CloudWatch Logs: /aws/lambda/{function_name}") + print() + + # ── push event to SQS with real function name ──────────────────────────── + now = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + event = { + "source": "aws.cloudwatch", + "detail-type": "CloudWatch Alarm State Change", + "time": now, + "region": region, + "detail": { + "alarmName": f"{function_name}-errors", + "state": { + "value": "ALARM", + "reason": ( + f"Threshold Crossed: {invocations} error datapoints" + " in the last 1 minute (threshold: 1)." + ), + }, + "configuration": { + "metrics": [{ + "metricStat": { + "metric": { + "namespace": "AWS/Lambda", + "name": "Errors", + "dimensions": {"FunctionName": function_name}, + }, + "period": 60, + "stat": "Sum", + }, + }], + }, + }, + } + + log("Sending event to SQS...") + msg_id = sqs.send_message( + QueueUrl=queue_url, + MessageBody=json.dumps(event), + )["MessageId"] + + sep = "━" * 60 + print(f"\n{GREEN}{sep}{NC}") + print(f"{GREEN}✓ Pipeline triggered{NC}") + print(f" MessageId : {msg_id}") + print(f" Function : {function_name}") + print(f" Real logs : CloudWatch → /aws/lambda/{function_name}") + print(f" Monitoring: http://localhost/monitoring") + print(f"{GREEN}{sep}{NC}\n") + warn("The agent will pull REAL CloudWatch logs from that function.") + warn("Investigation takes ~30–90s. Watch the Monitoring page.") + print(f"\nTo clean up when done:") + print(f" {YELLOW}uv run python scripts/test_pipeline.py --cleanup {function_name} {role_name}{NC}") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Fire a real Lambda failure through the monitoring pipeline") + parser.add_argument("--cleanup", nargs=2, metavar=("FUNCTION", "ROLE"), help="Tear down resources") + parser.add_argument("--region", default=os.environ.get("AWS_REGION", "us-east-1")) + parser.add_argument("--invocations", type=int, default=5) + args = parser.parse_args() + + if args.cleanup: + cleanup(args.cleanup[0], args.cleanup[1], args.region) + else: + run(args.region, args.invocations) + + +if __name__ == "__main__": + main() From b7f6255b21e88bac82786047d8891b13b06f41a5 Mon Sep 17 00:00:00 2001 From: AhmadHammad21 Date: Tue, 12 May 2026 21:22:38 +0300 Subject: [PATCH 4/9] fix: rewrite test_pipeline.py to use existing Lambda, no IAM creation needed --- scripts/test_pipeline.py | 248 +++++++++++++++++---------------------- 1 file changed, 107 insertions(+), 141 deletions(-) diff --git a/scripts/test_pipeline.py b/scripts/test_pipeline.py index 5e1fed8..cc27d37 100644 --- a/scripts/test_pipeline.py +++ b/scripts/test_pipeline.py @@ -1,32 +1,34 @@ #!/usr/bin/env python3 """ -test_pipeline.py — create a real Lambda that errors, fire it through the monitoring pipeline. +test_pipeline.py — trigger a real Lambda failure through the monitoring pipeline. + +No IAM role creation needed — picks an existing Lambda from your account, +invokes it with a bad payload to generate real CloudWatch Logs errors, +then pushes an alarm event to SQS so the agent investigates. Usage: - uv run python scripts/test_pipeline.py # run test - uv run python scripts/test_pipeline.py --region eu-west-1 # override region - uv run python scripts/test_pipeline.py --invocations 10 # more errors - uv run python scripts/test_pipeline.py --cleanup FUNCTION ROLE + uv run python scripts/test_pipeline.py # auto-pick a Lambda + uv run python scripts/test_pipeline.py --function my-function # use specific Lambda + uv run python scripts/test_pipeline.py --region eu-west-1 + uv run python scripts/test_pipeline.py --invocations 10 + uv run python scripts/test_pipeline.py --list # list available Lambdas """ from __future__ import annotations import argparse import datetime -import io import json import os import sys -import time -import zipfile import boto3 -from botocore.exceptions import ClientError +from botocore.exceptions import ClientError, NoCredentialsError -GREEN = "\033[0;32m" +GREEN = "\033[0;32m" YELLOW = "\033[1;33m" -RED = "\033[0;31m" -NC = "\033[0m" +RED = "\033[0;31m" +NC = "\033[0m" def log(msg: str) -> None: print(f"{GREEN}[+]{NC} {msg}") @@ -34,141 +36,105 @@ def warn(msg: str) -> None: print(f"{YELLOW}[!]{NC} {msg}") def die(msg: str) -> None: print(f"{RED}[✗]{NC} {msg}", file=sys.stderr); sys.exit(1) -LAMBDA_CODE = """\ -import random - -ERRORS = [ - "Connection pool exhausted: all 100 connections in use by active queries", - "DynamoDB ProvisionedThroughputExceededException on table 'orders' — read capacity 0/500 RCU remaining", - "Downstream service timeout: payments-service did not respond within 29000ms", - "Redis READONLY error: connected to replica — write commands not accepted", - "JWT verification failed: token expired 3601s ago (iat=1715000000)", - "Unhandled exception: Cannot read property 'userId' of undefined at processOrder:47", - "Out of memory: Lambda allocated 128 MB, peak usage 129 MB — increase memory limit", -] - -def handler(event, context): - error = random.choice(ERRORS) - raise RuntimeError(f"[TEST FAILURE] {error}") -""" - - -def make_zip() -> bytes: - buf = io.BytesIO() - with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: - zf.writestr("index.py", LAMBDA_CODE) - return buf.getvalue() - - def get_queue_url() -> str: init_path = os.path.join(os.path.dirname(__file__), "..", "data", "init.json") if os.path.exists(init_path): - with open(init_path) as f: - url = json.load(f).get("sqs_queue_url", "") - if url: - return url + try: + with open(init_path) as f: + url = json.load(f).get("sqs_queue_url", "") + if url: + return url + except Exception: + pass return os.environ.get("SQS_QUEUE_URL", "") -def cleanup(function_name: str, role_name: str, region: str) -> None: - session = boto3.Session() - lam = session.client("lambda", region_name=region) - iam = session.client("iam") +def list_functions(lam) -> list[str]: + names: list[str] = [] + paginator = lam.get_paginator("list_functions") + for page in paginator.paginate(): + for fn in page.get("Functions", []): + names.append(fn["FunctionName"]) + return sorted(names) - warn(f"Deleting Lambda: {function_name}") - try: - lam.delete_function(FunctionName=function_name) - log("Lambda deleted") - except ClientError: - warn("Already gone") - warn(f"Deleting IAM role: {role_name}") +def run(region: str, invocations: int, function_name: str | None, list_only: bool) -> None: try: - iam.detach_role_policy( - RoleName=role_name, - PolicyArn="arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", - ) - except ClientError: - pass - try: - iam.delete_role(RoleName=role_name) - log("Role deleted") - except ClientError: - warn("Already gone") - - log("Cleanup done.") + session = boto3.Session() + lam = session.client("lambda", region_name=region) + sqs = session.client("sqs", region_name=region) + except NoCredentialsError: + die("No AWS credentials found. Configure via environment variables or ~/.aws/credentials.") + # ── credential check ────────────────────────────────────────────────────── + try: + sts = session.client("sts", region_name=region) + identity = sts.get_caller_identity() + log(f"AWS identity: {identity.get('Arn', 'unknown')}") + except ClientError as e: + die(f"AWS credentials error: {e}\nRun 'aws sso login' or set AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY.") + + # ── list mode ───────────────────────────────────────────────────────────── + functions = list_functions(lam) + if list_only: + if not functions: + warn("No Lambda functions found in this account/region.") + else: + print(f"\nAvailable Lambda functions in {region}:") + for fn in functions: + print(f" {fn}") + return + + # ── pick Lambda ─────────────────────────────────────────────────────────── + if function_name: + # verify it exists + try: + lam.get_function(FunctionName=function_name) + except ClientError: + die(f"Lambda function '{function_name}' not found in {region}.") + else: + if not functions: + die(f"No Lambda functions found in {region}. Use --function to specify one.") + function_name = functions[0] + log(f"Auto-selected Lambda: {function_name} (use --list to see all, --function to pick)") -def run(region: str, invocations: int) -> None: queue_url = get_queue_url() if not queue_url: die("No SQS queue URL found. Run 'Create Infrastructure' in Settings → AWS Configuration first.") - suffix = int(time.time()) - function_name = f"opendevops-test-error-{suffix}" - role_name = f"opendevops-test-role-{suffix}" - log(f"Region : {region}") log(f"Queue : {queue_url}") log(f"Function : {function_name}") print() - session = boto3.Session() - iam = session.client("iam") - lam = session.client("lambda", region_name=region) - sqs = session.client("sqs", region_name=region) - - # ── create IAM role ────────────────────────────────────────────────────── - log("Creating IAM execution role...") - trust = json.dumps({ - "Version": "2012-10-17", - "Statement": [{ - "Effect": "Allow", - "Principal": {"Service": "lambda.amazonaws.com"}, - "Action": "sts:AssumeRole", - }], - }) - role_arn = iam.create_role( - RoleName=role_name, - AssumeRolePolicyDocument=trust, - )["Role"]["Arn"] - iam.attach_role_policy( - RoleName=role_name, - PolicyArn="arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", - ) - log(f"Role ARN : {role_arn}") - log("Waiting 10s for IAM role to propagate...") - time.sleep(10) - - # ── deploy Lambda ──────────────────────────────────────────────────────── - log("Deploying Lambda function...") - lam.create_function( - FunctionName=function_name, - Runtime="python3.12", - Role=role_arn, - Handler="index.handler", - Code={"ZipFile": make_zip()}, - Timeout=10, - ) - - waiter = lam.get_waiter("function_active") - waiter.wait(FunctionName=function_name) - log("Lambda deployed.") - - # ── invoke to generate real errors + CloudWatch logs ──────────────────── - log(f"Invoking {invocations} times to generate real errors...") + # ── invoke Lambda with intentionally bad payload ────────────────────────── + log(f"Invoking '{function_name}' {invocations}× with a bad payload to generate real errors...") + bad_payload = b'{"__test_force_error": true, "trigger": "opendevops-pipeline-test"}' + succeeded = 0 for i in range(1, invocations + 1): - lam.invoke( - FunctionName=function_name, - InvocationType="RequestResponse", - Payload=b'{"test": true}', - ) - print(f" invocation {i}/{invocations}") - - log(f"Errors written to CloudWatch Logs: /aws/lambda/{function_name}") + try: + resp = lam.invoke( + FunctionName=function_name, + InvocationType="RequestResponse", + Payload=bad_payload, + ) + status = resp.get("StatusCode", 0) + fn_error = resp.get("FunctionError", "") + if fn_error or status >= 400: + print(f" invocation {i}/{invocations} → error (good)") + else: + print(f" invocation {i}/{invocations} → success (function handled bad input gracefully)") + succeeded += 1 + except ClientError as e: + warn(f" invocation {i}/{invocations} → invoke failed: {e}") + + if succeeded == 0: + warn("All invocations failed to reach the function — no logs may exist.") + else: + log(f"Real invocation logs at: CloudWatch → /aws/lambda/{function_name}") print() - # ── push event to SQS with real function name ──────────────────────────── + # ── push alarm event to SQS ─────────────────────────────────────────────── now = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") event = { "source": "aws.cloudwatch", @@ -176,12 +142,12 @@ def run(region: str, invocations: int) -> None: "time": now, "region": region, "detail": { - "alarmName": f"{function_name}-errors", + "alarmName": f"{function_name}-test-errors", "state": { "value": "ALARM", "reason": ( f"Threshold Crossed: {invocations} error datapoints" - " in the last 1 minute (threshold: 1)." + " in the last 1 minute (threshold: 1). Triggered by pipeline test." ), }, "configuration": { @@ -200,11 +166,14 @@ def run(region: str, invocations: int) -> None: }, } - log("Sending event to SQS...") - msg_id = sqs.send_message( - QueueUrl=queue_url, - MessageBody=json.dumps(event), - )["MessageId"] + log("Sending alarm event to SQS...") + try: + msg_id = sqs.send_message( + QueueUrl=queue_url, + MessageBody=json.dumps(event), + )["MessageId"] + except ClientError as e: + die(f"Failed to send SQS message: {e}") sep = "━" * 60 print(f"\n{GREEN}{sep}{NC}") @@ -216,21 +185,18 @@ def run(region: str, invocations: int) -> None: print(f"{GREEN}{sep}{NC}\n") warn("The agent will pull REAL CloudWatch logs from that function.") warn("Investigation takes ~30–90s. Watch the Monitoring page.") - print(f"\nTo clean up when done:") - print(f" {YELLOW}uv run python scripts/test_pipeline.py --cleanup {function_name} {role_name}{NC}") def main() -> None: - parser = argparse.ArgumentParser(description="Fire a real Lambda failure through the monitoring pipeline") - parser.add_argument("--cleanup", nargs=2, metavar=("FUNCTION", "ROLE"), help="Tear down resources") - parser.add_argument("--region", default=os.environ.get("AWS_REGION", "us-east-1")) + parser = argparse.ArgumentParser( + description="Trigger a real Lambda failure through the monitoring pipeline" + ) + parser.add_argument("--function", help="Lambda function name to use (auto-picks first if omitted)") + parser.add_argument("--region", default=os.environ.get("AWS_REGION", "us-east-1")) parser.add_argument("--invocations", type=int, default=5) + parser.add_argument("--list", action="store_true", help="List available Lambda functions and exit") args = parser.parse_args() - - if args.cleanup: - cleanup(args.cleanup[0], args.cleanup[1], args.region) - else: - run(args.region, args.invocations) + run(args.region, args.invocations, args.function, args.list) if __name__ == "__main__": From 5adcb2558229de6fb117e60becd05f4e12414333 Mon Sep 17 00:00:00 2001 From: AhmadHammad21 Date: Tue, 12 May 2026 21:43:55 +0300 Subject: [PATCH 5/9] fix: use services_affected[0] as service name fallback in event consumer delivery --- src/agent/event_consumer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/agent/event_consumer.py b/src/agent/event_consumer.py index 96821d8..fc6894a 100644 --- a/src/agent/event_consumer.py +++ b/src/agent/event_consumer.py @@ -120,7 +120,10 @@ async def _run_investigation(prompt: str) -> dict[str, Any] | None: async def _deliver(result: dict[str, Any], event: dict) -> None: """Send investigation result to SNS + Slack and persist to the monitor store.""" - service = result.get("service_name", result.get("service", "unknown")) + services_affected = result.get("services_affected", []) + service = result.get("service_name", result.get("service", + services_affected[0] if services_affected else "unknown" + )) root_cause = result.get("root_cause_summary", "") mitigation = result.get("mitigation_steps", []) confidence = result.get("confidence", "MEDIUM") From 80bbeffbd8b47796e9752b45f38a053aa505e61e Mon Sep 17 00:00:00 2001 From: AhmadHammad21 Date: Tue, 12 May 2026 22:05:29 +0300 Subject: [PATCH 6/9] fix: write polling investigation results to monitoring page via add_alert --- src/agent/poller.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/agent/poller.py b/src/agent/poller.py index 6b2f33c..e428af4 100644 --- a/src/agent/poller.py +++ b/src/agent/poller.py @@ -10,14 +10,13 @@ from __future__ import annotations import asyncio -from datetime import datetime, timedelta, UTC +from datetime import UTC, datetime, timedelta from typing import Any from loguru import logger from config import settings - # In-memory dedup: maps a trigger key → datetime last investigated. # Resets on process restart (intentional — re-check on startup is fine). _last_investigated: dict[str, datetime] = {} @@ -37,6 +36,7 @@ def _mark_investigated(key: str) -> None: async def _run_investigation(prompt: str, trigger_key: str) -> dict[str, Any] | None: """Invoke the agent synchronously in a thread and return the submit_investigation args.""" import uuid + from agent.core import get_agent thread_id = str(uuid.uuid4()) @@ -64,18 +64,32 @@ async def _run_investigation(prompt: str, trigger_key: str) -> dict[str, Any] | async def _persist_and_notify(prompt: str, result: dict[str, Any], session_id: str) -> None: - from agent.turns import save_turn, notify_slack + from agent.monitor_store import add_alert + from agent.turns import notify_slack, save_turn + tool_calls_log = [{"tool": "submit_investigation", "args": result, "result": {}}] usage = {"model": settings.llm_model, "latency_ms": 0} await save_turn(session_id, prompt, "", tool_calls_log, usage) await notify_slack(session_id, tool_calls_log) + services_affected = result.get("services_affected", []) + service = services_affected[0] if services_affected else "unknown" + root_cause = result.get("root_cause_summary", "") + mitigation = result.get("mitigation_steps", []) + confidence = result.get("confidence", "MEDIUM") + resolution = "\n".join(mitigation) if isinstance(mitigation, list) else str(mitigation) + add_alert( + service=service, error=root_cause, resolution=resolution, + confidence=confidence, sns_sent=False, + ) + async def _check_alarms() -> None: """Check CloudWatch alarms — each new ALARM state triggers an investigation.""" - from tools.cloudwatch import get_alarms import uuid + from tools.cloudwatch import get_alarms + data = await asyncio.get_event_loop().run_in_executor(None, get_alarms, "ALARM") for alarm in data.get("alarms", []): name = alarm.get("name", "unknown") @@ -104,9 +118,10 @@ async def _check_alarms() -> None: async def _check_lambda_errors() -> None: """Check Lambda functions for error rates above the configured threshold.""" - from tools.lambda_ import list_lambda_functions, get_lambda_error_rate import uuid + from tools.lambda_ import get_lambda_error_rate, list_lambda_functions + funcs = await asyncio.get_event_loop().run_in_executor(None, list_lambda_functions) for fn in funcs.get("functions", [])[:20]: # cap at 20 to avoid runaway API calls name = fn.get("name", "") From f89d6503a40246eb22c6caaa4711f089d453c03c Mon Sep 17 00:00:00 2001 From: AhmadHammad21 Date: Tue, 12 May 2026 22:11:35 +0300 Subject: [PATCH 7/9] docs: add polling vs event-driven comparison table and monitoring page note --- docs/slack_and_polling.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/slack_and_polling.md b/docs/slack_and_polling.md index 2df6582..04fffe9 100644 --- a/docs/slack_and_polling.md +++ b/docs/slack_and_polling.md @@ -9,6 +9,21 @@ OpenDevOps Agent can send investigation results to a Slack channel in two modes: --- +## Polling vs Event-Driven — which do you need? + +| | Proactive Polling | Event-Driven (EventBridge → SQS) | +|---|---|---| +| **Setup required** | Just set `POLL_INTERVAL_MINUTES` in `.env` | Create AWS infra via Settings → AWS Configuration | +| **Detection latency** | Up to N minutes (your poll interval) | Near real-time (~seconds after event fires) | +| **CloudWatch alarm required** | No — checks Lambda error rates directly | Yes — EventBridge only fires when an alarm trips | +| **Works out of the box** | Yes | Requires CloudWatch alarms on your resources | +| **Results in /monitoring** | Yes | Yes | +| **Slack notifications** | Yes | Yes | + +**Recommendation:** enable both. Polling works immediately with zero AWS setup and catches Lambda error rate spikes. Event-driven is faster and catches a wider range of AWS failures (ECS, RDS, EC2, GuardDuty, etc.) once you have CloudWatch alarms in place. They are complementary — both write to the same `/monitoring` page and Slack channel. + +--- + ## Configuration Add these to your `.env`: @@ -58,6 +73,7 @@ Calls `list_lambda_functions` (capped at 20 functions) and checks each one's err - Builds a prompt describing the error rate - Runs a full agent investigation - Posts the result to Slack +- Saves the result to the `alerts` table — visible in the `/monitoring` page ### Dedup An in-memory dict (`_last_investigated`) maps each trigger key (`alarm:` or `lambda_errors:`) to the last time it was investigated. If the cooldown (`POLL_REINVESTIGATE_HOURS`) hasn't elapsed, the alarm or function is skipped. This resets on process restart (intentional — re-checking on startup is fine). From 010c87be2d4e3d0b9202dc6a879599585e1e7f57 Mon Sep 17 00:00:00 2001 From: AhmadHammad21 Date: Tue, 12 May 2026 22:38:23 +0300 Subject: [PATCH 8/9] feat: create aggregate Lambda CloudWatch alarm in setup_event_infra Automatically creates opendevops-lambda-errors-aggregate alarm (no dimensions, >= 1 error/min) when infrastructure is provisioned. This alarm triggers the existing EventBridge alarm-state rule on any Lambda error in the account, making the event-driven pipeline work without any per-function CloudWatch alarm setup. Teardown removes the alarm alongside the SQS queue and EventBridge rules. --- src/agent/event_infra.py | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/src/agent/event_infra.py b/src/agent/event_infra.py index 2b4928c..7d41185 100644 --- a/src/agent/event_infra.py +++ b/src/agent/event_infra.py @@ -68,12 +68,19 @@ def _session() -> boto3.Session: return boto3.Session(profile_name=settings.aws_profile) if settings.aws_profile else boto3.Session() +ALARM_NAME = "opendevops-lambda-errors-aggregate" + + def setup_event_infra() -> dict: - """Create SQS queue and EventBridge rules. Returns {queue_url, queue_arn, rule_arns}.""" + """Create SQS queue, EventBridge rules, and aggregate Lambda alarm. + + Returns {queue_url, queue_arn, rule_arns}. + """ s = _session() region = settings.aws_region sqs = s.client("sqs", region_name=region) events = s.client("events", region_name=region) + cw = s.client("cloudwatch", region_name=region) sts = s.client("sts", region_name=region) account_id = sts.get_caller_identity()["Account"] @@ -117,16 +124,33 @@ def setup_event_infra() -> dict: Targets=[{"Id": "opendevops-sqs", "Arn": queue_arn}], ) - logger.info("Event infra created: queue={} rules={}", queue_url, len(rule_arns)) + # Aggregate CloudWatch alarm — fires when ANY Lambda in the account errors. + # No dimensions = account-wide. EventBridge alarm-state rule routes it to SQS. + cw.put_metric_alarm( + AlarmName=ALARM_NAME, + AlarmDescription="OpenDevOps Agent — triggers investigation on any Lambda error", + Namespace="AWS/Lambda", + MetricName="Errors", + Statistic="Sum", + Period=60, + EvaluationPeriods=1, + Threshold=1, + ComparisonOperator="GreaterThanOrEqualToThreshold", + TreatMissingData="notBreaching", + ) + logger.info( + "Event infra created: queue={} rules={} alarm={}", queue_url, len(rule_arns), ALARM_NAME + ) return {"queue_url": queue_url, "queue_arn": queue_arn, "rule_arns": rule_arns} def teardown_event_infra(queue_url: str, rule_arns: dict[str, str]) -> None: - """Remove EventBridge rules and SQS queue.""" + """Remove EventBridge rules, aggregate CloudWatch alarm, and SQS queue.""" s = _session() region = settings.aws_region sqs = s.client("sqs", region_name=region) events = s.client("events", region_name=region) + cw = s.client("cloudwatch", region_name=region) for rule_name in rule_arns: try: @@ -135,6 +159,11 @@ def teardown_event_infra(queue_url: str, rule_arns: dict[str, str]) -> None: except Exception as e: logger.warning("Failed to delete rule {}: {}", rule_name, e) + try: + cw.delete_alarms(AlarmNames=[ALARM_NAME]) + except Exception as e: + logger.warning("Failed to delete alarm {}: {}", ALARM_NAME, e) + try: sqs.delete_queue(QueueUrl=queue_url) except Exception as e: From b305f93353689234dbd4b385c9e23ca54fb5313c Mon Sep 17 00:00:00 2001 From: AhmadHammad21 Date: Tue, 12 May 2026 23:14:33 +0300 Subject: [PATCH 9/9] docs: expand monitoring.md with polling vs event-driven architecture - Rewrites docs/monitoring.md to cover both detection modes with comparison table, full flows, latency explanation, and SQS vs SNS disambiguation - Adds --lambda-only flag to test_pipeline.py for real alarm path testing - Removes bash test_pipeline.sh (replaced by Python script) - Suppresses LiteLLM 'Provider List' info spam in app startup --- docs/monitoring.md | 247 ++++++++++++++++++++++++++++++++++----- scripts/test_pipeline.py | 28 ++++- scripts/test_pipeline.sh | 195 ------------------------------- src/api/app.py | 4 + 4 files changed, 244 insertions(+), 230 deletions(-) delete mode 100644 scripts/test_pipeline.sh diff --git a/docs/monitoring.md b/docs/monitoring.md index 301b8e0..63e3563 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -1,16 +1,207 @@ # Monitoring Dashboard -The Monitoring page shows a live feed of incidents detected by the event-driven consumer. Each entry is a completed investigation triggered by an AWS infrastructure event. +The Monitoring page shows a live feed of every incident detected and investigated by the agent — whether triggered by the proactive poller, an EventBridge event, or a manual test. This page explains both the UI and the two detection systems that feed it. --- -## Accessing the Dashboard +## Detection Modes -Navigate to **Monitoring** in the sidebar. The page is visible to all authenticated users; the data is shared org-wide. +OpenDevOps Agent has two complementary detection modes. Both write to the same `/monitoring` page and Slack channel. + +| | Proactive Polling | Event-Driven (EventBridge → SQS) | +|---|---|---| +| **How it works** | Agent polls CloudWatch alarms + Lambda error rates on a timer | AWS fires EventBridge rules → SQS → consumer → agent | +| **Setup** | Set `POLL_INTERVAL_MINUTES` in `.env` | Create infrastructure via Settings → AWS Configuration | +| **Detection latency** | Up to N minutes (your poll interval) | 2–5 min for alarm-based events; near-instant for direct events (ECS, RDS, GuardDuty) | +| **CloudWatch alarm required** | No — reads Lambda error rate directly via API | Yes — EventBridge only fires when an alarm changes state | +| **Works without AWS setup** | Yes — no SQS/EventBridge needed | No — requires SQS queue + EventBridge rules | +| **Event types covered** | CloudWatch alarms in ALARM state + Lambda error rates | CloudWatch alarms, ECS task failures, Lambda async errors, RDS events, EC2 state changes, CodePipeline failures, GuardDuty findings, AWS Health events | +| **Appears in /monitoring** | Yes | Yes | +| **Slack notifications** | Yes (if `SLACK_WEBHOOK_URL` set) | Yes (if `SLACK_WEBHOOK_URL` set) | + +**Recommendation:** enable both. Polling works immediately with zero AWS infrastructure and is faster for Lambda error detection. Event-driven catches a wider surface area (ECS, RDS, EC2, GuardDuty, etc.) once infrastructure is created. + +--- + +## Proactive Polling + +### What it does + +Every `POLL_INTERVAL_MINUTES` minutes, the background poller runs two checks: + +1. **CloudWatch alarms** — fetches every alarm currently in `ALARM` state; runs a full agent investigation for each new one +2. **Lambda error rates** — checks the last-hour error rate for up to 20 Lambda functions; runs an investigation for any function above `POLL_ERROR_THRESHOLD`% + +### Flow + +``` +Background timer fires every N minutes + → get_alarms("ALARM") — any alarm in ALARM state? + → list_lambda_functions() — check each function's error rate + → error_rate > threshold? + → agent.ainvoke() — full ReAct investigation + → add_alert() — persists to alerts table → /monitoring + → notify_slack() — posts to Slack +``` + +### When to use + +- You want incident detection with zero AWS infrastructure setup +- Fast Lambda error detection (1-minute polling catches errors in ~1–2 minutes) +- You don't have CloudWatch alarms pre-configured + +### Configuration + +```bash +POLL_INTERVAL_MINUTES=5 # 0 = disabled (default) +POLL_ERROR_THRESHOLD=5.0 # Lambda error rate % to trigger investigation +POLL_REINVESTIGATE_HOURS=1 # cooldown — skip re-investigating the same alarm within N hours +``` + +Set `POLL_INTERVAL_MINUTES=0` to disable entirely. The poller only starts when this value is greater than zero. + +### Deduplication + +An in-memory map (`_last_investigated`) tracks when each alarm or Lambda was last investigated. If the cooldown (`POLL_REINVESTIGATE_HOURS`) hasn't elapsed, the trigger is skipped. This resets on process restart — intentionally, so startup re-checks everything. + +### Source + +`src/agent/poller.py` — see also [slack_and_polling.md](slack_and_polling.md) for Slack integration details. --- -## Incident Feed +## Event-Driven Detection + +### What it does + +An SQS long-poll consumer receives real AWS infrastructure events forwarded by EventBridge rules, runs a deterministic context-collection pass, then runs a full agent investigation. + +### Flow + +``` +AWS service emits an event (ECS task stopped, Lambda errors, RDS failover…) + → CloudWatch / EventBridge evaluates matching rule + → SQS queue (opendevops-agent-events) + → event_consumer_loop() — long-polls every 20s + → _is_real_failure() — filters noise (e.g. EC2 "running", healthy RDS events) + → collect_context() — deterministic boto3 pre-fetch (logs, metrics, config) + → agent.ainvoke() — full ReAct investigation + → add_alert() — persists to alerts table → /monitoring + → SNS publish — fan-out to email, Lambda, etc. + → notify_slack() — posts to Slack +``` + +### When to use + +- You want broad coverage beyond Lambda (ECS, RDS, EC2, GuardDuty, AWS Health) +- You already have CloudWatch alarms configured on your resources +- You want SNS fan-out for email/PagerDuty/custom integrations + +### Why 2–5 minutes for alarm-based events? + +CloudWatch alarm-triggered EventBridge events have an inherent delay: + +1. Lambda generates errors → CloudWatch collects metric (~1 min) +2. Alarm evaluation period fires (minimum 60s period) +3. Alarm transitions OK → ALARM → EventBridge fires (near-instant) +4. SQS consumer picks up message (up to 20s long-poll) +5. Agent investigation runs (30–90s) + +Direct events (ECS task stopped, GuardDuty finding, RDS failover) skip the alarm evaluation and arrive in seconds. + +### Event types and rules + +| EventBridge Rule | Events captured | +|---|---| +| `opendevops-alarm-state` | CloudWatch Alarm state changes (OK → ALARM) | +| `opendevops-lambda-failure` | Lambda async invocation failures | +| `opendevops-lambda-throttle` | Lambda throttling errors | +| `opendevops-ecs-task-stopped` | ECS task stopped with non-zero exit / OOM | +| `opendevops-ec2-state` | EC2 instance terminated | +| `opendevops-rds-events` | RDS failures, failovers, recovery | +| `opendevops-health` | AWS Health personal health events | +| `opendevops-codedeploy-failure` | CodeDeploy deployment failures | +| `opendevops-guardduty` | GuardDuty findings | + +### Aggregate Lambda alarm + +When you create infrastructure (Settings → AWS Configuration → Create Infrastructure), the setup wizard also creates a CloudWatch alarm named `opendevops-lambda-errors-aggregate`. This alarm: + +- Monitors `AWS/Lambda` `Errors` metric with **no dimensions** — meaning it watches ALL Lambda functions in the account +- Trips when any Lambda has ≥ 1 error in a 60-second window +- Triggers the `opendevops-alarm-state` EventBridge rule → SQS → consumer + +This means you get event-driven Lambda coverage without creating per-function alarms. + +### Setup + +Go to **Settings → AWS Configuration → Create Infrastructure**. This creates: +- SQS queue: `opendevops-agent-events` +- 9 EventBridge rules (all prefixed `opendevops-`) +- Aggregate CloudWatch alarm: `opendevops-lambda-errors-aggregate` + +To remove it, click **Teardown**. See [event_detection.md](event_detection.md) for full setup details and IAM permission requirements. + +### Context collectors + +Before the LLM runs, `collect_context()` makes deterministic boto3 calls to pre-fetch facts about the affected resource (logs, metrics, config). This reduces LLM tool calls by front-loading the most relevant data. Capped at 3,000 chars to stay within token budgets. + +### Source + +`src/agent/event_consumer.py`, `src/agent/event_infra.py`, `src/agent/context_collectors.py` — see also [event_detection.md](event_detection.md). + +--- + +## SQS vs SNS — What Each Does + +These are often confused. They serve different roles: + +| | SQS | SNS | +|---|---|---| +| **Direction** | Input to agent | Output from agent | +| **Role** | Receives EventBridge events for the agent to process | Fan-out investigated results to subscribers | +| **Who writes to it** | AWS EventBridge rules | Agent, after investigation completes | +| **Who reads from it** | `event_consumer_loop()` | Email, Lambda, Telegram, custom HTTP endpoints | +| **Required** | Yes, for event-driven detection | No — optional output channel | + +SQS is the inbox. SNS is the outbox. You need SQS for event-driven detection; SNS is optional and only relevant if you want to route investigation results to external systems beyond Slack. + +--- + +## Choosing Between the Two + +| Scenario | Recommendation | +|---|---| +| Getting started, no AWS setup | Polling only (`POLL_INTERVAL_MINUTES=5`) | +| Lambda error monitoring only | Polling — faster and simpler | +| ECS, RDS, EC2, GuardDuty coverage | Event-driven required | +| Already have CloudWatch alarms | Event-driven — reuses your existing alarms | +| Want fastest Lambda detection | Both — polling at 1 min catches errors before the alarm trips | +| Want email/PagerDuty alerts | Event-driven + SNS | + +--- + +## Testing the Pipeline + +**Test polling** — set `POLL_INTERVAL_MINUTES=1`, trigger Lambda errors, wait ~1–2 minutes. + +**Test event-driven (fast — bypasses CloudWatch):** +```bash +uv run python scripts/test_pipeline.py --function my-function +``` +Pushes a synthetic alarm event directly to SQS. Result appears on `/monitoring` in ~60–90s. + +**Test event-driven (real path — waits for alarm):** +```bash +uv run python scripts/test_pipeline.py --function my-function --lambda-only +``` +Invokes the Lambda with a bad payload, then exits. The `opendevops-lambda-errors-aggregate` alarm trips in ~1–2 minutes and fires the full EventBridge → SQS → agent path. + +--- + +## Monitoring Dashboard UI + +### Incident Feed Each alert card shows: @@ -22,46 +213,28 @@ Each alert card shows: Alerts are sorted by most recent first. Click any card to open the Alert Detail page. ---- - -## Alert Detail Page +### Alert Detail Page -Shows the full investigation result for a single alert: +Shows the full investigation result: - Root cause in a highlighted panel -- Resolution steps (if the agent produced them) +- Resolution steps produced by the agent - Timestamp and SNS notification status -### Investigate button +**Investigate button** — opens a new chat session with a pre-seeded prompt for deeper analysis. The prompt auto-submits when the chat page loads. -Opens a new chat session with a pre-seeded prompt: +### Service Health Panel -``` -Investigate this incident further: +The top of the page shows tracked services and their last-known status (`healthy`, `error`, `unknown`). Updated in-memory each time the event consumer processes an event for a service. -Service: ECS / my-api -Error: Container OOM killed -Confidence: HIGH -Time: 2026-05-09T18:30:00Z +### Send Test Event (admin only) -Please provide deeper root cause analysis, check related services, -and suggest preventive measures. -``` - -The prompt auto-submits when the chat page loads. Use this to drill deeper than the automated investigation or to explore related services. - ---- - -## Service Health Panel - -The top of the Monitoring page shows a summary of tracked services and their last-known status (`healthy`, `error`, `unknown`). Status is updated in-memory each time the event consumer processes an event for that service. +The **Send test event** button in the Monitoring page header pushes a synthetic Lambda alarm to SQS. Requires event infrastructure to be set up. Shows an error toast if `sqs_queue_url` is not configured. --- ## API -The monitoring endpoints require authentication: - ``` GET /api/monitoring/alerts?limit=50 → list recent alerts GET /api/monitoring/alerts/{id} → single alert detail @@ -76,5 +249,17 @@ GET /api/monitoring/services → service health summary |---|---| | `frontend/src/pages/MonitoringPage.tsx` | Live incident feed UI | | `frontend/src/pages/AlertDetailPage.tsx` | Alert detail + investigate deeplink | -| `src/api/routers/monitoring.py` | REST endpoints (auth-protected) | +| `src/api/routers/monitoring.py` | REST endpoints | | `src/agent/monitor_store.py` | In-memory service status + alert persistence | +| `src/agent/poller.py` | Proactive polling loop | +| `src/agent/event_consumer.py` | SQS long-poll consumer | +| `src/agent/event_infra.py` | Create/teardown SQS + EventBridge + alarm | +| `src/agent/context_collectors.py` | Per-event-type boto3 context enrichment | +| `scripts/test_pipeline.py` | Pipeline test script (direct SQS push or Lambda invoke) | + +--- + +## Related Documentation + +- [event_detection.md](event_detection.md) — event-driven architecture deep dive, IAM permissions, context collectors +- [slack_and_polling.md](slack_and_polling.md) — Slack integration, polling configuration, deduplication diff --git a/scripts/test_pipeline.py b/scripts/test_pipeline.py index cc27d37..eed3fe1 100644 --- a/scripts/test_pipeline.py +++ b/scripts/test_pipeline.py @@ -12,6 +12,7 @@ uv run python scripts/test_pipeline.py --region eu-west-1 uv run python scripts/test_pipeline.py --invocations 10 uv run python scripts/test_pipeline.py --list # list available Lambdas + uv run python scripts/test_pipeline.py --lambda-only # invoke + wait for alarm (no manual SQS push) """ from __future__ import annotations @@ -24,6 +25,8 @@ import boto3 from botocore.exceptions import ClientError, NoCredentialsError +from dotenv import load_dotenv +load_dotenv() GREEN = "\033[0;32m" YELLOW = "\033[1;33m" @@ -58,7 +61,7 @@ def list_functions(lam) -> list[str]: return sorted(names) -def run(region: str, invocations: int, function_name: str | None, list_only: bool) -> None: +def run(region: str, invocations: int, function_name: str | None, list_only: bool, lambda_only: bool = False) -> None: try: session = boto3.Session() lam = session.client("lambda", region_name=region) @@ -134,6 +137,21 @@ def run(region: str, invocations: int, function_name: str | None, list_only: boo log(f"Real invocation logs at: CloudWatch → /aws/lambda/{function_name}") print() + # ── lambda-only: skip manual SQS push, let alarm fire naturally ────────── + if lambda_only: + sep = "━" * 60 + print(f"\n{GREEN}{sep}{NC}") + print(f"{GREEN}✓ Lambda invoked — waiting for alarm to fire automatically{NC}") + print(f" Function : {function_name}") + print(f" Alarm : opendevops-lambda-errors-aggregate") + print(f" Real logs : CloudWatch → /aws/lambda/{function_name}") + print(f" Monitoring: http://localhost/monitoring") + print(f"{GREEN}{sep}{NC}\n") + warn("CloudWatch evaluates every 60s — alarm should trip in ~1-2 minutes.") + warn("EventBridge fires → SQS → agent investigates automatically.") + warn("Watch server logs for: 'Processing event: aws.cloudwatch'") + return + # ── push alarm event to SQS ─────────────────────────────────────────────── now = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") event = { @@ -166,7 +184,7 @@ def run(region: str, invocations: int, function_name: str | None, list_only: boo }, } - log("Sending alarm event to SQS...") + log("Sending alarm event to SQS manually (bypasses EventBridge)...") try: msg_id = sqs.send_message( QueueUrl=queue_url, @@ -177,7 +195,7 @@ def run(region: str, invocations: int, function_name: str | None, list_only: boo sep = "━" * 60 print(f"\n{GREEN}{sep}{NC}") - print(f"{GREEN}✓ Pipeline triggered{NC}") + print(f"{GREEN}✓ Pipeline triggered (manual SQS push){NC}") print(f" MessageId : {msg_id}") print(f" Function : {function_name}") print(f" Real logs : CloudWatch → /aws/lambda/{function_name}") @@ -195,8 +213,10 @@ def main() -> None: parser.add_argument("--region", default=os.environ.get("AWS_REGION", "us-east-1")) parser.add_argument("--invocations", type=int, default=5) parser.add_argument("--list", action="store_true", help="List available Lambda functions and exit") + parser.add_argument("--lambda-only", action="store_true", dest="lambda_only", + help="Invoke Lambda and let the CloudWatch alarm fire automatically (no manual SQS push)") args = parser.parse_args() - run(args.region, args.invocations, args.function, args.list) + run(args.region, args.invocations, args.function, args.list, args.lambda_only) if __name__ == "__main__": diff --git a/scripts/test_pipeline.sh b/scripts/test_pipeline.sh deleted file mode 100644 index 3148843..0000000 --- a/scripts/test_pipeline.sh +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env bash -# test_pipeline.sh — spin up a real Lambda that errors, push it through the monitoring pipeline. -# -# Usage: -# bash scripts/test_pipeline.sh # run test -# bash scripts/test_pipeline.sh --region=eu-west-1 # override region -# bash scripts/test_pipeline.sh --invocations=10 # more errors -# bash scripts/test_pipeline.sh --cleanup FUNCTION ROLE # tear down after test - -set -euo pipefail - -# ── colours ────────────────────────────────────────────────────────────────── -GREEN='\033[0;32m'; YELLOW='\033[1;33m'; RED='\033[0;31m'; NC='\033[0m' -log() { echo -e "${GREEN}[+]${NC} $*"; } -warn() { echo -e "${YELLOW}[!]${NC} $*"; } -die() { echo -e "${RED}[✗]${NC} $*" >&2; exit 1; } - -# ── cleanup mode ───────────────────────────────────────────────────────────── -if [[ "${1:-}" == "--cleanup" ]]; then - FUNCTION_NAME="${2:-}"; ROLE_NAME="${3:-}" - [[ -z "$FUNCTION_NAME" || -z "$ROLE_NAME" ]] && die "Usage: $0 --cleanup FUNCTION_NAME ROLE_NAME" - REGION="${AWS_REGION:-us-east-1}" - warn "Deleting Lambda: $FUNCTION_NAME" - aws lambda delete-function --function-name "$FUNCTION_NAME" --region "$REGION" 2>/dev/null && log "Lambda deleted" || warn "Already gone" - warn "Deleting IAM role: $ROLE_NAME" - aws iam detach-role-policy --role-name "$ROLE_NAME" \ - --policy-arn "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" 2>/dev/null || true - aws iam delete-role --role-name "$ROLE_NAME" 2>/dev/null && log "Role deleted" || warn "Already gone" - log "Cleanup done." - exit 0 -fi - -# ── config ──────────────────────────────────────────────────────────────────── -REGION="${AWS_REGION:-us-east-1}" -INVOCATIONS=5 - -for arg in "$@"; do - case $arg in - --region=*) REGION="${arg#*=}" ;; - --invocations=*) INVOCATIONS="${arg#*=}" ;; - esac -done - -SUFFIX=$(date +%s) -FUNCTION_NAME="opendevops-test-error-${SUFFIX}" -ROLE_NAME="opendevops-test-role-${SUFFIX}" - -# ── find SQS queue URL ──────────────────────────────────────────────────────── -QUEUE_URL="" -if [[ -f "data/init.json" ]]; then - QUEUE_URL=$(python3 -c "import json; d=json.load(open('data/init.json')); print(d.get('sqs_queue_url',''))" 2>/dev/null || echo "") -fi -if [[ -z "$QUEUE_URL" ]]; then - QUEUE_URL="${SQS_QUEUE_URL:-}" -fi -if [[ -z "$QUEUE_URL" ]]; then - die "No SQS queue URL found. Run 'Create Infrastructure' in Settings → AWS Configuration first." -fi - -log "Region : $REGION" -log "Queue : $QUEUE_URL" -log "Function : $FUNCTION_NAME" -echo "" - -# ── create IAM role ──────────────────────────────────────────────────────────── -log "Creating IAM execution role..." -TRUST_POLICY='{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"lambda.amazonaws.com"},"Action":"sts:AssumeRole"}]}' - -ROLE_ARN=$(aws iam create-role \ - --role-name "$ROLE_NAME" \ - --assume-role-policy-document "$TRUST_POLICY" \ - --query 'Role.Arn' \ - --output text) - -aws iam attach-role-policy \ - --role-name "$ROLE_NAME" \ - --policy-arn "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" - -log "Role ARN : $ROLE_ARN" -log "Waiting 10s for IAM role to propagate..." -sleep 10 - -# ── create Lambda ZIP ───────────────────────────────────────────────────────── -TMP_DIR=$(mktemp -d) -trap 'rm -rf "$TMP_DIR"' EXIT - -cat > "$TMP_DIR/index.py" << 'PYEOF' -import random - -ERRORS = [ - "Connection pool exhausted: all 100 connections in use by active queries", - "DynamoDB ProvisionedThroughputExceededException on table 'orders' — read capacity 0/500 RCU remaining", - "Downstream service timeout: payments-service did not respond within 29000ms", - "Redis READONLY error: connected to replica — write commands not accepted", - "JWT verification failed: token expired 3601s ago (iat=1715000000)", - "Unhandled exception: Cannot read property 'userId' of undefined at processOrder:47", - "Out of memory: Lambda allocated 128 MB, peak usage 129 MB — increase memory limit", -] - -def handler(event, context): - error = random.choice(ERRORS) - raise RuntimeError(f"[TEST FAILURE] {error}") -PYEOF - -(cd "$TMP_DIR" && zip -q function.zip index.py) - -# ── deploy Lambda ───────────────────────────────────────────────────────────── -log "Deploying Lambda function..." -aws lambda create-function \ - --function-name "$FUNCTION_NAME" \ - --runtime python3.12 \ - --role "$ROLE_ARN" \ - --handler index.handler \ - --zip-file "fileb://$TMP_DIR/function.zip" \ - --region "$REGION" \ - --timeout 10 \ - --output text > /dev/null - -aws lambda wait function-active \ - --function-name "$FUNCTION_NAME" \ - --region "$REGION" - -log "Lambda deployed." - -# ── invoke it to generate real errors + CloudWatch logs ─────────────────────── -log "Invoking $INVOCATIONS times to generate real errors..." -for i in $(seq 1 "$INVOCATIONS"); do - aws lambda invoke \ - --function-name "$FUNCTION_NAME" \ - --payload '{"test": true}' \ - --invocation-type RequestResponse \ - --region "$REGION" \ - /tmp/lambda-out.json > /dev/null 2>&1 || true - printf " invocation %d/%d\n" "$i" "$INVOCATIONS" -done - -log "Errors written to CloudWatch Logs: /aws/lambda/$FUNCTION_NAME" -echo "" - -# ── push real event to SQS ──────────────────────────────────────────────────── -# Uses the real function name so the agent can look up actual CloudWatch logs. -NOW=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - -EVENT=$(cat << EOF -{ - "source": "aws.cloudwatch", - "detail-type": "CloudWatch Alarm State Change", - "time": "${NOW}", - "region": "${REGION}", - "detail": { - "alarmName": "${FUNCTION_NAME}-errors", - "state": { - "value": "ALARM", - "reason": "Threshold Crossed: ${INVOCATIONS} error datapoints in the last 1 minute (threshold: 1)." - }, - "configuration": { - "metrics": [{ - "metricStat": { - "metric": { - "namespace": "AWS/Lambda", - "name": "Errors", - "dimensions": {"FunctionName": "${FUNCTION_NAME}"} - }, - "period": 60, - "stat": "Sum" - } - }] - } - } -} -EOF -) - -log "Sending event to SQS..." -MSG_ID=$(aws sqs send-message \ - --queue-url "$QUEUE_URL" \ - --message-body "$EVENT" \ - --region "$REGION" \ - --query 'MessageId' \ - --output text) - -echo "" -echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" -echo -e "${GREEN}✓ Pipeline triggered${NC}" -echo " MessageId : $MSG_ID" -echo " Function : $FUNCTION_NAME" -echo " Real logs : CloudWatch → /aws/lambda/$FUNCTION_NAME" -echo " Monitoring: http://localhost/monitoring" -echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}" -echo "" -warn "The agent will pull REAL CloudWatch logs from that function." -warn "Investigation takes ~30–90s. Watch the Monitoring page." -echo "" -echo -e "To clean up when done:" -echo -e " ${YELLOW}bash scripts/test_pipeline.sh --cleanup $FUNCTION_NAME $ROLE_NAME${NC}" diff --git a/src/api/app.py b/src/api/app.py index 58bf746..662842f 100644 --- a/src/api/app.py +++ b/src/api/app.py @@ -4,11 +4,15 @@ from contextlib import asynccontextmanager from pathlib import Path +import litellm from fastapi import FastAPI from fastapi.responses import HTMLResponse from fastapi.staticfiles import StaticFiles from loguru import logger +# Suppress LiteLLM's "Provider List" info spam — we don't need provider discovery hints in logs +litellm.suppress_debug_info = True + from agent.core import init_agent from agent.db import db from api.routers import auth, chat, dashboard, history, sessions, settings, users