diff --git a/.dockerignore b/.dockerignore index e2b45d7..a7ca768 100644 --- a/.dockerignore +++ b/.dockerignore @@ -16,3 +16,4 @@ docker-compose.yml *.db *.pcap *.pcapng +!pcaps/demo.pcap diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..f4345a9 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ +version: 2 +updates: + - package-ecosystem: pip + directory: "/" + schedule: {interval: weekly} + - package-ecosystem: github-actions + directory: "/" + schedule: {interval: weekly} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae57209..565e4b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,10 +51,67 @@ jobs: run: pip install -r requirements.txt - name: Run tests with coverage - run: PYTHONPATH=. pytest tests/ -v --cov=app + run: PYTHONPATH=. pytest tests/ -v --cov=app --cov-report=term-missing --cov-fail-under=58 - name: Ruff lint run: ruff check . - name: Ruff format check run: ruff format --check . + + audit: + runs-on: ubuntu-latest + timeout-minutes: 30 + # Advisory only: a discovered CVE surfaces here without blocking merges. + # The required check for branch protection stays `test`. + continue-on-error: true + steps: + - name: Check out repository + uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Set up Python 3.11 + uses: actions/setup-python@v6 + with: + python-version: "3.11" + cache: pip + + - name: Install pip-audit + run: pip install pip-audit + + - name: Audit dependencies + run: pip-audit -r requirements.txt + + docker: + runs-on: ubuntu-latest + timeout-minutes: 40 + # Advisory only: the Dockerfile is the canonical deploy artifact (two + # Zeek OBS-repo installs, WeasyPrint/kaleido system libs, multi-stage + # builder->runtime->test) but the `test` job above never builds it, so a + # broken apt pin or requirements resolution failure would only surface + # when someone runs `make docker-build`. This job builds the `test` + # target and runs its suite, which also has real zeek installed (the + # `test` job above skips zeek-dependent e2e tests). It is deliberately + # NOT the required check — that stays `test`. + steps: + - name: Check out repository + uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build test-stage image (mirrors make docker-verify) + uses: docker/build-push-action@v6 + with: + context: . + target: test + tags: pcap-hunter:test + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Run in-image test suite (ruff format + lint + pytest, with real zeek) + run: docker run --rm pcap-hunter:test diff --git a/.gitignore b/.gitignore index c08213e..40e7f1d 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ htmlcov/ data/ *.pcap !tests/fixtures/*.pcap +!pcaps/demo.pcap *.log .DS_Store diff --git a/CHANGELOG.md b/CHANGELOG.md index f882c19..eee6186 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ First stable release. Production-ready installer, hardened pipeline, polished UX ### Changed - **PDF cover page redesign** — logo + tagline above the title, with classification banner and metadata block. -- **Kaleido upgraded to 1.x** — 0.x reaches end-of-life September 2025; 1.x is the active branch. +- **Kaleido pinned to `0.2.1`** — kaleido 1.x refuses to install alongside the pinned plotly 5.x ("not compatible"), so 0.2.1 is the only working pairing; it bundles its own headless Chromium and needs no system browser. See `requirements.txt` for the full rationale, and revisit together with a future `plotly>=6.1` upgrade. - **Testing discipline overhauled** — production-shape test data (real `CorrelationSignal` dataclasses, real DataFrames, nested dicts the pipeline actually produces) instead of simplified inputs. Documented in `CLAUDE.md` with bug-pattern history. New integration tests for every PDF section and chart. - **Version bumped to 1.0.0** with consolidated release notes. diff --git a/CLAUDE.md b/CLAUDE.md index 2cab302..dcfc326 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -198,6 +198,7 @@ GitHub Actions (`.github/workflows/ci.yml`): - Default PyShark limit: 200,000 packets - OSINT top IPs default: 50 - `MAX_FLOW_SAMPLES`: 5,000 per-flow packet timestamps/lengths (true totals kept in `count`/`first_ts`/`last_ts`) +- `ZEEK_TABLE_MAX_ROWS`: 50,000 per-log row cap on in-memory Zeek tables (DNS analysis + UI preview read these capped frames; JA3 reads the full uncapped log via `zeek_log_paths`); truncation appends `WARNING_ZEEK_TRUNCATED` to `PipelineResult.warnings` - `RUN_DIR_RETENTION_SECONDS`: 7 days — per-run `data/zeek|carved//` dirs pruned on the next run - Subprocess timeouts: `ZEEK_TIMEOUT_SECONDS` 600, `PCAP_COUNT_TIMEOUT_SECONDS` 120, `CARVE_TIMEOUT_SECONDS` 300, `TLS_EXTRACT_TIMEOUT_SECONDS` 300, `LLM_PROBE_TIMEOUT_SECONDS` 15 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..bd34310 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,142 @@ +# Contributing to PCAP Hunter + +Thanks for your interest in contributing. This is a concise guide to how the +repo actually works day-to-day — see `CLAUDE.md` for the full architecture +and conventions reference. + +## Getting set up + +```bash +git clone +cd pcap-hunter +make install # delegates to scripts/install.py (cross-platform) +``` + +`make install` installs both system binaries (tshark, Zeek, etc.) and Python +dependencies, then verifies them. Useful variants: + +```bash +make install-system # system binaries only +make install-python # Python packages only +make check-deps # or `make doctor` — verify everything is present +``` + +Run the app locally with: + +```bash +make run # streamlit run app/main.py (checks deps first) +``` + +## Before you commit: `make verify` + +**`make verify` is the pre-commit gate and it must pass before every +commit.** It runs, in order: + +1. `ruff format --check .` — formatting check +2. `ruff check .` — lint +3. `PYTHONPATH=. pytest tests/ -q` — full test suite + +```bash +make verify +``` + +CI (`.github/workflows/ci.yml`) runs the same three checks on every push and +PR to `main`, plus an advisory `pip-audit` dependency-scan job and a Docker +build/test job. If `make verify` passes locally, it will pass in CI. + +You can also run the pieces individually: + +```bash +make test # PYTHONPATH=. pytest tests/ -v --cov=app --cov-report=term-missing --cov-fail-under=58 +make test-pdf # focused PDF/chart smoke tests — run after touching pdf_generator.py, chart_images.py, or the charts module +make lint # ruff check . +make format # ruff format . +``` + +Always run tests with `PYTHONPATH=.` (or via `make test`/`make verify`, +which set it for you) — the codebase uses absolute imports +(`from app.pipeline.beacon import rank_beaconing`) and needs it on the path. + +### Host-Python caveat + +macOS machines frequently have multiple coexisting Python installs (Framework +Python, Homebrew Python, pyenv, etc.). `make test`/`make verify` use whichever +interpreter `streamlit` is installed under (see the `PYTHON` detection at the +top of the `Makefile`), but if your environment has `pytest` and other +dependencies split across interpreters, `make verify` on the host can fail +in ways that don't reflect a real problem. If you hit interpreter confusion: + +- confirm `pip show pytest` and `pip show streamlit` agree on the same + interpreter, or +- fall back to the canonical, environment-independent path below. + +### Docker: the canonical build-and-verify path + +Any verification that depends on a clean install (dependency changes, +install-path changes, or anything you want to be **certain** works outside +your local environment) should go through Docker rather than the host: + +```bash +make docker-verify # builds the `test` image and runs format+lint+tests inside it +make docker-up # build + run the UI at http://localhost:8501 +make docker-down +``` + +This mirrors what CI's `docker` job does and is the same environment the app +ships in, so it's the most trustworthy signal for anything build-shaped. + +## Code conventions + +- **Style**: Ruff, line length 120, double quotes, 4-space indent (see the + `select` list and per-file ignores in `pyproject.toml` for the exact rules) +- **Imports**: absolute only (`from app.pipeline.beacon import ...`), + stdlib → third-party → local, ordering enforced by ruff's `I` rule +- **Naming**: `snake_case.py` modules, `PascalCase` classes, `snake_case` + functions, `UPPER_SNAKE_CASE` constants, leading underscore for private + helpers +- **Data modeling**: prefer `dataclass` for structured data, `Enum` for + fixed categories +- **Errors/logging**: custom exceptions inherit from `Exception`; use the + `logging` module, never `print()` +- **Type hints**: used extensively, with `from __future__ import annotations` + for forward compatibility +- **Docstrings**: Google-style (`Args`/`Returns`) on public functions + +## Tests + +- One test file per major module: `tests/test_.py` +- Test classes `Test`, test functions `test_` +- Cover both the happy path and edge cases (empty, `None`, malformed input) +- **Use production-shape test data.** If a function consumes + `list[CorrelationSignal]` dataclasses, pass real dataclass instances in + tests, not dicts with similar-looking keys — simplified test inputs have + previously let real bugs ship. See `tests/test_pdf_integration.py` for the + expected shapes. +- No shared `conftest.py` fixtures — tests are independent +- New PDF sections need a corresponding assertion in + `tests/test_pdf_integration.py::test_html_contains_every_expected_section`; + new PDF charts need a kaleido smoke test in `tests/test_chart_rendering.py` + +## Commit messages + +Conventional-commits style, lowercase description after the prefix: + +``` +feat: add single-ioc exact-match lookup endpoint +fix: escape crlf in cef output to prevent syslog log injection +docs: refresh readme and user manuals +chore: cover all runtime deps in dependency check +``` + +Common prefixes: `feat:`, `fix:`, `docs:`, `style:`, `chore:`. + +## Submitting changes + +1. Make your change, keeping it focused. +2. Run `make verify` (and `make docker-verify` if the change touches + dependencies, install paths, or anything build-shaped). +3. Commit using the conventions above. +4. Open a pull request against `main` describing what changed and why. + +CI must pass (tests + coverage floor, lint, format check) before a PR can be +merged. diff --git a/Dockerfile b/Dockerfile index 87b7866..1d8809e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,6 +61,10 @@ RUN pip install --no-cache-dir /wheels/* # The previous flattened COPY (app/ -> /app/) broke `uvicorn app.api.app`. COPY app/ ./app/ +# Synthetic first-run demo capture (see pcaps/_make_demo.py) — powers the +# "Load demo capture" button so a fresh install has something to analyze. +COPY pcaps/demo.pcap ./pcaps/demo.pcap + # Non-root + data dirs RUN useradd -m runner && mkdir -p /data /app/data && chown -R runner:runner /app /data USER runner diff --git a/Makefile b/Makefile index ee33bcc..6bb7b9d 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ check-deps doctor: # ------------------------------------------------------------------------- test: - PYTHONPATH=. pytest tests/ -v --cov=app + PYTHONPATH=. pytest tests/ -v --cov=app --cov-report=term-missing --cov-fail-under=58 # Focused smoke test: PDF generation with charts + correlations + beacon DF. # Run this after any change to pdf_generator.py, chart_images.py, main.py diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..16692f7 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,67 @@ +# Security Policy + +## Supported versions + +PCAP Hunter is developed on a single rolling `main` branch — there is no +long-term-support branch. Security fixes are applied to `main` and released +as the next version; only the latest released version is supported. + +| Version | Supported | +|---------|-----------| +| latest (`main`) | yes | +| older releases | no | + +## Reporting a vulnerability + +Please do not open a public GitHub issue for security vulnerabilities. + +Instead, report privately using one of: + +- Open a private GitHub Security Advisory for this repository + (repo → **Security** tab → **Advisories** → **Report a vulnerability**), or +- Email the maintainer directly (see the repository's commit history / + GitHub profile for a current contact) with a description of the issue, + affected version/commit, and reproduction steps. + +Please include enough detail to reproduce the issue (PCAP sample or steps, +affected module/endpoint, expected vs. actual behavior). We'll acknowledge +reports and follow up with next steps as the issue is triaged. + +## Scope and hardening expectations + +A few things are worth calling out explicitly because of what this tool does: + +- **PCAP Hunter parses hostile, untrusted input by design.** PCAP files come + from real (potentially adversary-controlled) network traffic, and the + pipeline runs several native tools (Zeek, tshark, PyShark) and file-format + parsers (TLS/X.509, YARA) against that input. Treat any parser crash, + memory-safety issue, or resource-exhaustion bug in these paths as a + security-relevant finding, not just a bug. +- **The app makes outbound network calls.** OSINT enrichment (VirusTotal, + AbuseIPDB, GreyNoise, OTX, Shodan, WHOIS, reverse DNS) and LLM calls (LM + Studio, OpenAI, Anthropic) send indicators and/or sanitized context to + third-party or self-hosted endpoints. If you operate in an environment + where that egress is sensitive, review `app/security/opsec.py` and the + Config tab before enabling providers, and consider network-level egress + controls. +- **The Streamlit UI has no built-in authentication.** It is designed to run + as a local, single-analyst tool. Do not expose it directly to an untrusted + network or the public internet. If remote/shared access is required, put + it behind a reverse proxy that terminates TLS and enforces authentication + (e.g., an OAuth2 proxy, SSO gateway, or VPN-only access) rather than + relying on Streamlit itself for access control. The same applies to the + optional integrations API (`app/api/`) — run it behind a proxy/firewall + and use its API-key auth; do not expose it unauthenticated to the internet. +- **Config secrets are encrypted at rest but machine-bound.** API keys saved + via the Config tab are encrypted with a PBKDF2 key derived from local + machine identifiers (see `ConfigManager` in `CLAUDE.md`). Don't commit + `~/.pcap_hunter_config.json` or `.env` files, and don't share them across + machines expecting the encryption to travel with them. + +## Dependency scanning + +Dependencies are scanned on every push/PR via an advisory `pip-audit` CI job +(see `.github/workflows/ci.yml`) and kept up to date via Dependabot +(`.github/dependabot.yml`, weekly for both `pip` and GitHub Actions). Findings +there don't block merges automatically but are reviewed as part of normal +maintenance. diff --git a/app/analysis/correlation.py b/app/analysis/correlation.py index 98ad404..4f4e6e3 100644 --- a/app/analysis/correlation.py +++ b/app/analysis/correlation.py @@ -28,6 +28,8 @@ "flow_asymmetry": 0.04, "shodan_vulns": 0.10, "shodan_exposure": 0.04, + "http_cleartext_cred": 0.10, + "http_suspicious_ua": 0.06, } VERDICT_THRESHOLDS = { @@ -78,6 +80,58 @@ def _get_verdict(score: float) -> str: return "low" +def _normalize_http_host(host: str) -> str: + """Strip a trailing ``:port`` suffix from an HTTP Host-header value. + + Zeek's ``http.log`` Host field may include an explicit port (e.g. + ``example.com:8443`` or ``203.0.113.5:8443``), but the IP/domain + indicators this lookup is later matched against are always bare. A naive + ``host.rsplit(":", 1)[0]`` would mangle a bracketed IPv6 literal like + ``[::1]`` (splitting on one of the address's own colons), so bracketed + hosts are only trimmed after the closing bracket and otherwise left + untouched. + """ + if not host: + return host + if host.startswith("["): + close = host.find("]") + return host[: close + 1] if close != -1 else host + if ":" in host: + head, _, tail = host.rpartition(":") + if tail.isdigit(): + return head + return host + + +def _build_http_lookup(http_analysis: dict | None) -> dict[str, list[tuple[str, Any, float]]]: + """Build a Host-header -> [(signal_name, value, score), ...] lookup from HTTP analysis. + + The Zeek ``http.log`` Host header may be a raw IP or a domain, so this + lookup is matched against both IP and domain indicators by the callers + (``_collect_ip_signals`` / ``_collect_domain_signals``). Host values are + normalised (trailing ``:port`` stripped) before being used as keys so a + Host like ``example.com:8443`` still attaches its signal to the + ``example.com`` indicator. + """ + lookup: dict[str, list[tuple[str, Any, float]]] = {} + if not http_analysis: + return lookup + + for cred in http_analysis.get("cleartext_credentials", []) or []: + host = cred.get("host") if isinstance(cred, dict) else None + if host: + host = _normalize_http_host(host) + lookup.setdefault(host, []).append(("http_cleartext_cred", cred.get("username", ""), 0.9)) + + for ua in http_analysis.get("suspicious_user_agents", []) or []: + host = ua.get("host") if isinstance(ua, dict) else None + if host: + host = _normalize_http_host(host) + lookup.setdefault(host, []).append(("http_suspicious_ua", ua.get("reason", ""), 0.6)) + + return lookup + + def _collect_ip_signals( ip: str, osint: dict, @@ -85,6 +139,7 @@ def _collect_ip_signals( tls_lookup: dict[str, list[str]], yara_ips: set[str], asymmetry_lookup: dict[str, float], + http_lookup: dict[str, list[tuple[str, Any, float]]], ) -> list[CorrelationSignal]: """Collect all signals for an IP indicator.""" signals: list[CorrelationSignal] = [] @@ -147,6 +202,10 @@ def _collect_ip_signals( CorrelationSignal("flow_asymmetry", round(asymmetry_lookup[ip], 2), asymmetry_lookup[ip], "flow_analysis") ) + # HTTP findings (Host header matched a raw IP) + for name, value, score in http_lookup.get(ip, []): + signals.append(CorrelationSignal(name, value, score, "http")) + return signals @@ -154,6 +213,7 @@ def _collect_domain_signals( domain: str, osint: dict, dns_analysis: dict, + http_lookup: dict[str, list[tuple[str, Any, float]]], ) -> list[CorrelationSignal]: """Collect all signals for a domain indicator.""" signals: list[CorrelationSignal] = [] @@ -182,13 +242,24 @@ def _collect_domain_signals( signals.append(CorrelationSignal("dns_tunneling", tunnel.get("score", 0), tunnel["score"], "dns")) break + # HTTP findings (Host header matched a domain) + for name, value, score in http_lookup.get(domain, []): + signals.append(CorrelationSignal(name, value, score, "http")) + return signals # Tier definitions — a strong OSINT hit alone can push to "high"; behavioural # signals need corroboration; contextual signals alone cap at "medium". _TIER1_DEFINITIVE = {"vt_detections", "greynoise_malicious"} -_TIER2_BEHAVIOURAL = {"beacon_score", "flow_asymmetry", "dns_tunneling", "dga_domain"} +_TIER2_BEHAVIOURAL = { + "beacon_score", + "flow_asymmetry", + "dns_tunneling", + "dga_domain", + "http_cleartext_cred", + "http_suspicious_ua", +} _TIER3_CONTEXTUAL = {"abuseipdb", "self_signed_cert", "expired_cert", "yara_match", "shodan_vulns", "shodan_exposure"} # Strong-signal floors — a single definitive signal sets a minimum score @@ -243,6 +314,7 @@ def correlate_indicators( tls_analysis: dict | None = None, yara_results: dict | None = None, asymmetry_results: list | None = None, + http_analysis: dict | None = None, ) -> list[CorrelationResult]: """ Correlate indicators across all analysis modules. @@ -255,6 +327,9 @@ def correlate_indicators( tls_analysis: TLS certificate analysis yara_results: YARA scan results asymmetry_results: Flow asymmetry results + http_analysis: HTTP analysis results (cleartext credentials, suspicious + user agents). The Host header may be an IP or a domain, so matches + are attributed to whichever indicator type it corresponds to. Returns: List of CorrelationResult sorted by composite score (highest first) @@ -301,12 +376,14 @@ def correlate_indicators( if dst: asymmetry_lookup[dst] = max(asymmetry_lookup.get(dst, 0), score) + http_lookup = _build_http_lookup(http_analysis) + results: list[CorrelationResult] = [] # Correlate IPs ips = [ip for ip in features.get("artifacts", {}).get("ips", []) if is_public_ipv4(ip)] for ip in ips: - signals = _collect_ip_signals(ip, osint, beacon_lookup, tls_lookup, yara_ips, asymmetry_lookup) + signals = _collect_ip_signals(ip, osint, beacon_lookup, tls_lookup, yara_ips, asymmetry_lookup, http_lookup) if not signals: continue composite = _compute_composite(signals) @@ -323,7 +400,7 @@ def correlate_indicators( # Correlate domains domains = features.get("artifacts", {}).get("domains", []) for domain in domains: - signals = _collect_domain_signals(domain, osint, dns_analysis) + signals = _collect_domain_signals(domain, osint, dns_analysis, http_lookup) if not signals: continue composite = _compute_composite(signals) diff --git a/app/api/app.py b/app/api/app.py index 4a22bbf..dee1239 100644 --- a/app/api/app.py +++ b/app/api/app.py @@ -196,7 +196,7 @@ def create_app() -> FastAPI: CORSMiddleware, allow_origins=settings.cors_origins, allow_credentials=False, - allow_methods=["GET", "POST", "DELETE"], + allow_methods=["GET", "POST", "PATCH", "DELETE"], allow_headers=["Authorization", "Content-Type", "If-None-Match", "X-Request-ID"], ) diff --git a/app/api/feed.py b/app/api/feed.py index 63bd4e8..77a578f 100644 --- a/app/api/feed.py +++ b/app/api/feed.py @@ -28,6 +28,7 @@ class IOCFilter: types: list[str] = field(default_factory=list) tag: str | None = None case_id: str | None = None + value: str | None = None # exact match on i.value, for single-IOC lookup limit: int = 1000 offset: int = 0 @@ -48,11 +49,13 @@ def query_iocs(repo: CaseRepository, filt: IOCFilter) -> list[dict[str, Any]]: MIN(a.analyzed_at) AS first_seen, MAX(a.analyzed_at) AS last_seen, GROUP_CONCAT(DISTINCT a.case_id) AS case_ids, - GROUP_CONCAT(DISTINCT t.name) AS tag_names + GROUP_CONCAT(DISTINCT t.name) AS tag_names, + GROUP_CONCAT(DISTINCT at.technique_id) AS technique_ids FROM iocs i JOIN analyses a ON i.analysis_id = a.id LEFT JOIN case_tags ct ON ct.case_id = a.case_id LEFT JOIN tags t ON t.id = ct.tag_id + LEFT JOIN analysis_techniques at ON at.analysis_id = i.analysis_id """ where: list[str] = [] params: list[Any] = [] @@ -72,6 +75,12 @@ def query_iocs(repo: CaseRepository, filt: IOCFilter) -> list[dict[str, Any]]: "a.case_id IN (SELECT ct2.case_id FROM case_tags ct2 JOIN tags t2 ON t2.id = ct2.tag_id WHERE t2.name = ?)" ) params.append(filt.tag) + if filt.value: + # Exact match — uses idx_iocs_type_value. Composes with the other + # WHERE predicates (e.g. types) via AND; aggregation/pagination below + # are unaffected since this only narrows the row set pre-GROUP BY. + where.append("i.value = ?") + params.append(filt.value) if where: sql += " WHERE " + " AND ".join(where) @@ -102,7 +111,7 @@ def query_iocs(repo: CaseRepository, filt: IOCFilter) -> list[dict[str, Any]]: "first_seen": d["first_seen"], "last_seen": d["last_seen"], "case_ids": [c for c in (d.get("case_ids") or "").split(",") if c], - "mitre_techniques": [], # Future: derive from analysis features + "mitre_techniques": sorted({t for t in (d.get("technique_ids") or "").split(",") if t}), } ) return out diff --git a/app/api/models.py b/app/api/models.py index a6508a8..7e5869b 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -73,6 +73,34 @@ class IOCFeedResponse(BaseModel): next_cursor: str | None = None +class CaseListItem(BaseModel): + """Light case shape for GET /api/v1/cases — no embedded analyses/notes.""" + + id: str + title: str + description: str = "" + status: str + severity: str + created_at: str | None = None + updated_at: str | None = None + closed_at: str | None = None + tags: list[str] = Field(default_factory=list) + + +class CasePatchRequest(BaseModel): + """Partial update for PATCH /api/v1/cases/{id} — all fields optional.""" + + title: str | None = Field(default=None, min_length=1, max_length=200) + description: str | None = Field(default=None, max_length=5000) + status: str | None = Field(default=None, pattern=r"^(open|in_progress|closed)$") + severity: str | None = Field(default=None, pattern=r"^(low|medium|high|critical)$") + tags: list[str] | None = None + + +class NoteRequest(BaseModel): + content: str = Field(..., min_length=1) + + class PcapSubmissionForm(BaseModel): """Multipart form fields for POST /pcaps.""" diff --git a/app/api/queue.py b/app/api/queue.py index 9666a24..ebae631 100644 --- a/app/api/queue.py +++ b/app/api/queue.py @@ -5,6 +5,7 @@ import json import logging import os +import time from abc import ABC, abstractmethod from concurrent.futures import ProcessPoolExecutor from dataclasses import dataclass, field @@ -182,8 +183,15 @@ def _persist_analysis( dns_analysis=result.dns_analysis or None, tls_analysis=result.tls_analysis or None, ) + analysis.attack_mapping = result.attack_mapping + analysis.mitre_techniques = result.mitre_techniques if result.beacon_df_records: analysis.features["beacon_records"] = result.beacon_df_records + if result.http_analysis: + # Analysis has no dedicated http_analysis column — stash it in + # features (mirroring beacon_records above); features_json is + # compressed+persisted, and the UI restore path reads it back out. + analysis.features["http_analysis"] = result.http_analysis analysis.iocs = repo.extract_iocs(analysis) result.analysis_id = repo.save_analysis(analysis) except Exception: @@ -191,6 +199,82 @@ def _persist_analysis( result.warnings.append(WARNING_PERSISTENCE_FAILED) +def _dispatch_webhook(url: str, payload: dict, timeout: int, max_retries: int) -> None: + """POST a job-completion webhook from the worker subprocess; never raises. + + Called from ``_worker_run`` on both terminal branches (done + failed). + A webhook delivery failure must never change the job's own terminal + state, so every failure mode here is caught and logged, not propagated. + + Re-validates the URL with ``is_safe_webhook_url`` immediately before the + network call — defense in depth against the target becoming unsafe + between submit-time validation (the router) and job completion (e.g. a + DNS change). ``hardened_session`` does not itself block private IPs. + + Args: + url: Destination webhook URL (already submit-time validated). + payload: JSON-serializable envelope to POST. + timeout: Per-attempt request timeout, seconds. + max_retries: Additional attempts after the first, on non-2xx status + or request exception. + """ + # Every failure mode -- including the lazy imports below -- is caught by + # this outer guard so this function NEVER raises. If it did, the + # exception would propagate into _worker_run's `except Exception`, whose + # unconditional `update_job_status(..., FAILED, ...)` would revert an + # already-DONE job to FAILED. The imports live inside the try for exactly + # that reason. + try: + from app.security.opsec import hardened_session, redact + from app.utils.network_utils import is_safe_webhook_url + + if not is_safe_webhook_url(url): + logger.warning("Webhook dispatch refused: %s no longer passes the SSRF guard", redact(url)) + return + + body = json.dumps(payload).encode("utf-8") + headers = {"Content-Type": "application/json"} + + # Optional HMAC signing so the receiver can authenticate the callback. + secret = os.environ.get("PCAP_HUNTER_API_WEBHOOK_SECRET") or None + if secret: + import hashlib + import hmac + + signature = hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest() + headers["X-PCAP-Hunter-Signature"] = f"sha256={signature}" + + attempts = max(1, max_retries + 1) + last_error = "unknown" + for attempt in range(1, attempts + 1): + try: + # allow_redirects=False: the SSRF guard only validated `url`. + # Following a redirect (hardened_session permits up to 3, and + # only blocks https->http downgrades) to an attacker-chosen + # Location -- e.g. http://169.254.169.254/ -- would reach an + # unvalidated internal host and reopen the SSRF hole. Webhooks + # do not need redirects. + resp = hardened_session(timeout=timeout).post(url, data=body, headers=headers, allow_redirects=False) + if 200 <= resp.status_code < 300: + logger.info( + "Webhook delivered to %s (attempt %d/%d, status %d)", + redact(url), + attempt, + attempts, + resp.status_code, + ) + return + last_error = f"http_{resp.status_code}" + except Exception as exc: + last_error = f"{type(exc).__name__}: {exc}" + if attempt < attempts: + time.sleep(0.25) + logger.warning("Webhook delivery to %s failed after %d attempt(s): %s", redact(url), attempts, last_error) + except Exception: + # No redact() here: the import that provides it may be what failed. + logger.exception("Webhook dispatch crashed unexpectedly") + + def _worker_run(job_id: str, db_path: str, pcap_path: str, options_dict: dict) -> None: """Top-level worker function (must be picklable for ProcessPoolExecutor).""" from app.utils.logger import get_logger @@ -254,6 +338,23 @@ def _on_event(event: ProgressEvent) -> None: result_blob = json.dumps(result.to_dict()).encode("utf-8") repo.complete_job(job_id, result_blob) + + webhook_url = options_dict.get("webhook_url") + if webhook_url: + from app.api.settings import APISettings + + settings = APISettings.from_env() + _dispatch_webhook( + webhook_url, + { + "job_id": job_id, + "case_id": job.case_id, + "status": "done", + "analysis_id": result.analysis_id or None, + }, + timeout=settings.webhook_timeout_seconds, + max_retries=settings.webhook_max_retries, + ) except Exception as exc: logger.exception("Job %s failed: %s", job_id, exc) repo.update_job_status( @@ -263,6 +364,23 @@ def _on_event(event: ProgressEvent) -> None: error_detail=str(exc)[:500], ) + webhook_url = options_dict.get("webhook_url") + if webhook_url: + from app.api.settings import APISettings + + settings = APISettings.from_env() + _dispatch_webhook( + webhook_url, + { + "job_id": job_id, + "case_id": job.case_id, + "status": "failed", + "analysis_id": None, + }, + timeout=settings.webhook_timeout_seconds, + max_retries=settings.webhook_max_retries, + ) + class InProcessJobQueue(JobQueue): """ProcessPoolExecutor-backed queue using SQLite for state.""" diff --git a/app/api/routers/cases.py b/app/api/routers/cases.py index 3bbbc3c..a44689e 100644 --- a/app/api/routers/cases.py +++ b/app/api/routers/cases.py @@ -1,4 +1,4 @@ -"""GET /api/v1/cases/{id}, DELETE, /report.pdf""" +"""Case CRUD: GET (get/list), PATCH, DELETE, /report.pdf, and notes.""" from __future__ import annotations @@ -10,11 +10,13 @@ import shutil import tempfile -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Query from fastapi.responses import FileResponse, JSONResponse, Response from app.api.deps import get_repo, require_full_scope +from app.api.models import CaseListItem, CasePatchRequest, NoteRequest from app.api.queue import cancel_queued_job +from app.database.models import CaseStatus, Severity logger = logging.getLogger(__name__) @@ -27,6 +29,39 @@ def _reports_dir() -> pathlib.Path: ) +# Registered BEFORE the "/{case_id}" routes below — a literal "" path never +# actually collides with "/{case_id}" (different segment counts), but keeping +# the list route first avoids relying on that being true forever. +@router.get("") +def list_cases( + status: str | None = Query(default=None), + tag: str | None = Query(default=None), + search: str | None = Query(default=None), + limit: int = Query(default=100, ge=1, le=500), + offset: int = Query(default=0, ge=0), + _scope=Depends(require_full_scope), + repo=Depends(get_repo), +): + status_filter = CaseStatus.from_str(status) if status else None + tags_filter = [tag] if tag else None + cases = repo.list_cases(status=status_filter, tags=tags_filter, search=search, limit=limit, offset=offset) + items = [ + CaseListItem( + id=c.id, + title=c.title, + description=c.description, + status=c.status.value, + severity=c.severity.value, + created_at=c.created_at.isoformat() if c.created_at else None, + updated_at=c.updated_at.isoformat() if c.updated_at else None, + closed_at=c.closed_at.isoformat() if c.closed_at else None, + tags=c.tags, + ) + for c in cases + ] + return {"cases": [item.model_dump() for item in items], "count": len(items)} + + @router.get("/{case_id}") def get_case(case_id: str, _scope=Depends(require_full_scope), repo=Depends(get_repo)): case = repo.get_case(case_id) @@ -176,3 +211,63 @@ def delete_case(case_id: str, _scope=Depends(require_full_scope), repo=Depends(g logger.warning("Artifact cleanup after deleting case %s failed: %s", case_id, exc) return Response(status_code=204) + + +@router.patch("/{case_id}") +def patch_case( + case_id: str, + body: CasePatchRequest, + _scope=Depends(require_full_scope), + repo=Depends(get_repo), +): + case = repo.get_case(case_id) + if not case: + raise HTTPException(status_code=404, detail="case_not_found") + + if body.title is not None: + case.title = body.title + if body.description is not None: + case.description = body.description + if body.severity is not None: + case.severity = Severity.from_str(body.severity) + if body.tags is not None: + case.tags = body.tags + + if body.status is not None: + new_status = CaseStatus.from_str(body.status) + if new_status == CaseStatus.CLOSED: + case.close() # sets closed_at + updated_at + else: + case.status = new_status + + repo.update_case(case) + return JSONResponse(content=case.to_dict()) + + +@router.post("/{case_id}/notes", status_code=201) +def add_note( + case_id: str, + body: NoteRequest, + _scope=Depends(require_full_scope), + repo=Depends(get_repo), +): + # add_note() does not validate case_id (FK enforcement is off), so check first. + case = repo.get_case(case_id) + if not case: + raise HTTPException(status_code=404, detail="case_not_found") + + note_id = repo.add_note(case_id, body.content) + return {"id": note_id, "content": body.content} + + +@router.get("/{case_id}/notes") +def list_notes( + case_id: str, + _scope=Depends(require_full_scope), + repo=Depends(get_repo), +): + case = repo.get_case(case_id) + if not case: + raise HTTPException(status_code=404, detail="case_not_found") + + return {"notes": [n.to_dict() for n in case.notes]} diff --git a/app/api/routers/iocs.py b/app/api/routers/iocs.py index f50f478..4fb5365 100644 --- a/app/api/routers/iocs.py +++ b/app/api/routers/iocs.py @@ -15,6 +15,8 @@ from app.api.deps import get_repo, require_feed_scope from app.api.feed import IOCFilter, query_iocs +from app.api.models import IOCFeedResponse +from app.utils.cef_export import feed_rows_to_cef router = APIRouter(prefix="/api/v1", tags=["egress"]) @@ -105,6 +107,30 @@ def iocs_json( return _conditional_response(request, body, "application/json", _last_modified(rows)) +# ── Single-IOC lookup ─────────────────────────────────────────────────────── + + +@router.get("/iocs/lookup", response_model=IOCFeedResponse) +def iocs_lookup( + value: str = Query(..., min_length=1), + min_score: int = Query(default=0, ge=0, le=100), + type: str | None = Query(default=None), + limit: int = Query(default=1000, ge=1, le=10000), + _scope=Depends(require_feed_scope), + repo=Depends(get_repo), +): + """Exact-match single-IOC lookup — the #1 SOAR enrichment pattern. + + A single value normally aggregates to 0 or 1 row, so cursor pagination + is unnecessary here; next_cursor is always null. An empty ``value`` is + rejected (422) rather than silently returning the whole feed. + """ + filt = _build_filter(since=None, min_score=min_score, types=type, tag=None, case_id=None, limit=limit, cursor=None) + filt.value = value + rows = query_iocs(repo, filt) + return {"iocs": rows, "count": len(rows), "next_cursor": None} + + # ── CSV feed ──────────────────────────────────────────────────────────────── CSV_HEADER = [ @@ -164,6 +190,37 @@ def iocs_csv( return _conditional_response(request, body, "text/csv", _last_modified(rows)) +# ── CEF feed ──────────────────────────────────────────────────────────────── + + +@router.get("/iocs.cef") +def iocs_cef( + request: Request, + since: str | None = Query(default=None), + min_score: int = Query(default=0, ge=0, le=100), + type: str | None = Query(default=None), + tag: str | None = Query(default=None), + case_id: str | None = Query(default=None), + limit: int = Query(default=1000, ge=1, le=10000), + cursor: str | None = Query(default=None), + _scope=Depends(require_feed_scope), + repo=Depends(get_repo), +): + """CEF/syslog egress feed for ArcSight/QRadar/Sentinel-style SIEM ingestion. + + Uses the dedicated ``feed_rows_to_cef`` adapter (not the priority-score-gated + ``_events_from_iocs``), so the CEF output faithfully mirrors exactly the rows + the same filters (min_score/type/etc.) would return via iocs.json/csv — + including LOW-severity IOCs that a naive score/100 -> priority_score mapping + would otherwise silently drop. + """ + filt = _build_filter(since, min_score, type, tag, case_id, limit, cursor) + rows = query_iocs(repo, filt) + + body = feed_rows_to_cef(rows, hostname="pcap-hunter").encode("utf-8") + return _conditional_response(request, body, "text/plain", _last_modified(rows)) + + # ── STIX 2.1 feed ────────────────────────────────────────────────────────── @@ -218,14 +275,6 @@ def _to_stix_bundle(rows: list[dict]) -> dict: def _row_to_stix_pattern(r: dict) -> str | None: - t = r["type"] - v = r["value"].replace("'", "\\'") - if t == "ip": - return f"[ipv4-addr:value = '{v}']" - if t == "domain": - return f"[domain-name:value = '{v}']" - if t == "url": - return f"[url:value = '{v}']" - if t == "hash": - return f"[file:hashes.'SHA-256' = '{v}']" - return None + from app.utils.stix_export import ioc_to_stix_pattern + + return ioc_to_stix_pattern(r["type"], r["value"]) diff --git a/app/api/routers/pcaps.py b/app/api/routers/pcaps.py index 9cdd515..b713afb 100644 --- a/app/api/routers/pcaps.py +++ b/app/api/routers/pcaps.py @@ -7,12 +7,14 @@ import uuid from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile +from starlette.concurrency import run_in_threadpool from app.api.deps import get_queue, get_repo, get_settings, require_full_scope from app.api.models import JobLinks, PcapSubmissionForm, PcapSubmissionResponse from app.api.queue import JobSubmission, QueueFullError from app.api.validation import is_valid_pcap_magic from app.database.models import Case, CaseStatus, Severity +from app.utils.network_utils import is_safe_webhook_url router = APIRouter(prefix="/api/v1/pcaps", tags=["ingress"]) @@ -34,11 +36,18 @@ async def submit_pcap( osint_enabled: bool = Form(default=True), llm_enabled: bool = Form(default=True), pyshark_packet_limit: int | None = Form(default=None), + webhook_url: str | None = Form(default=None), _scope=Depends(require_full_scope), repo=Depends(get_repo), queue=Depends(get_queue), settings=Depends(get_settings), ) -> PcapSubmissionResponse: + # Fail fast, before any upload I/O: reject unsafe webhook targets up front. + # is_safe_webhook_url resolves DNS (socket.getaddrinfo, blocking), so run + # it off the event loop to avoid stalling other in-flight requests. + if webhook_url and not await run_in_threadpool(is_safe_webhook_url, webhook_url): + raise HTTPException(status_code=422, detail="invalid_webhook_url") + case_id = uuid.uuid4().hex[:8] out_path = _uploads_dir() / f"{case_id}.pcap" @@ -99,6 +108,7 @@ async def submit_pcap( "do_zeek": True, "pre_count": True, "pyshark_packet_limit": pyshark_packet_limit, + "webhook_url": webhook_url, } try: job_id = queue.enqueue( diff --git a/app/api/settings.py b/app/api/settings.py index 5e64cfc..1f52191 100644 --- a/app/api/settings.py +++ b/app/api/settings.py @@ -31,6 +31,9 @@ class APISettings: job_ttl_days: int require_https: bool cors_origins: list[str] + webhook_timeout_seconds: int + webhook_max_retries: int + webhook_secret: str | None = None @classmethod def from_env(cls) -> "APISettings": @@ -56,4 +59,7 @@ def from_env(cls) -> "APISettings": cors_origins=[ o.strip() for o in os.environ.get("PCAP_HUNTER_API_CORS_ORIGINS", "").split(",") if o.strip() ], + webhook_timeout_seconds=int(os.environ.get("PCAP_HUNTER_API_WEBHOOK_TIMEOUT_SECONDS", "10")), + webhook_max_retries=int(os.environ.get("PCAP_HUNTER_API_WEBHOOK_MAX_RETRIES", "2")), + webhook_secret=os.environ.get("PCAP_HUNTER_API_WEBHOOK_SECRET") or None, ) diff --git a/app/config.py b/app/config.py index 4b83c4b..715b82e 100644 --- a/app/config.py +++ b/app/config.py @@ -62,6 +62,14 @@ # below this; keep-first preserves true inter-arrival deltas (sampling would not). MAX_FLOW_SAMPLES = 5000 +# Per-log row cap on Zeek tables held in memory/session state. DNS analysis +# and the UI preview read these capped frames, so DNS/TLS fidelity is bounded +# by this value; JA3 extraction reads the full uncapped log via zeek_log_paths. +# Raised from the old hardcoded 2000 (which silently truncated busy captures); +# still bounded to protect session-state size. A WARNING_ZEEK_TRUNCATED is +# emitted when any log exceeds it. +ZEEK_TABLE_MAX_ROWS = 50000 + # Reverse DNS RDNS_CACHE_TTL_HOURS = 168 # 7 days RDNS_MAX_WORKERS = 10 # Concurrent rDNS lookups @@ -85,6 +93,21 @@ BEACON_MIN_PACKETS = 4 # Minimum packets to evaluate for beaconing BEACON_TOP_N = 20 # Number of top beacon candidates to report +# HTTP Analysis +HTTP_SUSPICIOUS_URI_LEN = 512 # URI length above this → suspicious (possible exfil/exploit) +HTTP_SUSPICIOUS_UA_TOKENS = frozenset( + { + "python-requests", + "curl", + "wget", + "powershell", + "go-http-client", + "nmap", + "sqlmap", + "masscan", + } +) + # Flow Analysis FLOW_ASYMMETRY_RATIO = 10 # Outbound/inbound ratio above this → suspicious FLOW_ASYMMETRY_MIN_BYTES = 1_000_000 # 1 MB minimum to flag asymmetry diff --git a/app/database/models.py b/app/database/models.py index 7d7e30b..4445ec5 100644 --- a/app/database/models.py +++ b/app/database/models.py @@ -156,6 +156,8 @@ class Analysis: yara_results: dict | None = None dns_analysis: dict | None = None tls_analysis: dict | None = None + attack_mapping: dict = field(default_factory=dict) + mitre_techniques: list[str] = field(default_factory=list) iocs: list[IOC] = field(default_factory=list) def to_dict(self) -> dict: @@ -172,6 +174,8 @@ def to_dict(self) -> dict: "yara_results": self.yara_results, "dns_analysis": self.dns_analysis, "tls_analysis": self.tls_analysis, + "attack_mapping": self.attack_mapping, + "mitre_techniques": list(self.mitre_techniques), "iocs": [ioc.to_dict() for ioc in self.iocs], } @@ -196,6 +200,8 @@ def from_dict(cls, data: dict) -> "Analysis": yara_results=data.get("yara_results"), dns_analysis=data.get("dns_analysis"), tls_analysis=data.get("tls_analysis"), + attack_mapping=data.get("attack_mapping", {}), + mitre_techniques=data.get("mitre_techniques", []), iocs=iocs, ) diff --git a/app/database/repository.py b/app/database/repository.py index 7ec6b0f..7262050 100644 --- a/app/database/repository.py +++ b/app/database/repository.py @@ -38,12 +38,17 @@ def _get_conn(self) -> sqlite3.Connection: """Get database connection.""" conn = sqlite3.connect(str(self._db_path)) conn.row_factory = sqlite3.Row + conn.execute("PRAGMA busy_timeout=30000") return conn def _init_schema(self): """Initialize database schema.""" conn = self._get_conn() try: + # Persisted once per database file; lets concurrent readers/writers + # (pool workers, API thread, Streamlit UI) coexist instead of + # failing immediately with "database is locked". + conn.execute("PRAGMA journal_mode=WAL") conn.executescript( """ -- Cases table @@ -73,6 +78,8 @@ def _init_schema(self): dns_json TEXT, tls_json TEXT ); + -- attack_json (ATT&CK mapping) is added below via idempotent ALTER + -- since this table may already exist from an earlier schema version. -- IOCs extracted from analyses CREATE TABLE IF NOT EXISTS iocs ( @@ -95,6 +102,14 @@ def _init_schema(self): updated_at TIMESTAMP ); + -- MITRE ATT&CK technique IDs derived per analysis (feeds the IOC feed's + -- mitre_techniques aggregation without post-SQL Python filtering). + CREATE TABLE IF NOT EXISTS analysis_techniques ( + analysis_id TEXT REFERENCES analyses(id) ON DELETE CASCADE, + technique_id TEXT NOT NULL, + UNIQUE(analysis_id, technique_id) + ); + -- Tags for organization CREATE TABLE IF NOT EXISTS tags ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -130,6 +145,7 @@ def _init_schema(self): CREATE INDEX IF NOT EXISTS idx_analyses_case ON analyses(case_id); CREATE INDEX IF NOT EXISTS idx_iocs_analysis ON iocs(analysis_id); CREATE INDEX IF NOT EXISTS idx_iocs_type_value ON iocs(ioc_type, value); + CREATE INDEX IF NOT EXISTS idx_analysis_techniques ON analysis_techniques(analysis_id); CREATE INDEX IF NOT EXISTS idx_notes_case ON notes(case_id); CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status); CREATE INDEX IF NOT EXISTS idx_jobs_case ON jobs(case_id); @@ -143,6 +159,12 @@ def _init_schema(self): conn.commit() except sqlite3.OperationalError: pass # column already exists + + try: + conn.execute("ALTER TABLE analyses ADD COLUMN attack_json TEXT") + conn.commit() + except sqlite3.OperationalError: + pass # column already exists finally: conn.close() @@ -341,6 +363,10 @@ def delete_case(self, case_id: str) -> bool: "DELETE FROM iocs WHERE analysis_id IN (SELECT id FROM analyses WHERE case_id = ?)", (case_id,), ) + conn.execute( + "DELETE FROM analysis_techniques WHERE analysis_id IN (SELECT id FROM analyses WHERE case_id = ?)", + (case_id,), + ) conn.execute( "DELETE FROM notes WHERE case_id = ? OR analysis_id IN (SELECT id FROM analyses WHERE case_id = ?)", (case_id, case_id), @@ -369,6 +395,7 @@ def clear_all(self) -> bool: try: conn.execute("DELETE FROM notes") conn.execute("DELETE FROM iocs") + conn.execute("DELETE FROM analysis_techniques") conn.execute("DELETE FROM analyses") conn.execute("DELETE FROM case_tags") conn.execute("DELETE FROM tags") @@ -406,13 +433,14 @@ def save_analysis(self, analysis: Analysis) -> str: yara_json = self._compress_json(analysis.yara_results) if analysis.yara_results else None dns_json = self._compress_json(analysis.dns_analysis) if analysis.dns_analysis else None tls_json = self._compress_json(analysis.tls_analysis) if analysis.tls_analysis else None + attack_json = self._compress_json(analysis.attack_mapping) if analysis.attack_mapping else None conn.execute( """ INSERT OR REPLACE INTO analyses (id, case_id, pcap_path, pcap_hash, packet_count, analyzed_at, - features_json, osint_json, report_md, yara_json, dns_json, tls_json) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + features_json, osint_json, report_md, yara_json, dns_json, tls_json, attack_json) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( analysis.id, @@ -427,6 +455,7 @@ def save_analysis(self, analysis: Analysis) -> str: yara_json, dns_json, tls_json, + attack_json, ), ) @@ -434,6 +463,17 @@ def save_analysis(self, analysis: Analysis) -> str: for ioc in analysis.iocs: self._save_ioc(conn, analysis.id, ioc) + # Save MITRE ATT&CK technique associations. Full replace (delete then + # insert) rather than accumulate: unlike iocs, a re-analysis's technique + # set should reflect only the most recent save, mirroring the + # full-replace semantics of attack_json/features_json on this row. + conn.execute("DELETE FROM analysis_techniques WHERE analysis_id = ?", (analysis.id,)) + for technique_id in analysis.mitre_techniques: + conn.execute( + "INSERT OR IGNORE INTO analysis_techniques (analysis_id, technique_id) VALUES (?, ?)", + (analysis.id, technique_id), + ) + conn.commit() logger.info("Saved analysis: %s", analysis.id) return analysis.id @@ -702,6 +742,9 @@ def _row_to_analysis(self, row: dict, conn: sqlite3.Connection) -> Analysis: yara_results = self._decompress_json(row.get("yara_json")) dns_analysis = self._decompress_json(row.get("dns_json")) tls_analysis = self._decompress_json(row.get("tls_json")) + # attack_json may be absent (NULL, or column missing on rows saved before + # this column existed) — always default to {} rather than None. + attack_mapping = self._decompress_json(row.get("attack_json")) or {} # Load IOCs ioc_rows = conn.execute("SELECT * FROM iocs WHERE analysis_id = ?", (row["id"],)).fetchall() @@ -716,6 +759,12 @@ def _row_to_analysis(self, row: dict, conn: sqlite3.Connection) -> Analysis: for r in ioc_rows ] + # Load MITRE ATT&CK technique IDs + technique_rows = conn.execute( + "SELECT technique_id FROM analysis_techniques WHERE analysis_id = ?", (row["id"],) + ).fetchall() + mitre_techniques = [r["technique_id"] for r in technique_rows] + return Analysis( id=row["id"], case_id=row.get("case_id") or "", @@ -729,6 +778,8 @@ def _row_to_analysis(self, row: dict, conn: sqlite3.Connection) -> Analysis: yara_results=yara_results, dns_analysis=dns_analysis, tls_analysis=tls_analysis, + attack_mapping=attack_mapping, + mitre_techniques=mitre_techniques, iocs=iocs, ) diff --git a/app/main.py b/app/main.py index fceee9c..7e87bcc 100644 --- a/app/main.py +++ b/app/main.py @@ -46,6 +46,7 @@ make_results_panel, make_tabs, render_active_filters, + render_attack_mapping, render_batch_summary, render_carved, render_chart_hint, @@ -54,6 +55,7 @@ render_dns_analysis, render_flow_asymmetry, render_flows, + render_http_analysis, render_hunting_checklist, render_ioc_search, render_ja3, @@ -226,6 +228,8 @@ def _run_single_pcap_pipeline( beacon_df=beacon_df if isinstance(beacon_df, pd.DataFrame) else None, dns_analysis=result.dns_analysis or {}, tls_analysis=result.tls_analysis or {}, + http_analysis=result.http_analysis or {}, + attack_mapping=result.attack_mapping or {}, packet_count=result.packet_count, ) @@ -324,11 +328,13 @@ def _run_single_pcap_pipeline( ("runtime_logs", []), ("map_reset_counter", 0), ("dns_analysis", None), + ("http_analysis", None), ("tls_analysis", None), ("yara_results", None), ("correlations", None), ("flow_asymmetry", None), ("port_anomalies", None), + ("attack_mapping", {}), ("__pcap_paths", []), ("__batch_mode", False), ("__batch_result", None), @@ -345,7 +351,8 @@ def _run_single_pcap_pipeline( with st.container(border=True): st.markdown("##### 🚀 Getting started") st.markdown( - "1. **Upload a PCAP** below (or type a container path).\n" + "1. **Upload a PCAP** below (or type a container path) — or click " + "**Load demo capture** to try a bundled synthetic sample.\n" "2. Click **Extract & Analyze** — the 10-stage pipeline parses packets (tshark/PyShark), " "runs Zeek, hunts DNS/TLS/beaconing anomalies, carves HTTP payloads, scans with YARA, " "and enriches IOCs via OSINT.\n" @@ -368,6 +375,18 @@ def _run_single_pcap_pipeline( with col_b: pcap_path_text = st.text_input("...or type a container path (e.g., /data/capture.pcap)", value="") + _demo_path = validate_pcap_path(str(pathlib.Path("pcaps/demo.pcap").resolve())) + if _demo_path: + if st.button( + "Load demo capture", + key="_load_demo", + help="Analyze a bundled synthetic capture (DNS/HTTP/beacon) to try the pipeline.", + ): + st.session_state["__pcap_path"] = _demo_path + st.session_state["__pcap_paths"] = [_demo_path] + st.session_state["__batch_mode"] = False + st.rerun() + ensure_dir(C.DATA_DIR) ensure_dir(C.ZEEK_DIR) ensure_dir(C.CARVE_DIR) @@ -437,6 +456,7 @@ def _run_single_pcap_pipeline( ("Zeek processing", do_zeek), ("DNS Analysis", do_zeek), # Requires Zeek dns.log ("TLS Certificate Analysis", do_zeek), # Requires Zeek ssl.log + ("HTTP Analysis", do_zeek), # Requires Zeek http.log ("Beaconing ranking", True), ("HTTP carving (tshark)", do_carve), ("YARA Scanning", do_carve and do_yara), # Requires carved files @@ -462,11 +482,14 @@ def _run_single_pcap_pipeline( "__pcap_path": pcap_path, "__pcap_paths": pcap_paths or [pcap_path], "dns_analysis": None, + "http_analysis": None, "tls_analysis": None, "yara_results": None, "correlations": None, "flow_asymmetry": None, "port_anomalies": None, + "attack_timeline": [], + "attack_mapping": {}, "__batch_result": None, } ) @@ -529,6 +552,7 @@ def _run_single_pcap_pipeline( ("Zeek processing", do_zeek), ("DNS Analysis", do_zeek), ("TLS Certificate Analysis", do_zeek), + ("HTTP Analysis", do_zeek), ("Beaconing ranking", True), ("HTTP carving (tshark)", do_carve), ("YARA Scanning", do_carve and do_yara), @@ -616,6 +640,16 @@ def _run_single_pcap_pipeline( st.session_state["beacon_df"] = batch_result.merged_beacons st.session_state["dns_analysis"] = batch_result.aggregated_dns st.session_state["tls_analysis"] = batch_result.aggregated_tls + # HTTP findings merged across every file (app.pipeline.batch.aggregate_http_analysis) + # — previously only mirrored the first successful file's findings. + st.session_state["http_analysis"] = batch_result.aggregated_http + # Rebuilt further below, once merged features/dns/tls/beacon_df are + # all available, from the AGGREGATED batch inputs (mirrors the + # per-file ATTACKMapper call in app/pipeline/runner.py). Previously + # this mirrored only the first successful file's mapping, which + # under-reported ATT&CK techniques found in files 2..N. Placeholder + # here in case that rebuild raises. + st.session_state["attack_mapping"] = {} # Carved payloads concatenated across all successful files st.session_state["carved"] = [ item for r in batch_result.pcap_results if not r.error for item in r.carved_items @@ -658,6 +692,7 @@ def _run_single_pcap_pipeline( dns_analysis=st.session_state.get("dns_analysis"), tls_analysis=st.session_state.get("tls_analysis"), yara_results=st.session_state.get("yara_results"), + http_analysis=st.session_state.get("http_analysis"), ) st.session_state["correlations"] = correlations @@ -667,6 +702,44 @@ def _run_single_pcap_pipeline( except Exception as e: logger.warning("Post-analysis failed: %s", e) + try: + from app.analysis.narrator import AttackNarrator + + _timeline = AttackNarrator().create_timeline( + features=st.session_state.get("features"), + dns_analysis=st.session_state.get("dns_analysis"), + yara_results=st.session_state.get("yara_results"), + beacon_results=( + get_df_state("beacon_df").to_dict("records") if not get_df_state("beacon_df").empty else [] + ), + tls_analysis=st.session_state.get("tls_analysis"), + ) + st.session_state["attack_timeline"] = [e.to_dict() for e in _timeline] + except Exception as e: + logger.debug("timeline precompute failed: %s", e) + st.session_state["attack_timeline"] = [] + + # Rebuild the ATT&CK mapping from the AGGREGATED batch inputs (same + # merged features/dns/tls/beacon_df used for the timeline above), + # mirroring the per-file ATTACKMapper call in app/pipeline/runner.py. + # Replaces the earlier first-successful-file-only placeholder so the + # dashboard ATT&CK panel and batch-quick-saved mitre_techniques + # reflect techniques detected across every file, not just file 1. + try: + from app.threat_intel import ATTACKMapper + + _mapping = ATTACKMapper().map_analysis( + features=st.session_state.get("features"), + dns_analysis=st.session_state.get("dns_analysis"), + tls_analysis=st.session_state.get("tls_analysis"), + beacon_results=( + get_df_state("beacon_df").to_dict("records") if not get_df_state("beacon_df").empty else [] + ), + ) + st.session_state["attack_mapping"] = _mapping.to_dict() + except Exception as e: + logger.warning("Batch ATT&CK mapping rebuild failed: %s", e) + batch_tracker.finish_all( f"Batch complete: {batch_result.summary['successful']}/{batch_result.summary['total_files']} files." ) @@ -704,6 +777,8 @@ def _run_single_pcap_pipeline( st.session_state["carved"] = result.carved_items st.session_state["dns_analysis"] = result.dns_analysis or None st.session_state["tls_analysis"] = result.tls_analysis or None + st.session_state["http_analysis"] = result.http_analysis or None + st.session_state["attack_mapping"] = result.attack_mapping or {} _precompute_dash_aggregates(features.get("flows")) # a fresh run supersedes any restored case @@ -732,6 +807,7 @@ def _run_single_pcap_pipeline( dns_analysis=st.session_state.get("dns_analysis"), tls_analysis=st.session_state.get("tls_analysis"), yara_results=st.session_state.get("yara_results"), + http_analysis=st.session_state.get("http_analysis"), ) st.session_state["correlations"] = correlations @@ -741,6 +817,23 @@ def _run_single_pcap_pipeline( except Exception as e: logger.warning("Post-analysis failed: %s", e) + try: + from app.analysis.narrator import AttackNarrator + + _timeline = AttackNarrator().create_timeline( + features=st.session_state.get("features"), + dns_analysis=st.session_state.get("dns_analysis"), + yara_results=st.session_state.get("yara_results"), + beacon_results=( + get_df_state("beacon_df").to_dict("records") if not get_df_state("beacon_df").empty else [] + ), + tls_analysis=st.session_state.get("tls_analysis"), + ) + st.session_state["attack_timeline"] = [e.to_dict() for e in _timeline] + except Exception as e: + logger.debug("timeline precompute failed: %s", e) + st.session_state["attack_timeline"] = [] + # ---- LLM REPORT (shared for single & batch) ---- features = st.session_state.get("features") or {} zeek_tables = st.session_state.get("zeek_tables") or {} @@ -1249,29 +1342,44 @@ def _run_single_pcap_pipeline( st.plotly_chart(fig, use_container_width=True) render_chart_hint("Node size = connections. Color: blue=low, red=high threat.") - # Attack timeline (full-width, if available) - try: - from app.analysis.narrator import AttackNarrator - - narrator = AttackNarrator() - timeline = narrator.create_timeline( - features=feats, - dns_analysis=st.session_state.get("dns_analysis"), - yara_results=st.session_state.get("yara_results"), - beacon_results=( - get_df_state("beacon_df").to_dict("records") if not get_df_state("beacon_df").empty else [] - ), - tls_analysis=st.session_state.get("tls_analysis"), - ) - if timeline: - timeline_dicts = [e.to_dict() for e in timeline] - st.plotly_chart( - plot_attack_timeline(timeline_dicts), - use_container_width=True, + # Attack timeline (full-width, if available) — stored at pipeline-completion + # time (see post-analysis blocks above) so it survives without needing to + # recompute here, and so the PDF report can reuse the same data. + # + # A restored case resets attack_timeline to [] (it isn't a persisted Analysis + # column — see app/ui/cases_tab.py::_restore_analysis_to_session) even though + # features/dns/tls/beacon data ARE restored. Lazily recompute from the + # CURRENT restored state in that case so restored cases don't lose the + # timeline chart; a fresh run's own post-analysis block always populates + # attack_timeline directly, so this only fires for the restore path. + timeline_dicts = st.session_state.get("attack_timeline") or [] + if not timeline_dicts and st.session_state.get("features"): + try: + from app.analysis.narrator import AttackNarrator + + _tl = AttackNarrator().create_timeline( + features=st.session_state.get("features"), + dns_analysis=st.session_state.get("dns_analysis"), + yara_results=st.session_state.get("yara_results"), + beacon_results=( + get_df_state("beacon_df").to_dict("records") if not get_df_state("beacon_df").empty else [] + ), + tls_analysis=st.session_state.get("tls_analysis"), ) - render_chart_hint("Diamond markers show events by severity and time.") - except Exception as e: - logger.debug("chart rendering failed: %s", e) + timeline_dicts = [e.to_dict() for e in _tl] + # Cache back so we don't re-run the narrator on every dashboard + # rerun (filter toggles, etc.) for a restored case. Safe against the + # cross-run stale-leak: a fresh run's reset clears this key, and this + # value was just derived from the CURRENT restored features. + st.session_state["attack_timeline"] = timeline_dicts + except Exception as e: + logger.debug("timeline lazy recompute failed: %s", e) + if timeline_dicts: + st.plotly_chart( + plot_attack_timeline(timeline_dicts), + use_container_width=True, + ) + render_chart_hint("Diamond markers show events by severity and time.") # --- Traffic profiling charts --- if filtered_flows: @@ -1363,6 +1471,16 @@ def _run_single_pcap_pipeline( st.markdown("---") + # MITRE ATT&CK Mapping (session state holds the dict form; reconstruct + # the AttackMapping object the renderer expects) + _attack = st.session_state.get("attack_mapping") + if _attack: + from app.threat_intel import AttackMapping + + render_attack_mapping(st.container(), AttackMapping.from_dict(_attack)) + + st.markdown("---") + # Hunting checklist render_hunting_checklist( st.container(), @@ -1446,6 +1564,7 @@ def _run_single_pcap_pipeline( correlations=st.session_state.get("correlations"), geoip_data=_geoip_data or None, attack_timeline=st.session_state.get("attack_timeline"), + attack_mapping=st.session_state.get("attack_mapping"), ) if pdf_report: @@ -1486,6 +1605,7 @@ def _run_single_pcap_pipeline( feats = st.session_state.get("features") or {} render_flows(results_panel, feats.get("flows")) render_dns_analysis(results_panel, st.session_state.get("dns_analysis")) + render_http_analysis(results_panel, st.session_state.get("http_analysis")) render_tls_certificates(results_panel, st.session_state.get("tls_analysis")) render_ja3( results_panel, diff --git a/app/pipeline/batch.py b/app/pipeline/batch.py index d1d2d1c..ba5fffb 100644 --- a/app/pipeline/batch.py +++ b/app/pipeline/batch.py @@ -17,6 +17,8 @@ import pandas as pd +from app.pipeline.http_analysis import MAX_HTTP_RESULTS + logger = logging.getLogger(__name__) # --- Constants --- @@ -85,6 +87,12 @@ class PCAPResult: beacon_df: pd.DataFrame | None = None dns_analysis: dict[str, Any] = field(default_factory=dict) tls_analysis: dict[str, Any] = field(default_factory=dict) + # HTTP request analysis (suspicious UA/cleartext creds/suspicious URI + # heuristics) from app.pipeline.http_analysis.analyze_http. + http_analysis: dict[str, Any] = field(default_factory=dict) + # MITRE ATT&CK mapping (AttackMapping.to_dict() shape) from the runner — + # may be {} on mapper failure; treated shape-agnostically downstream. + attack_mapping: dict = field(default_factory=dict) packet_count: int = 0 error: str | None = None @@ -463,6 +471,78 @@ def aggregate_tls_analysis(results: list[PCAPResult]) -> dict[str, Any]: } +def aggregate_http_analysis(results: list[PCAPResult]) -> dict[str, Any]: + """ + Aggregate HTTP request analysis from multiple PCAPs. + + Mirrors the shape app.pipeline.http_analysis.analyze_http returns for a + single file, so downstream consumers (correlation, ATT&CK mapping, the + dashboard HTTP panel) don't need batch-specific handling. + + Args: + results: List of PCAPResult + + Returns: + Aggregated HTTP analysis dictionary. + """ + total_requests = 0 + # Summed per-file, like aggregate_dns_analysis's total_records — an + # approximation when the same host appears in multiple files, since only + # per-file counts (not raw host sets) are persisted on PCAPResult. + unique_hosts = 0 + methods: Counter = Counter() + status_codes: Counter = Counter() + all_uas = [] + all_creds = [] + all_uris = [] + + for r in results: + if r.error or not r.http_analysis: + continue + + http = r.http_analysis + if http.get("skipped") or http.get("error"): + continue + + total_requests += http.get("total_requests", 0) + unique_hosts += http.get("unique_hosts", 0) + + for method, count in (http.get("methods") or {}).items(): + methods[method] += count + for code, count in (http.get("status_codes") or {}).items(): + status_codes[code] += count + + for ua in http.get("suspicious_user_agents", []) or []: + ua_copy = dict(ua) + ua_copy["_source_file"] = r.filename + all_uas.append(ua_copy) + + for cred in http.get("cleartext_credentials", []) or []: + cred_copy = dict(cred) + cred_copy["_source_file"] = r.filename + all_creds.append(cred_copy) + + for uri in http.get("suspicious_uris", []) or []: + uri_copy = dict(uri) + uri_copy["_source_file"] = r.filename + all_uris.append(uri_copy) + + return { + "total_requests": total_requests, + "unique_hosts": unique_hosts, + "methods": dict(methods), + "status_codes": dict(status_codes), + "suspicious_user_agents": all_uas[:MAX_HTTP_RESULTS], + "cleartext_credentials": all_creds[:MAX_HTTP_RESULTS], + "suspicious_uris": all_uris[:MAX_HTTP_RESULTS], + "alerts": { + "suspicious_ua_count": len(all_uas), + "cleartext_cred_count": len(all_creds), + "suspicious_uri_count": len(all_uris), + }, + } + + @dataclass class BatchResult: """Complete batch processing result.""" @@ -474,6 +554,7 @@ class BatchResult: merged_beacons: pd.DataFrame aggregated_dns: dict[str, Any] aggregated_tls: dict[str, Any] + aggregated_http: dict[str, Any] summary: dict[str, Any] @@ -559,6 +640,7 @@ def merge_all(self) -> BatchResult: merged_beacons = merge_beacon_candidates(self.results) aggregated_dns = aggregate_dns_analysis(self.results) aggregated_tls = aggregate_tls_analysis(self.results) + aggregated_http = aggregate_http_analysis(self.results) # Build summary successful = sum(1 for r in self.results if not r.error) @@ -602,6 +684,7 @@ def merge_all(self) -> BatchResult: merged_beacons=merged_beacons, aggregated_dns=aggregated_dns, aggregated_tls=aggregated_tls, + aggregated_http=aggregated_http, summary=summary, ) diff --git a/app/pipeline/beacon.py b/app/pipeline/beacon.py index 1aa2228..b243cf3 100644 --- a/app/pipeline/beacon.py +++ b/app/pipeline/beacon.py @@ -53,6 +53,53 @@ "5353": 0.15, # mDNS } +# Ports where a very regular, high-confidence signal is allowed to soften +# (not erase) the benign-service penalty above. Restricted to HTTPS/HTTP — +# the transports real C2 frameworks actually abuse to blend in. DNS/NTP and +# the other BENIGN_SERVICE_PORTS entries are deliberately excluded: they are +# genuinely periodic infrastructure traffic by design, not something C2 uses +# as cover, so no amount of "regularity" should un-suppress them. +SOFTENABLE_BENIGN_PORTS: frozenset[str] = frozenset({"443"}) + +# Average packet size (bytes) below which a flow's payloads look C2-like. +# Real C2 beacons are SMALL packets; CDN/streaming keep-alives carry large +# payloads. Reused by both the softening gate below and the large-payload +# penalty in rank_beaconing. +C2_PAYLOAD_MAX_BYTES = 500 + +# Softened multiplier applied instead of BENIGN_SERVICE_PORTS[dport] when a +# flow on a softenable port meets ALL THREE C2-like conditions: +# 1. very high-confidence — raw score >= 0.85 +# 2. essentially jitter-free — jitter_pct <= 15 +# 3. small average payload — pkt_lens present and mean < C2_PAYLOAD_MAX_BYTES +# That combination is exactly what genuine small-packet HTTPS C2 looks like. +# Tuned against tests/test_beacon.py so a perfectly periodic small-payload +# 443 flow (raw score ~1.0) clears BEACON_SCORE_THRESHOLD (0.6) with margin: +# 1.0 * 0.69 = 0.69 > 0.6. The naive "penalty * 4" (0.15 -> 0.6) is NOT enough: +# a 0.9-raw flow would land at 0.9 * 0.6 = 0.54, still below threshold. +# +# 0.69, not 0.7: the ATT&CK mapper's beacon->C2 rule (DETECTION_RULES +# ["beacon_score"]["threshold"] in app/threat_intel/attack_mapping.py) fires +# at score >= 0.7 and auto-sets overall_severity="high". Since the max +# softened output is final_score(<=1.0) * SOFTENED_PENALTY, 0.7 would let a +# perfectly periodic, zero-jitter softened beacon land EXACTLY on the ATT&CK +# threshold — a benign small-payload HTTPS flow with flawless periodicity +# would then get flagged as T1071.001 C2 at HIGH severity purely from +# timing, which is exactly the false positive this softening bucket exists +# to avoid. 0.69 keeps the max softened score strictly below 0.7 (decoupled +# from the ATT&CK rule) while staying comfortably above the 0.6 beacon +# candidate floor and the 0.5 correlation ingest gate, so genuine HTTPS +# beacons still surface as candidates — they just don't trip the ATT&CK +# C2 technique on timing alone. +# +# The small-payload condition (3) is load-bearing: a machine-regular CDN +# heartbeat (zero jitter, LARGE 1200-byte payloads) is indistinguishable +# from C2 by timing+jitter alone, so requiring small payloads keeps those +# fully penalised. When pkt_lens is absent/empty we CANNOT confirm the flow +# is C2-like, so we conservatively do NOT soften. Ordinary jittery HTTPS +# keep-alives also fail the >=0.85 gate and keep the full 0.15 penalty. +SOFTENED_PENALTY = 0.69 + def periodicity_score(ts: list[float]) -> dict[str, object]: """Score timestamp periodicity for beaconing detection. @@ -205,10 +252,28 @@ def rank_beaconing(flows: list[dict[str, object]], top_n: int = 20) -> pd.DataFr # Applied BEFORE any threshold checks so that benign traffic # is scored down before it can appear as a candidate. + pkt_lens = f.get("pkt_lens", []) + # 1. Benign service ports FIRST — this is the most common FP source # (e.g., HTTPS keep-alives to CDNs on port 443) if dport in BENIGN_SERVICE_PORTS: - final_score *= BENIGN_SERVICE_PORTS[dport] + penalty = BENIGN_SERVICE_PORTS[dport] + # A very regular, high-confidence, SMALL-PAYLOAD signal on a + # softenable port (443) is exactly what real HTTPS C2 looks like + # — soften the penalty so it can still surface above threshold, + # instead of guaranteeing a sub-threshold score. Ordinary HTTPS + # keep-alives score moderately/show jitter, and CDN heartbeats + # carry large payloads — both keep the full penalty. + jitter_pct = jitter.get("jitter_pct") + if jitter_pct is None: + jitter_pct = 100.0 + # Small average payload is required and conservative: when pkt_lens + # is absent/empty we cannot confirm the flow is C2-like, so we do + # NOT soften. + small_payload = bool(pkt_lens) and (sum(pkt_lens) / len(pkt_lens) < C2_PAYLOAD_MAX_BYTES) + if dport in SOFTENABLE_BENIGN_PORTS and final_score >= 0.85 and jitter_pct <= 15 and small_payload: + penalty = SOFTENED_PENALTY + final_score *= penalty # 2. Well-known infrastructure IPs (DNS resolvers, NTP servers) if dst in INFRA_ALLOWLIST or src in INFRA_ALLOWLIST: @@ -220,10 +285,9 @@ def rank_beaconing(flows: list[dict[str, object]], top_n: int = 20) -> pd.DataFr # 4. High-volume large-payload flows (streaming/downloads, not C2) # Real C2 beacons are small, infrequent packets. - pkt_lens = f.get("pkt_lens", []) if pkt_lens and len(ts) > 200: avg_pkt_size = sum(pkt_lens) / len(pkt_lens) - if avg_pkt_size > 500: + if avg_pkt_size > C2_PAYLOAD_MAX_BYTES: final_score *= 0.25 rows.append( diff --git a/app/pipeline/http_analysis.py b/app/pipeline/http_analysis.py new file mode 100644 index 0000000..ae053b7 --- /dev/null +++ b/app/pipeline/http_analysis.py @@ -0,0 +1,308 @@ +"""HTTP request analysis for threat detection. + +Provides detection for: +- Suspicious User-Agent strings (missing UA, known scripting/scanning tools) +- Cleartext credentials (Basic-auth username observed in Zeek http.log) +- Suspicious URIs (risky file extensions, oversized URIs, raw-IP file downloads) +""" + +from __future__ import annotations + +import logging +import re +from collections import Counter +from dataclasses import dataclass +from typing import Any + +import pandas as pd + +from app.config import HTTP_SUSPICIOUS_UA_TOKENS, HTTP_SUSPICIOUS_URI_LEN +from app.pipeline.state import PhaseHandle +from app.pipeline.zeek import load_zeek_any + +logger = logging.getLogger(__name__) + +# --- Result Limits --- +MAX_HTTP_RESULTS = 100 # Cap each result list to bound output size + +# Raw dotted-quad IPv4 pattern, used by the raw-IP-host heuristic. +IPV4_PATTERN = re.compile(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$") + +# File extensions associated with executables/scripts — risky when served over plain HTTP. +RISKY_EXTENSIONS = (".exe", ".dll", ".ps1", ".scr", ".bin") + +# Values Zeek/ASCII logs use to mean "missing" for a field. +_MISSING_VALUES = {"-", "nan", "none", ""} + + +@dataclass +class HTTPRequest: + """Parsed Zeek http.log record.""" + + ts: float + src: str + dst: str + host: str + uri: str + method: str + user_agent: str + username: str + status_code: int + + +def _clean(value: Any) -> str: + """Normalize a raw Zeek field to a stripped string; '-'/NaN/None/empty become ''.""" + if value is None: + return "" + text = str(value).strip() + if text.lower() in _MISSING_VALUES: + return "" + return text + + +def _clean_int(value: Any) -> int: + """Best-effort int parse; missing/unparseable values become 0.""" + text = _clean(value) + if not text: + return 0 + try: + return int(float(text)) + except (ValueError, TypeError): + return 0 + + +def parse_http_log(df: pd.DataFrame) -> list[HTTPRequest]: + """ + Parse Zeek http.log DataFrame into HTTPRequest objects. + + Uses dual-name column access (dotted Zeek names with underscore fallback) + and is string-tolerant: ASCII-mode logs deliver every field as text, with + "-" as the literal missing-value marker. + + Args: + df: DataFrame from Zeek http.log + + Returns: + List of HTTPRequest objects + """ + records = [] + + for row in df.to_dict(orient="records"): + try: + try: + ts = float(row.get("ts", 0) or 0) + except (ValueError, TypeError): + ts = 0.0 + + records.append( + HTTPRequest( + ts=ts, + src=_clean(row.get("id.orig_h", row.get("id_orig_h", ""))), + dst=_clean(row.get("id.resp_h", row.get("id_resp_h", ""))), + host=_clean(row.get("host", "")), + uri=_clean(row.get("uri", "")), + method=_clean(row.get("method", "")), + user_agent=_clean(row.get("user_agent", "")), + username=_clean(row.get("username", "")), + status_code=_clean_int(row.get("status_code", "")), + ) + ) + except (ValueError, TypeError) as e: + logger.debug("Failed to parse HTTP record: %s", e) + continue + + return records + + +def detect_suspicious_ua(req: HTTPRequest) -> str | None: + """ + Detect a suspicious User-Agent on an HTTP request. + + Flags: + - Empty/missing User-Agent on a request that has a Host (incomplete/scripted client) + - User-Agent naming a known scripting/scanning tool (case-insensitive substring) + + Args: + req: Parsed HTTP request + + Returns: + Reason string if suspicious, else None + """ + if not req.user_agent: + if req.host: + return "missing user-agent" + return None + + ua_lower = req.user_agent.lower() + for token in HTTP_SUSPICIOUS_UA_TOKENS: + if token in ua_lower: + return f"known tool user-agent ({token})" + + return None + + +def detect_cleartext_credentials(req: HTTPRequest) -> dict[str, str] | None: + """ + Detect cleartext (Basic-auth) credentials on an HTTP request. + + Only the username is emitted — never the password, even if the log carried one. + + Args: + req: Parsed HTTP request + + Returns: + {"host", "uri", "username"} dict if credentials were seen, else None + """ + if not req.username: + return None + return {"host": req.host, "uri": req.uri, "username": req.username} + + +def detect_suspicious_uri(req: HTTPRequest) -> str | None: + """ + Detect a suspicious URI on an HTTP request. + + Flags: + - Risky file extension (.exe/.dll/.ps1/.scr/.bin) + - Very long URI (> HTTP_SUSPICIOUS_URI_LEN chars) + - Raw IPv4 host serving what looks like a file download + + Args: + req: Parsed HTTP request + + Returns: + Reason string if suspicious, else None + """ + if not req.uri: + return None + + uri_lower = req.uri.lower() + reasons = [] + + matched_ext = next((ext for ext in RISKY_EXTENSIONS if ext in uri_lower), None) + if matched_ext: + reasons.append(f"risky file extension ({matched_ext})") + + if len(req.uri) > HTTP_SUSPICIOUS_URI_LEN: + reasons.append(f"long URI ({len(req.uri)} chars)") + + if matched_ext and IPV4_PATTERN.match(req.host): + reasons.append("raw-IP host serving file download") + + return "; ".join(reasons) if reasons else None + + +def analyze_http( + zeek_tables: dict[str, pd.DataFrame], + http_log_path: str | None = None, + phase: PhaseHandle | None = None, +) -> dict[str, Any]: + """ + Comprehensive HTTP request analysis from Zeek http.log. + + Args: + zeek_tables: Dictionary of Zeek log DataFrames (row-capped, see ZEEK_TABLE_MAX_ROWS) + http_log_path: Optional path to the on-disk http.log. When provided, the full + uncapped log is read via load_zeek_any instead of the capped in-memory table. + phase: PhaseHandle for progress updates + + Returns: + Dictionary with HTTP analysis results + """ + if phase and phase.should_skip(): + phase.done("HTTP analysis skipped.") + return {"skipped": True} + + if phase: + phase.set(5, "Parsing HTTP logs...") + + http_df: pd.DataFrame | None = None + if http_log_path: + try: + http_df = load_zeek_any(http_log_path) + except Exception as e: + logger.debug("Failed to load full http.log from %s: %s", http_log_path, e) + http_df = None + + if http_df is None: + http_df = zeek_tables.get("http.log") + + if http_df is None: + if phase: + phase.done("No HTTP data available.") + return {"skipped": True} + + if http_df.empty: + if phase: + phase.done("No HTTP data available.") + return {"error": "No HTTP log data", "records": 0} + + records = parse_http_log(http_df) + if not records: + if phase: + phase.done("No valid HTTP records found.") + return {"error": "No HTTP log data", "records": 0} + + if phase: + phase.set(30, f"Analyzing {len(records)} HTTP requests...") + + methods = Counter(r.method for r in records if r.method) + status_codes = Counter(str(r.status_code) for r in records if r.status_code) + unique_hosts = len({r.host for r in records if r.host}) + + suspicious_uas = [] + cleartext_creds = [] + suspicious_uris = [] + + total = len(records) + for i, r in enumerate(records): + if phase and i % 500 == 0: + pct = 30 + int((i / total) * 50) + phase.set(pct, f"Scanning request {i + 1}/{total}...") + + ua_reason = detect_suspicious_ua(r) + if ua_reason: + suspicious_uas.append({"host": r.host, "user_agent": r.user_agent, "uri": r.uri, "reason": ua_reason}) + + cred = detect_cleartext_credentials(r) + if cred: + cleartext_creds.append(cred) + + uri_reason = detect_suspicious_uri(r) + if uri_reason: + suspicious_uris.append({"host": r.host, "uri": r.uri, "reason": uri_reason}) + + if phase: + phase.set(85, "Finalizing HTTP analysis...") + + result = { + "total_requests": len(records), + "unique_hosts": unique_hosts, + "methods": dict(methods), + "status_codes": dict(status_codes), + "suspicious_user_agents": suspicious_uas[:MAX_HTTP_RESULTS], + "cleartext_credentials": cleartext_creds[:MAX_HTTP_RESULTS], + "suspicious_uris": suspicious_uris[:MAX_HTTP_RESULTS], + "alerts": { + "suspicious_ua_count": len(suspicious_uas), + "cleartext_cred_count": len(cleartext_creds), + "suspicious_uri_count": len(suspicious_uris), + }, + } + + if phase: + alerts = result["alerts"] + alert_msg = [] + if alerts["suspicious_ua_count"]: + alert_msg.append(f"{alerts['suspicious_ua_count']} suspicious UA") + if alerts["cleartext_cred_count"]: + alert_msg.append(f"{alerts['cleartext_cred_count']} cleartext creds") + if alerts["suspicious_uri_count"]: + alert_msg.append(f"{alerts['suspicious_uri_count']} suspicious URIs") + + summary = f"Analyzed {len(records)} HTTP requests, {unique_hosts} hosts." + if alert_msg: + summary += f" Alerts: {', '.join(alert_msg)}." + phase.done(summary) + + return result diff --git a/app/pipeline/osint.py b/app/pipeline/osint.py index 05e4cf5..b06183d 100644 --- a/app/pipeline/osint.py +++ b/app/pipeline/osint.py @@ -13,6 +13,7 @@ from app.pipeline.state import PhaseHandle from app.security.opsec import hardened_session from app.utils.common import is_public_ipv4, resolve_ip +from app.utils.network_utils import is_enrichable_domain logger = logging.getLogger(__name__) @@ -432,7 +433,7 @@ def enrich( prev_doms = prev.get("domains") or {} all_ips = [ip for ip in artifacts.get("ips", []) if is_public_ipv4(ip)] - all_doms = artifacts.get("domains", []) + all_doms = [d for d in artifacts.get("domains", []) if is_enrichable_domain(d)] # Deduplicate: skip indicators already enriched in a previous batch run new_ips = [ip for ip in all_ips if ip not in prev_ips] diff --git a/app/pipeline/runner.py b/app/pipeline/runner.py index 814e230..4ef132b 100644 --- a/app/pipeline/runner.py +++ b/app/pipeline/runner.py @@ -38,6 +38,7 @@ from app.pipeline.beacon import rank_beaconing from app.pipeline.carve import CarveError, carve_http_payloads from app.pipeline.dns_analysis import analyze_dns +from app.pipeline.http_analysis import analyze_http from app.pipeline.pcap_count import count_packets_fast from app.pipeline.progress import Progress from app.pipeline.pyshark_pass import parse_pcap_pyshark @@ -56,8 +57,10 @@ WARNING_ZEEK_NO_LOGS = "zeek_no_logs" WARNING_DNS_ANALYSIS_FAILED = "dns_analysis_failed" WARNING_TLS_CERTS_FAILED = "tls_certs_failed" +WARNING_HTTP_ANALYSIS_FAILED = "http_analysis_failed" WARNING_BEACON_FAILED = "beacon_failed" WARNING_CARVE_FAILED = "carve_failed" +WARNING_ZEEK_TRUNCATED = "zeek_tables_truncated" def _derive_run_id(case_id: str) -> str: @@ -119,8 +122,10 @@ class PipelineResult: warnings: list[str] = field(default_factory=list) summary_narrative: str | None = None mitre_techniques: list[str] = field(default_factory=list) + attack_mapping: dict = field(default_factory=dict) dns_analysis: dict = field(default_factory=dict) tls_analysis: dict = field(default_factory=dict) + http_analysis: dict = field(default_factory=dict) beacon_df_records: list[dict] = field(default_factory=list) # Intermediate state — available to callers (e.g. Streamlit) that need to run @@ -143,8 +148,10 @@ def to_dict(self) -> dict: "warnings": list(self.warnings), "summary_narrative": self.summary_narrative, "mitre_techniques": list(self.mitre_techniques), + "attack_mapping": dict(self.attack_mapping), "dns_analysis": dict(self.dns_analysis), "tls_analysis": dict(self.tls_analysis), + "http_analysis": dict(self.http_analysis), "beacon_df_records": list(self.beacon_df_records), } @@ -162,10 +169,11 @@ def run_pipeline( Stages 2 (PyShark) and 3 (Zeek) run concurrently via ThreadPoolExecutor when both are enabled — they're I/O-bound subprocesses against the same pcap and independent until the merge step. After that join, stages 4–7 (DNS, TLS, beaconing, carving) - fan out into a second ThreadPoolExecutor — they are mutually independent, and the - main thread assembles ``stages_run``/``warnings`` in canonical order after the join - so the output stays deterministic. Each stage is gated by its corresponding - ``PipelineOptions`` flag. Failures in individual stages are recorded in + plus HTTP analysis (gated on http.log being present) fan out into a second + ThreadPoolExecutor — they are mutually independent, and the main thread assembles + ``stages_run``/``warnings`` in canonical order after the join so the output stays + deterministic. Each stage is gated by its corresponding ``PipelineOptions`` flag. + Failures in individual stages are recorded in ``result.warnings`` rather than aborting the run. """ start = time.time() @@ -194,6 +202,7 @@ def run_pipeline( total_pkts: int | None = None dns_result: dict = {} tls_result: dict = {} + http_result: dict = {} beacon_records: list[dict] = [] carved: list[dict] = [] @@ -260,7 +269,11 @@ def _run_zeek(h) -> None: df = load_zeek_any(log_path) except Exception: df = pd.DataFrame() - zeek_tables[name] = df.head(2000) + if len(df) > C.ZEEK_TABLE_MAX_ROWS: + df = df.head(C.ZEEK_TABLE_MAX_ROWS) + if WARNING_ZEEK_TRUNCATED not in warnings: + warnings.append(WARNING_ZEEK_TRUNCATED) + zeek_tables[name] = df stages_run.append("zeek") h.done("Zeek logs loaded.") else: @@ -294,13 +307,14 @@ def _run_zeek(h) -> None: except Exception as exc: logger.warning("merge_zeek_dns failed: %s", exc) - # --- Stages 4-7: post-parse analysis fan-out --- - # DNS + TLS read zeek_tables; beacon reads features["flows"]; carve reads only the - # pcap. They are mutually independent, so run them concurrently. Phase handles are - # created on the main thread (Streamlit ScriptRunContext requirement) — workers only - # call set()/done(). Workers never mutate shared state: each writes its own key in + # --- Stages 4-7 (+ HTTP analysis): post-parse analysis fan-out --- + # DNS + TLS + HTTP read zeek_tables (HTTP only runs when http.log is present); + # beacon reads features["flows"]; carve reads only the pcap. They are mutually + # independent, so run them concurrently. Phase handles are created on the main + # thread (Streamlit ScriptRunContext requirement) — workers only call + # set()/done(). Workers never mutate shared state: each writes its own key in # `outcomes`, and the carve hash-backfill happens after the join. - canonical = ("dns_analysis", "tls_certs", "beacon", "carve") + canonical = ("dns_analysis", "tls_certs", "http_analysis", "beacon", "carve") outcomes: dict[str, dict] = {} def _run_dns(h) -> None: @@ -323,6 +337,17 @@ def _run_tls(h) -> None: outcomes.setdefault("tls_certs", {"warning": WARNING_TLS_CERTS_FAILED}) h.done("TLS analysis failed.") + def _run_http(h) -> None: + try: + outcomes["http_analysis"] = { + "result": analyze_http(zeek_tables, http_log_path=zeek_log_paths.get("http.log"), phase=h) or {} + } + h.done("HTTP analysis complete.") + except Exception as exc: + logger.error("HTTP analysis failed: %s", exc) + outcomes.setdefault("http_analysis", {"warning": WARNING_HTTP_ANALYSIS_FAILED}) + h.done("HTTP analysis failed.") + def _run_beacon(h) -> None: try: h.set(30, "Scoring flows…") @@ -359,6 +384,12 @@ def _run_carve(h) -> None: if options.do_zeek and zeek_tables: jobs.append((_run_dns, progress.start_phase("DNS Analysis"))) jobs.append((_run_tls, progress.start_phase("TLS Certificate Analysis"))) + # Same gate as DNS/TLS (not "http.log" in zeek_tables): the phase must always + # run when Zeek ran so the UI's total_phases denominator matches the phases + # actually started. analyze_http reads the uncapped http.log when present and + # returns {"skipped": True} on all-HTTPS / DNS-only captures — mirroring how + # DNS/TLS run and return empty/skipped when their log is absent. + jobs.append((_run_http, progress.start_phase("HTTP Analysis"))) if features.get("flows"): jobs.append((_run_beacon, progress.start_phase("Beaconing ranking"))) if options.do_carve: @@ -384,6 +415,8 @@ def _run_carve(h) -> None: dns_result = out["result"] elif name == "tls_certs": tls_result = out["result"] + elif name == "http_analysis": + http_result = out["result"] elif name == "beacon": beacon_records = out["result"] elif name == "carve": @@ -397,6 +430,27 @@ def _run_carve(h) -> None: features["artifacts"]["hashes"].append(sha) features["artifacts"]["hashes"] = uniq_sorted(features["artifacts"]["hashes"]) + # --- ATT&CK mapping (post-fan-out, not a canonical pipeline stage) --- + # Runs against whatever features/dns/tls/beacon data are final at this point. + # The runner has no yara/osint results (those run in the callers' main.py/queue.py + # after this function returns), so map_analysis is called without them — expected. + # A mapper bug must never break the pipeline, hence the broad except. + attack_mapping_dict: dict = {} + mitre_techniques: list[str] = [] + try: + from app.threat_intel import ATTACKMapper + + _mapping = ATTACKMapper().map_analysis( + features=features, + dns_analysis=dns_result, + tls_analysis=tls_result, + beacon_results=beacon_records, + ) + attack_mapping_dict = _mapping.to_dict() + mitre_techniques = [t.technique_id for t in _mapping.techniques] + except Exception as exc: + logger.error("ATT&CK mapping failed: %s", exc) + return PipelineResult( case_id=case_id, analysis_id=None, # caller writes the Analysis row and fills this in @@ -404,8 +458,11 @@ def _run_carve(h) -> None: duration_seconds=time.time() - start, stages_run=stages_run, warnings=warnings, + mitre_techniques=mitre_techniques, + attack_mapping=attack_mapping_dict, dns_analysis=dns_result, tls_analysis=tls_result, + http_analysis=http_result, beacon_df_records=beacon_records, features=features, zeek_tables=zeek_tables, diff --git a/app/reports/pdf_generator.py b/app/reports/pdf_generator.py index 5dab5d2..0f594d1 100644 --- a/app/reports/pdf_generator.py +++ b/app/reports/pdf_generator.py @@ -124,6 +124,7 @@ class PDFReportGenerator: ("summary", "Executive Summary"), ("charts", "Visual Overview"), ("correlations", "Threat Correlation Summary"), + ("attack", "MITRE ATT&CK Mapping"), ("iocs", "Indicators of Compromise"), ("osint", "OSINT Analysis"), ("dns", "DNS Analysis"), @@ -182,6 +183,7 @@ def generate( correlations: list | None = None, geoip_data: list | None = None, attack_timeline: list | None = None, + attack_mapping: dict | None = None, ) -> PDFReport | None: """ Generate a complete PDF report. @@ -196,6 +198,8 @@ def generate( case_info: Optional case information. beacon_df: Optional beacon scoring DataFrame. correlations: Optional list of correlation results. + attack_mapping: Optional ``AttackMapping.to_dict()`` shape (or an + ``AttackMapping`` instance) from ``st.session_state["attack_mapping"]``. Returns: PDFReport object or None if generation fails. @@ -218,6 +222,7 @@ def generate( correlations=correlations, geoip_data=geoip_data, attack_timeline=attack_timeline, + attack_mapping=attack_mapping, ) # Generate PDF with accurate page count @@ -261,6 +266,7 @@ def _build_html( correlations: list | None = None, geoip_data: list | None = None, attack_timeline: list | None = None, + attack_mapping: dict | None = None, ) -> str: """Build the complete HTML document.""" # --- Section assembly: render first, number after (see _h2). --- @@ -287,6 +293,10 @@ def _build_html( if correlations: candidates.append(("correlations", self._render_correlation_section(correlations))) + # MITRE ATT&CK Mapping + if attack_mapping: + candidates.append(("attack", self._render_attack_section(attack_mapping))) + # Key Findings / IOC Summary candidates.append(("iocs", self._render_ioc_table(features, osint))) @@ -943,6 +953,68 @@ def _render_correlation_section(self, correlations: list) -> str: {detail_section}
+""" + + def _render_attack_section(self, attack_mapping) -> str: + """Render the MITRE ATT&CK technique mapping section. + + Accepts either an ``AttackMapping`` dataclass instance or its + ``to_dict()`` form — the shape persisted in + ``st.session_state["attack_mapping"]`` — and normalizes dicts via + ``AttackMapping.from_dict()`` so the rest of this method always works + with dataclass attributes. Returns "" (dropping the section) when + there are no detected techniques. + """ + from app.threat_intel.attack_mapping import AttackMapping + + if not attack_mapping: + return "" + + mapping = AttackMapping.from_dict(attack_mapping) if isinstance(attack_mapping, dict) else attack_mapping + + if not mapping.techniques: + return "" + + tactics_summary = mapping.tactics_summary or {} + tactics_text = ( + ", ".join( + f"{self._escape(tactic)} ({count})" + for tactic, count in sorted(tactics_summary.items(), key=lambda kv: -kv[1]) + ) + if tactics_summary + else "N/A" + ) + + technique_rows = [] + for tech in mapping.techniques: + evidence = "; ".join(tech.evidence[:3]) if tech.evidence else "" + technique_rows.append( + f"{self._escape(tech.technique_id)}" + f"{self._escape(tech.technique_name)}" + f"{self._escape(tech.tactic)}" + f"{tech.confidence:.0%}" + f"{self._escape(evidence)}" + ) + technique_table = "\n".join(technique_rows) + + return f""" +
+ {self._h2("attack")} + + + + + + + +
MetricValue
Kill Chain Phase{self._escape(mapping.kill_chain_phase)}
Overall Severity{self._escape(str(mapping.overall_severity).upper())}
Tactics Detected{tactics_text}
+

Detected Techniques

+ + + {technique_table} +
IDNameTacticConfidenceEvidence
+
+
""" def _render_beacon_section(self, beacon_df: "pd.DataFrame") -> str: @@ -1396,6 +1468,7 @@ def generate_pdf_report( correlations: list | None = None, geoip_data: list | None = None, attack_timeline: list | None = None, + attack_mapping: dict | None = None, ) -> PDFReport | None: """ Convenience function to generate a PDF report. @@ -1410,6 +1483,8 @@ def generate_pdf_report( config: Optional report configuration. beacon_df: Optional beacon scoring DataFrame. correlations: Optional list of correlation results. + attack_mapping: Optional ``AttackMapping.to_dict()`` shape (or an + ``AttackMapping`` instance) from ``st.session_state["attack_mapping"]``. Returns: PDFReport object or None. @@ -1426,4 +1501,5 @@ def generate_pdf_report( correlations=correlations, geoip_data=geoip_data, attack_timeline=attack_timeline, + attack_mapping=attack_mapping, ) diff --git a/app/threat_intel/attack_mapping.py b/app/threat_intel/attack_mapping.py index 953ec4a..437a1a7 100644 --- a/app/threat_intel/attack_mapping.py +++ b/app/threat_intel/attack_mapping.py @@ -6,6 +6,8 @@ import logging from dataclasses import dataclass, field +from app.pipeline.ja3 import lookup_ja3 + logger = logging.getLogger(__name__) # Valid IOC types for validation @@ -18,6 +20,17 @@ MAX_JA3_FINGERPRINTS = 50 MAX_FLOWS = 1000 +# Confidence assigned to a malicious JA3 match, keyed by the authoritative +# severity from app.pipeline.ja3.KNOWN_JA3_FINGERPRINTS. Unknown/missing +# severities fall back to JA3_DEFAULT_CONFIDENCE. +JA3_SEVERITY_CONFIDENCE = { + "critical": 0.9, + "high": 0.75, + "medium": 0.6, + "low": 0.4, +} +JA3_DEFAULT_CONFIDENCE = 0.75 + # Average packet size estimate (bytes) when only packet count is available AVG_PACKET_SIZE_ESTIMATE = 800 @@ -42,6 +55,22 @@ def to_dict(self) -> dict: "evidence": self.evidence, } + @classmethod + def from_dict(cls, d: dict) -> TechniqueMatch: + """Reconstruct a ``TechniqueMatch`` from its ``to_dict()`` form. + + Missing keys fall back to safe defaults so a partial/malformed dict + (e.g. from a stale session-state entry) does not raise. + """ + d = d or {} + return cls( + technique_id=d.get("technique_id", ""), + technique_name=d.get("technique_name", ""), + tactic=d.get("tactic", ""), + confidence=d.get("confidence", 0.0), + evidence=list(d.get("evidence") or []), + ) + @dataclass class AttackMapping: @@ -61,6 +90,23 @@ def to_dict(self) -> dict: "overall_severity": self.overall_severity, } + @classmethod + def from_dict(cls, d: dict) -> AttackMapping: + """Reconstruct an ``AttackMapping`` from its ``to_dict()`` form. + + Handles an empty or partial dict gracefully — the mapper-failure + default stored in ``st.session_state["attack_mapping"]`` is ``{}``, + and ``from_dict({})`` must return a valid empty ``AttackMapping()`` + rather than raising. + """ + d = d or {} + return cls( + techniques=[TechniqueMatch.from_dict(t) for t in d.get("techniques") or []], + tactics_summary=dict(d.get("tactics_summary") or {}), + kill_chain_phase=d.get("kill_chain_phase", "unknown"), + overall_severity=d.get("overall_severity", "low"), + ) + # Kill chain phases in order of advancement KILL_CHAIN_ORDER = [ @@ -398,40 +444,49 @@ def _check_dns(self, dns_analysis: dict) -> list[TechniqueMatch]: return techniques def _check_tls(self, tls_analysis: dict) -> list[TechniqueMatch]: - """Check TLS certificate analysis for anomalies.""" + """Check TLS certificate analysis for anomalies. + + ``analyze_certificates()`` (app/pipeline/tls_certs.py) reports per-cert + flags in a ``certificates`` list (``is_self_signed``/``is_expired`` booleans) + and aggregate counts in ``alerts`` (``self_signed_count``/``expired_count``/ + ``high_risk_count``) — there is no list of ``{"type": ..., "cert": ...}`` + alert objects. Read the actual shape rather than a hypothetical one. + """ techniques = [] - alerts = tls_analysis.get("alerts", []) + certificates = tls_analysis.get("certificates", []) - for alert in alerts: - alert_type = alert.get("type", "") + def _cert_label(cert: dict) -> str: + return cert.get("subject_cn") or cert.get("fingerprint_sha256") or "unknown" - if alert_type == "self_signed": - rule = self.detection_rules["self_signed_cert"] - cert = alert.get("cert", "unknown") - for tech in rule["techniques"]: - techniques.append( - TechniqueMatch( - technique_id=tech["id"], - technique_name=tech["name"], - tactic=tech["tactic"], - confidence=0.7, - evidence=[f"Self-signed certificate: {cert}"], - ) + self_signed = [c for c in certificates if c.get("is_self_signed")] + if self_signed: + rule = self.detection_rules["self_signed_cert"] + cert_label = _cert_label(self_signed[0]) + for tech in rule["techniques"]: + techniques.append( + TechniqueMatch( + technique_id=tech["id"], + technique_name=tech["name"], + tactic=tech["tactic"], + confidence=0.7, + evidence=[f"Self-signed certificate: {cert_label}"], ) + ) - elif alert_type == "expired": - rule = self.detection_rules["expired_cert"] - cert = alert.get("cert", "unknown") - for tech in rule["techniques"]: - techniques.append( - TechniqueMatch( - technique_id=tech["id"], - technique_name=tech["name"], - tactic=tech["tactic"], - confidence=0.5, - evidence=[f"Expired certificate: {cert}"], - ) + expired = [c for c in certificates if c.get("is_expired")] + if expired: + rule = self.detection_rules["expired_cert"] + cert_label = _cert_label(expired[0]) + for tech in rule["techniques"]: + techniques.append( + TechniqueMatch( + technique_id=tech["id"], + technique_name=tech["name"], + tactic=tech["tactic"], + confidence=0.5, + evidence=[f"Expired certificate: {cert_label}"], ) + ) return techniques @@ -504,24 +559,19 @@ def _check_ja3_from_features(self, features: dict) -> list[TechniqueMatch]: if not ja3_list: return techniques - # Known malicious JA3 patterns (subset for demonstration) - # In production, this would query a threat intel database - known_malware_ja3 = { - "72a589da586844d7f0818ce684948eea": "Emotet", - "a0e9f5d64349fb13191bc781f81f42e1": "TrickBot", - } - for ja3 in ja3_list: ja3_hash = ja3 if isinstance(ja3, str) else ja3.get("hash", "") - if ja3_hash in known_malware_ja3: - malware_name = known_malware_ja3[ja3_hash] + match = lookup_ja3(ja3_hash) + if match and match.get("malware"): + client = match.get("client", "Unknown") + confidence = JA3_SEVERITY_CONFIDENCE.get(match.get("severity"), JA3_DEFAULT_CONFIDENCE) techniques.append( TechniqueMatch( technique_id="T1071.001", technique_name="Web Protocols", tactic="command-and-control", - confidence=0.85, - evidence=[f"Known malware JA3 fingerprint detected: {malware_name}"], + confidence=confidence, + evidence=[f"Known malware JA3 fingerprint detected: {client}"], ) ) diff --git a/app/ui/cases_tab.py b/app/ui/cases_tab.py index 09f1033..96c3b24 100644 --- a/app/ui/cases_tab.py +++ b/app/ui/cases_tab.py @@ -46,6 +46,55 @@ def _get_repo() -> CaseRepository: return st.session_state["case_repo"] +def _session_mitre_techniques() -> list[str]: + """Derive MITRE ATT&CK technique IDs from the session's attack mapping. + + Mirrors how the API/queue path derives ``Analysis.mitre_techniques`` from + ``result.mitre_techniques`` (see ``app/api/queue.py``) so that analyses + saved through the UI populate the IOC feed's ``mitre_techniques`` field + the same way as analyses saved via the API — both share ``data/cases.db``. + + Returns: + List of technique IDs (e.g. ``["T1071.001"]``), or an empty list if + no attack mapping is present in session state. + """ + return [ + t.get("technique_id") + for t in (st.session_state.get("attack_mapping") or {}).get("techniques", []) + if t.get("technique_id") + ] + + +def _session_analysis_features() -> dict: + """Build the ``features`` dict for a UI-saved ``Analysis``, stashing http_analysis/beacon_records. + + Mirrors how ``app/api/queue.py:_persist_analysis`` stashes + ``result.http_analysis`` and ``result.beacon_df_records`` into + ``result.features`` before saving -- ``Analysis`` has no dedicated column + for either, and the case-restore path (``_restore_analysis_to_session``) + reads them back out of ``features``. Without this, a case saved through + the UI's manual-save paths (``_quick_save_analysis`` / + ``_add_current_analysis_to_case``) would lose its HTTP findings + (cleartext creds / suspicious UA / suspicious URIs) and beacon records on + restore, even though the API-saved path keeps them. + + Returns a shallow copy of ``st.session_state["features"]`` -- the live + session dict is never mutated in place, since it may still be read by + other code in the same rerun. + + Returns: + A features dict with ``http_analysis`` and ``beacon_records`` set + from session state (``None``/``[]`` respectively when absent). + """ + features = dict(st.session_state.get("features") or {}) + features["http_analysis"] = st.session_state.get("http_analysis") + beacon_df = st.session_state.get("beacon_df") + features["beacon_records"] = ( + beacon_df.to_dict("records") if isinstance(beacon_df, pd.DataFrame) and not beacon_df.empty else [] + ) + return features + + def _restore_analysis_to_session(analysis: Analysis) -> None: """Load a saved analysis back into session state for the Dashboard/Results tabs. @@ -67,7 +116,13 @@ def _restore_analysis_to_session(analysis: Analysis) -> None: st.session_state["osint"] = analysis.osint or {} st.session_state["dns_analysis"] = analysis.dns_analysis st.session_state["tls_analysis"] = analysis.tls_analysis + # http_analysis has no dedicated Analysis column — the API path stashes it + # inside features (mirroring beacon_records; see app/api/queue.py). Falls + # back to None (not {}) when absent, matching dns/tls_analysis's "didn't + # run" semantics rather than a false "ran clean". + st.session_state["http_analysis"] = features.get("http_analysis") st.session_state["yara_results"] = analysis.yara_results + st.session_state["attack_mapping"] = analysis.attack_mapping or {} # Model default for report is "" but the app's no-report sentinel is None. st.session_state["report"] = analysis.report or None # Everything below isn't persisted on Analysis — reset it all, otherwise the @@ -82,6 +137,7 @@ def _restore_analysis_to_session(analysis: Analysis) -> None: st.session_state["correlations"] = None st.session_state["flow_asymmetry"] = None st.session_state["port_anomalies"] = None + st.session_state["attack_timeline"] = [] st.session_state["rdns_map"] = {} st.session_state["filter_ips"] = set() st.session_state["filter_protos"] = set() @@ -597,12 +653,14 @@ def _quick_save_analysis(): case_id=case_id, pcap_path=st.session_state.get("__pcap_path", ""), packet_count=st.session_state.get("__total_pkts", 0), - features=features, + features=_session_analysis_features(), osint=st.session_state.get("osint") or {}, report=st.session_state.get("report") or "", yara_results=st.session_state.get("yara_results"), dns_analysis=st.session_state.get("dns_analysis"), tls_analysis=st.session_state.get("tls_analysis"), + attack_mapping=st.session_state.get("attack_mapping") or {}, + mitre_techniques=_session_mitre_techniques(), ) # Extract IOCs @@ -630,12 +688,14 @@ def _add_current_analysis_to_case(case: Case): case_id=case.id, pcap_path=st.session_state.get("__pcap_path", ""), packet_count=st.session_state.get("__total_pkts", 0), - features=features, + features=_session_analysis_features(), osint=st.session_state.get("osint") or {}, report=st.session_state.get("report") or "", yara_results=st.session_state.get("yara_results"), dns_analysis=st.session_state.get("dns_analysis"), tls_analysis=st.session_state.get("tls_analysis"), + attack_mapping=st.session_state.get("attack_mapping") or {}, + mitre_techniques=_session_mitre_techniques(), ) analysis.iocs = repo.extract_iocs(analysis) diff --git a/app/ui/layout.py b/app/ui/layout.py index dabfd3c..e3023b5 100644 --- a/app/ui/layout.py +++ b/app/ui/layout.py @@ -1766,6 +1766,110 @@ def render_dns_analysis(result_col, dns_analysis: dict | None): st.dataframe(df_top, width="stretch", hide_index=True) +def render_http_analysis(result_col, http_analysis: dict | None): + """Render HTTP request analysis: suspicious User-Agents, cleartext credentials, suspicious URIs.""" + with result_col: + expanded = bool( + http_analysis is not None + and ( + http_analysis.get("alerts", {}).get("cleartext_cred_count", 0) + or http_analysis.get("alerts", {}).get("suspicious_uri_count", 0) + or http_analysis.get("alerts", {}).get("suspicious_ua_count", 0) + ) + ) + with st.expander("HTTP Analysis", expanded=expanded): + # analyze_http always returns a truthy dict when it executes, so + # None means the stage was skipped or failed — never "ran clean". + if http_analysis is None: + st.info( + "📭 HTTP analysis didn't run in this session (stage skipped or failed) — re-run with Zeek enabled." + ) + return + if http_analysis.get("skipped"): + st.info("📭 HTTP analysis was skipped for this run.") + return + if http_analysis.get("error"): + st.info(f"📭 HTTP analysis: {http_analysis['error']}") + return + + alerts = http_analysis.get("alerts", {}) + ua_count = alerts.get("suspicious_ua_count", 0) + cred_count = alerts.get("cleartext_cred_count", 0) + uri_count = alerts.get("suspicious_uri_count", 0) + + # Summary metrics + col1, col2, col3, col4, col5 = st.columns(5) + with col1: + st.metric("HTTP Requests", http_analysis.get("total_requests", 0)) + with col2: + st.metric("Unique Hosts", http_analysis.get("unique_hosts", 0)) + with col3: + if ua_count: + st.metric("Suspicious UAs", ua_count, delta="Warning", delta_color="inverse") + else: + st.metric("Suspicious UAs", 0) + with col4: + if cred_count: + st.metric("Cleartext Creds", cred_count, delta="Warning", delta_color="inverse") + else: + st.metric("Cleartext Creds", 0) + with col5: + if uri_count: + st.metric("Suspicious URIs", uri_count, delta="Warning", delta_color="inverse") + else: + st.metric("Suspicious URIs", 0) + + # Alert banners + if cred_count: + st.error(f"**Cleartext Credentials:** {cred_count} Basic-auth username(s) seen in plaintext HTTP!") + if uri_count: + st.error(f"**Suspicious URIs:** {uri_count} risky/oversized/raw-IP-hosted URIs detected!") + if ua_count: + st.warning(f"**Suspicious User-Agents:** {ua_count} missing/known-tool user-agents detected!") + + # Tabs for detailed data + tab_creds, tab_ua, tab_uri = st.tabs(["Cleartext Credentials", "Suspicious User-Agents", "Suspicious URIs"]) + + with tab_creds: + creds_list = http_analysis.get("cleartext_credentials", []) + if creds_list: + df_creds = pd.DataFrame(creds_list) + display_cols = ["host", "uri", "username"] + display_cols = [c for c in display_cols if c in df_creds.columns] + render_export_buttons( + df_creds[display_cols], "http_cleartext_creds", key_suffix="http_creds", is_dataframe=True + ) + st.dataframe(df_creds[display_cols], width="stretch", hide_index=True) + else: + st.caption("No cleartext credentials detected.") + + with tab_ua: + ua_list = http_analysis.get("suspicious_user_agents", []) + if ua_list: + df_ua = pd.DataFrame(ua_list) + display_cols = ["host", "user_agent", "uri", "reason"] + display_cols = [c for c in display_cols if c in df_ua.columns] + render_export_buttons( + df_ua[display_cols], "http_suspicious_ua", key_suffix="http_ua", is_dataframe=True + ) + st.dataframe(df_ua[display_cols], width="stretch", hide_index=True) + else: + st.caption("No suspicious user-agents detected.") + + with tab_uri: + uri_list = http_analysis.get("suspicious_uris", []) + if uri_list: + df_uri = pd.DataFrame(uri_list) + display_cols = ["host", "uri", "reason"] + display_cols = [c for c in display_cols if c in df_uri.columns] + render_export_buttons( + df_uri[display_cols], "http_suspicious_uri", key_suffix="http_uri", is_dataframe=True + ) + st.dataframe(df_uri[display_cols], width="stretch", hide_index=True) + else: + st.caption("No suspicious URIs detected.") + + def render_tls_certificates(result_col, tls_analysis: dict | None): """Render TLS certificate analysis results.""" with result_col: diff --git a/app/utils/cef_export.py b/app/utils/cef_export.py index 0325aa1..8e1fb5c 100644 --- a/app/utils/cef_export.py +++ b/app/utils/cef_export.py @@ -40,6 +40,30 @@ } +def _escape_cef_extension(value: str) -> str: + """Escape a CEF extension value per the CEF spec. + + Backslash MUST be escaped first, so the literal ``\\n``/``\\r`` sequences + introduced below for real newline/CR characters are not themselves + re-escaped. Embedded CR/LF are rendered as the literal two-character + escape sequences (not stripped) so attacker-influenced IOC values (e.g. + derived from PCAP data) can't inject a second CEF/syslog line into the + ingesting SIEM. + """ + return str(value).replace("\\", "\\\\").replace("=", "\\=").replace("\r", "\\r").replace("\n", "\\n") + + +def _escape_cef_header_field(value: str) -> str: + """Escape a CEF header field (e.g. ``name``) per the CEF spec. + + Backslash and pipe are escaped per spec. Header fields have no + standard escape sequence for a literal newline, so embedded CR/LF are + collapsed to a space instead — this keeps the record on a single line + without silently deleting the word-break the character implied. + """ + return value.replace("\\", "\\\\").replace("|", "\\|").replace("\r", " ").replace("\n", " ") + + @dataclass class CEFEvent: """A single CEF event ready for syslog emission.""" @@ -54,13 +78,11 @@ def to_cef(self) -> str: """Render as a CEF-formatted string (without syslog header).""" ext_parts = [] for k, v in self.extensions.items(): - # CEF extension values: escape = and backslash - safe_v = str(v).replace("\\", "\\\\").replace("=", "\\=") - ext_parts.append(f"{k}={safe_v}") + ext_parts.append(f"{k}={_escape_cef_extension(v)}") ext_str = " ".join(ext_parts) - # Escape pipe characters in header fields - name_safe = self.name.replace("|", "\\|") + # Escape pipe/backslash and neutralize CR/LF in header fields + name_safe = _escape_cef_header_field(self.name) return ( f"CEF:{_CEF_VERSION}|{_VENDOR}|{_PRODUCT}|{_PRODUCT_VERSION}" @@ -277,6 +299,50 @@ def generate_cef_events( return events +def feed_rows_to_cef(rows: list[dict], hostname: str = "pcap-hunter") -> str: + """Render feed-query IOC rows (type/value/score/severity/tags/...) as CEF syslog lines. + + One CEF event per row — no priority-score gating; the caller's feed filters + (min_score/type) already decide inclusion. This is distinct from + :func:`_events_from_iocs`, which expects ``ScoredIOC``-shaped objects with a + 0.0-1.0 ``priority_score`` and silently drops anything below 0.4 — feed rows + instead carry an integer ``score`` (25/50/75/100, derived from severity), and + a CEF feed pull must faithfully return exactly the rows the feed query + returned, including LOW-severity ones. + + Args: + rows: Feed rows as returned by :func:`app.api.feed.query_iocs` — each a + dict with at least ``type``, ``value``, ``score``, ``severity``, and + ``tags`` keys. + hostname: Hostname for the syslog header. + + Returns: + Newline-separated CEF/syslog lines, one per row. Empty string if + ``rows`` is empty. + """ + events: list[CEFEvent] = [] + for row in rows: + severity_label = str(row.get("severity") or "medium").lower() + severity = _SEVERITY_MAP.get(severity_label, 5) + tags = row.get("tags") or [] + tag_str = ";".join(str(t) for t in tags) + + events.append( + CEFEvent( + signature_id="IOC-FEED-001", + name=f"IOC Feed Entry ({severity_label})", + severity=severity, + extensions={ + "value": str(row.get("value", "")), + "type": str(row.get("type", "")), + "score": str(row.get("score", "")), + "tags": tag_str, + }, + ) + ) + return "\n".join(format_syslog(e, hostname=hostname) for e in events) + + def export_cef_text( correlations: list | None = None, beacon_df: Any = None, diff --git a/app/utils/ioc_export.py b/app/utils/ioc_export.py index 2f66477..f6e6a39 100644 --- a/app/utils/ioc_export.py +++ b/app/utils/ioc_export.py @@ -367,29 +367,10 @@ def _export_stix_basic(self, ioc_types: list[str] | None = None, min_score: floa return json.dumps(bundle, indent=2).encode("utf-8") def _ioc_to_stix_pattern(self, ioc: IOCRecord) -> str | None: - """Convert IOC to STIX pattern.""" - from app.utils.stix_export import _escape_stix_value - - escaped = _escape_stix_value(ioc.value) - if ioc.ioc_type == "ip": - return f"[ipv4-addr:value = '{escaped}']" - elif ioc.ioc_type == "domain": - return f"[domain-name:value = '{escaped}']" - elif ioc.ioc_type == "hash": - # Determine hash type by length - if len(ioc.value) == 32: - return f"[file:hashes.MD5 = '{escaped}']" - elif len(ioc.value) == 40: - return f"[file:hashes.'SHA-1' = '{escaped}']" - elif len(ioc.value) == 64: - return f"[file:hashes.'SHA-256' = '{escaped}']" - elif ioc.ioc_type == "url": - return f"[url:value = '{escaped}']" - elif ioc.ioc_type == "ja3": - # JA3 doesn't have a standard STIX pattern - return None - - return None + """Convert IOC to STIX pattern via the shared serializer.""" + from app.utils.stix_export import ioc_to_stix_pattern + + return ioc_to_stix_pattern(ioc.ioc_type, ioc.value) def generate_ioc_filename(format_type: str) -> str: diff --git a/app/utils/network_utils.py b/app/utils/network_utils.py index f29d10e..46bf8af 100644 --- a/app/utils/network_utils.py +++ b/app/utils/network_utils.py @@ -117,6 +117,53 @@ def _validate_domain(domain: str) -> bool: return True +_PRIVATE_TLDS = frozenset( + { + "local", + "internal", + "lan", + "corp", + "home", + "intranet", + "test", + "example", + "invalid", + "localhost", + } +) + + +def is_enrichable_domain(domain: str) -> bool: + """True only for public, routable domains worth sending to OSINT providers. + + Prevents leaking internal infrastructure names (dc01.internal.corp, *.local) + to third-party SaaS (VirusTotal submissions are visible to paid subscribers). + + Rejects: invalid domains per ``_validate_domain`` (dot required, no + underscore), single-label names, reverse-DNS artifacts + (``*.in-addr.arpa``/``*.ip6.arpa``), IP-shaped strings, and private/internal + TLDs (local/internal/lan/corp/home/intranet/test/example/invalid/localhost). + """ + if not domain or not _validate_domain(domain): # dot required, no underscore + return False + lower = domain.lower().rstrip(".") + + # _validate_domain's regex allows all-numeric labels, so IP literals like + # "1.2.3.4" pass it — reject them explicitly since they aren't domains. + try: + ipaddress.ip_address(lower) + return False + except ValueError: + pass + + if lower.endswith((".in-addr.arpa", ".ip6.arpa")): + return False + tld = lower.rsplit(".", 1)[-1] + if tld in _PRIVATE_TLDS: + return False + return True + + def get_whois_info(target: str) -> dict | str: """ Retrieve WHOIS information for a domain or IP. @@ -197,6 +244,65 @@ def is_public_ipv4(s: str) -> bool: return False +def is_safe_webhook_url(url: str) -> bool: + """SSRF guard for user-supplied outbound webhook URLs. + + ``hardened_session`` (app/security/opsec.py) hardens TLS/redirect/timeout + behavior but does **not** block requests to private IPs — this is the + only SSRF protection for the completion-webhook feature (API v2). + + True only when: the scheme is ``http``/``https``, the host resolves + (``socket.getaddrinfo``), and *every* resolved address is a public/global + IP — not loopback, private, link-local, reserved, multicast, or + unspecified. Requiring all resolved addresses to be public (rather than + just the first) closes a DNS-rebinding-style bypass where a hostname + round-robins between a public and a private address. + + Args: + url: Candidate webhook URL. + + Returns: + False for non-http(s) schemes, missing host, resolution failures, or + any resolved address that is not public. + """ + try: + parsed = urllib.parse.urlparse(url) + except ValueError: + return False + if parsed.scheme not in ("http", "https"): + return False + host = parsed.hostname + if not host: + return False + + # getaddrinfo handles hostnames and IP literals uniformly and returns + # every resolved address (A + AAAA), which is what we need to check. + try: + infos = socket.getaddrinfo(host, None) + except (socket.gaierror, socket.herror, OSError, UnicodeError): + return False + if not infos: + return False + + for info in infos: + ip_str = info[4][0].split("%", 1)[0] # strip IPv6 zone id (e.g. "fe80::1%eth0") + try: + ip = ipaddress.ip_address(ip_str) + except ValueError: + return False + # ipaddress.is_global is True for some multicast ranges (e.g. IPv4 + # 224.0.0.1, IPv6 ff02::1) — reject those explicitly rather than + # trusting is_global alone. + if ip.is_multicast or ip.is_unspecified: + return False + if isinstance(ip, ipaddress.IPv4Address): + if not is_public_ipv4(ip_str): + return False + elif not ip.is_global or ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved: + return False + return True + + def pick_top_public_ips(features: dict, n: int) -> list[str]: """ Return top-N public IPv4s by packet volume across flows. diff --git a/app/utils/stix_export.py b/app/utils/stix_export.py index 98ce611..07d030b 100644 --- a/app/utils/stix_export.py +++ b/app/utils/stix_export.py @@ -19,6 +19,29 @@ def _escape_stix_value(value: str) -> str: return value.replace("\\", "\\\\").replace("'", "\\'") +def stix_hash_property(value: str) -> str | None: + """STIX hash property name from hex-digest length. None if unrecognized.""" + return {32: "MD5", 40: "'SHA-1'", 64: "'SHA-256'", 128: "'SHA-512'"}.get(len(value)) + + +def ioc_to_stix_pattern(ioc_type: str, value: str) -> str | None: + """Build a STIX 2.1 comparison-expression pattern for an IOC. None if unmappable.""" + escaped = _escape_stix_value(value) + if ioc_type == "ip": + kind = "ipv6-addr" if ":" in value else "ipv4-addr" + return f"[{kind}:value = '{escaped}']" + if ioc_type == "domain": + return f"[domain-name:value = '{escaped}']" + if ioc_type == "url": + return f"[url:value = '{escaped}']" + if ioc_type == "hash": + prop = stix_hash_property(value) + return f"[file:hashes.{prop} = '{escaped}']" if prop else None + if ioc_type == "ja3": + return f"[x509-certificate:hashes.'JA3' = '{escaped}']" + return None + + # Try to import stix2 library try: import stix2 @@ -82,32 +105,7 @@ def _create_identity(self) -> dict: def _ioc_to_pattern(self, ioc: "IOCRecord") -> str | None: """Convert IOC to STIX pattern.""" - escaped = _escape_stix_value(ioc.value) - if ioc.ioc_type == "ip": - # Check if IPv6 - if ":" in ioc.value: - return f"[ipv6-addr:value = '{escaped}']" - return f"[ipv4-addr:value = '{escaped}']" - elif ioc.ioc_type == "domain": - return f"[domain-name:value = '{escaped}']" - elif ioc.ioc_type == "hash": - # Determine hash type by length - hash_len = len(ioc.value) - if hash_len == 32: - return f"[file:hashes.MD5 = '{escaped}']" - elif hash_len == 40: - return f"[file:hashes.'SHA-1' = '{escaped}']" - elif hash_len == 64: - return f"[file:hashes.'SHA-256' = '{escaped}']" - elif hash_len == 128: - return f"[file:hashes.'SHA-512' = '{escaped}']" - elif ioc.ioc_type == "url": - return f"[url:value = '{escaped}']" - elif ioc.ioc_type == "ja3": - # JA3 as x509 extension (non-standard but useful) - return f"[x509-certificate:hashes.'JA3' = '{escaped}']" - - return None + return ioc_to_stix_pattern(ioc.ioc_type, ioc.value) def _get_indicator_labels(self, ioc: "IOCRecord") -> list[str]: """Get indicator labels based on IOC data.""" diff --git a/docs/API.md b/docs/API.md index b01d6f0..e5903f6 100644 --- a/docs/API.md +++ b/docs/API.md @@ -236,11 +236,17 @@ All routes implemented by the server, by router group: | PCAP Ingestion | POST | `/api/v1/pcaps` | `full` | | Job Tracking | GET | `/api/v1/jobs/{job_id}` | `full` | | Job Tracking | GET | `/api/v1/jobs/{job_id}/result` | `full` | +| Case Management | GET | `/api/v1/cases` | `full` | | Case Management | GET | `/api/v1/cases/{case_id}` | `full` | | Case Management | GET | `/api/v1/cases/{case_id}/report.pdf` | `full` | | Case Management | DELETE | `/api/v1/cases/{case_id}` | `full` | +| Case Management | PATCH | `/api/v1/cases/{case_id}` | `full` | +| Case Management | POST | `/api/v1/cases/{case_id}/notes` | `full` | +| Case Management | GET | `/api/v1/cases/{case_id}/notes` | `full` | | IOC Feed | GET | `/api/v1/iocs.json` | `feed` | +| IOC Feed | GET | `/api/v1/iocs/lookup` | `feed` | | IOC Feed | GET | `/api/v1/iocs.csv` | `feed` | +| IOC Feed | GET | `/api/v1/iocs.cef` | `feed` | | IOC Feed | GET | `/api/v1/iocs.stix` (alias: `/api/v1/iocs/stix`) | `feed` | | Admin | POST | `/api/v1/admin/keys` | `full` | | Admin | GET | `/api/v1/admin/keys` | `full` | @@ -340,11 +346,35 @@ Submit a PCAP file for background analysis. The upload is streamed to disk in 1 | `osint_enabled` | boolean | No | `true` | Run OSINT enrichment after analysis (see below) | | `llm_enabled` | boolean | No | `true` | Accepted for forward compatibility — LLM reports are **not yet supported headless** (see below) | | `pyshark_packet_limit` | integer | No | server default (200,000) | Cap on packets to deep-parse | +| `webhook_url` | string | No | (none) | `http`/`https` callback URL to POST when the job reaches a terminal state (see [Completion webhook](#completion-webhook-webhook_url) below). Validated at submit time; rejected with `422` if unsafe | **OSINT enrichment (`osint_enabled`):** when enabled, the worker enriches the top public IPs after analysis using provider keys from the saved Streamlit config (`cfg_*_key` values) or, as a fallback, the environment (`OTX_KEY`, `VT_KEY`, `ABUSEIPDB_KEY`, `GREYNOISE_KEY`, `SHODAN_KEY`). If no provider keys are configured, the job still completes — with the warning code `osint_not_configured` in the result. Note: the API path always queries providers fresh; the OSINT response cache is not used headless. **LLM reports (`llm_enabled`):** LLM report generation is not yet supported on the API path. The field is accepted so existing clients keep working, but jobs complete with the warning code `llm_unsupported_on_api_path` in the result — including default submissions, since the field defaults to `true`. +#### Completion webhook (`webhook_url`) + +When `webhook_url` is supplied, the API POSTs a small JSON envelope to that URL once the job reaches a terminal state (`done` or `failed`) — useful for SOAR playbooks that would otherwise have to poll `GET /jobs/{job_id}`. + +- **SSRF-validated twice:** once at submit time (before any upload I/O — an unsafe URL fails fast with `422 invalid_webhook_url`), and again immediately before the callback POST fires (defense in depth against the target becoming unsafe between submission and job completion, e.g. a DNS change). A URL is safe only if its scheme is `http`/`https` and **every** address it resolves to is a public/global IP — not loopback, private, link-local, reserved, or multicast. `hardened_session` alone does not block private IPs, so this check is the only SSRF guard for this feature. +- **Delivery happens in the analysis worker subprocess**, after the job's own terminal state is already persisted. A webhook delivery failure (timeout, connection error, non-2xx status) is logged and retried but **never** changes the job's `done`/`failed` status — polling `GET /jobs/{job_id}` remains the source of truth. +- **Retries:** up to `PCAP_HUNTER_API_WEBHOOK_MAX_RETRIES` additional attempts (default `2`, so 3 attempts total) with a short fixed delay between attempts, each bounded by `PCAP_HUNTER_API_WEBHOOK_TIMEOUT_SECONDS` (default `10`). Any 2xx response stops retrying immediately. +- **Redirects are not followed** (`allow_redirects=False`) — only submit-time validation covers the URL you supplied; following a redirect to an attacker-chosen `Location` (e.g. a cloud metadata endpoint) would reopen the SSRF hole. +- **Optional HMAC signature:** if `PCAP_HUNTER_API_WEBHOOK_SECRET` is set on the server, every callback carries an `X-PCAP-Hunter-Signature: sha256=` header (HMAC-SHA256 of the raw JSON body, keyed with the secret) so the receiver can authenticate the request. + +**Callback envelope** (`Content-Type: application/json`): + +```json +{ + "job_id": "j_7d4e9f21", + "case_id": "c4a1b2d9", + "status": "done", + "analysis_id": "9c2d1e0f-4a7" +} +``` + +`status` is `"done"` or `"failed"`; `analysis_id` is `null` for a failed job, and also `null` on a done job whose analysis failed to persist (mirrors `analysis_id` on the [job result](#get-apiv1jobsjob_idresult) endpoint). + **Sample request:** ```bash @@ -355,7 +385,8 @@ curl -X POST http://localhost:8000/api/v1/pcaps \ -F 'tags=["soar:tines","source:edr_alert"]' \ -F "severity_hint=high" \ -F "osint_enabled=true" \ - -F "pyshark_packet_limit=100000" + -F "pyshark_packet_limit=100000" \ + -F "webhook_url=https://soar.example.com/hooks/pcap-hunter" ``` **Sample response — 202 Accepted:** @@ -382,6 +413,7 @@ curl -X POST http://localhost:8000/api/v1/pcaps \ | 413 | `pcap_too_large` | Upload exceeded `PCAP_HUNTER_API_MAX_PCAP_BYTES`; the partial file is deleted | | 415 | `pcap_invalid_format` | First bytes are not a known pcap/pcapng magic; the file is deleted | | 422 | `validation_error` | Malformed form field (e.g. non-boolean `osint_enabled`) | +| 422 | `invalid_webhook_url` | `webhook_url` is not `http`/`https`, its host doesn't resolve, or any resolved address is private/loopback/link-local/reserved/multicast — checked before any upload I/O | | 429 | `rate_limit_exceeded` | DB key over its per-minute limit (`Retry-After` header set) | | 503 | `queue_full` | Active jobs ≥ `PCAP_HUNTER_API_QUEUE_DEPTH`; response carries `Retry-After: 60` | @@ -611,6 +643,58 @@ Example `409`: ### Case Management +#### `GET /api/v1/cases` + +List cases with optional filters — the light-weight complement to `GET /api/v1/cases/{case_id}`: each entry omits embedded `analyses[]`/`notes[]` so a full case listing stays cheap even when individual cases carry large feature blobs. Read-only and idempotent. Requires `full` scope. Note: an unrecognized `status` value does not `422` — it silently falls back to filtering on `open` (the same lenient string→enum conversion `PATCH` uses internally), so double-check spelling rather than relying on a validation error to catch a typo. + +**Auth:** `full` scope required + +**Parameters:** + +| Param | In | Type | Required | Default | Description | +|-------|----|------|----------|---------|-------------| +| `status` | query | string | No | (none) | Filter to one status: `open`, `in_progress`, or `closed`. An unrecognized value is treated as `open`, not rejected (see note above) | +| `tag` | query | string | No | (none) | Filter to cases carrying this tag | +| `search` | query | string | No | (none) | Case-insensitive substring match against title and description | +| `limit` | query | integer | No | `100` | Results per page, 1–500 (422 outside that range) | +| `offset` | query | integer | No | `0` | Results to skip, ≥ 0 | + +**Sample request:** + +```bash +curl "http://localhost:8000/api/v1/cases?status=in_progress&limit=50" \ + -H "Authorization: Bearer phk_4f8a2b9c1d3e5f60718293a4b5c6d7e8" +``` + +**Sample response — 200 OK:** + +```json +{ + "cases": [ + { + "id": "c4a1b2d9", + "title": "Incident 2026-0042", + "description": "", + "status": "in_progress", + "severity": "high", + "created_at": "2026-06-12T09:14:02.731842", + "updated_at": "2026-06-12T09:15:41.557209", + "closed_at": null, + "tags": ["soar:tines", "source:edr_alert"] + } + ], + "count": 1 +} +``` + +**Error responses:** + +| Status | Code | When | +|--------|------|------| +| 401 | `missing_or_malformed_auth` / `invalid_key` | Auth failure | +| 403 | `insufficient_scope` | Feed-scope key | +| 422 | `validation_error` | Query constraint violated (e.g. `limit=0`, `limit=1000`) | + #### `GET /api/v1/cases/{case_id}` Fetch the full case record — title, status, severity, tags, plus every persisted analysis (with its extracted IOCs) and case notes embedded. API-submitted cases are created with `status = in_progress` and the requested `severity_hint`; the same record is visible in the Streamlit Cases tab. Read-only and idempotent. Requires `full` scope. Note that embedded `analyses[]` can be large (the `features` object holds the full feature extraction); fetch the job result instead if you only need pipeline output. @@ -786,13 +870,150 @@ Example `409`: } ``` +#### `PATCH /api/v1/cases/{case_id}` + +Partially update a case — title, description, status, severity, and/or tags. Omitted fields are left unchanged; there is no way to distinguish "field omitted" from "set to its current value" (both are no-ops). Setting `status` to `closed` also sets `closed_at` (via the same path the Streamlit "close case" action uses); setting any other status does not clear a previously-set `closed_at`. `tags` is a **full replace**, not a merge — send the complete desired tag list. Requires `full` scope. + +**Auth:** `full` scope required + +**Content-Type:** `application/json` + +**Parameters (JSON body, all optional):** + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `title` | string | No | New title (1–200 chars) | +| `description` | string | No | New description (max 5000 chars) | +| `status` | string | No | `open`, `in_progress`, or `closed` | +| `severity` | string | No | `low`, `medium`, `high`, or `critical` | +| `tags` | array of string | No | Full replacement tag list | + +**Sample request:** + +```bash +curl -X PATCH http://localhost:8000/api/v1/cases/c4a1b2d9 \ + -H "Authorization: Bearer phk_4f8a2b9c1d3e5f60718293a4b5c6d7e8" \ + -H "Content-Type: application/json" \ + -d '{"status": "closed", "severity": "critical", "tags": ["soar:tines", "confirmed"]}' +``` + +**Sample response — 200 OK** (full updated case record, same shape as `GET /api/v1/cases/{case_id}`): + +```json +{ + "id": "c4a1b2d9", + "title": "Incident 2026-0042", + "description": "", + "status": "closed", + "severity": "critical", + "created_at": "2026-06-12T09:14:02.731842", + "updated_at": "2026-06-12T09:20:11.004221", + "closed_at": "2026-06-12T09:20:11.004221", + "tags": ["soar:tines", "confirmed"], + "analyses": [], + "notes": [] +} +``` + +**Error responses:** + +| Status | Code | When | +|--------|------|------| +| 401 | `missing_or_malformed_auth` / `invalid_key` | Auth failure | +| 403 | `insufficient_scope` | Feed-scope key | +| 404 | `case_not_found` | Case ID does not exist | +| 422 | `validation_error` | `status`/`severity` not one of the accepted values, `title` empty, or a field exceeds its max length | + +#### `POST /api/v1/cases/{case_id}/notes` + +Add an analyst note to a case. Notes are plain text with no formatting; there is no update or delete endpoint for an individual note. Not idempotent — every call adds a new note. Requires `full` scope. + +**Auth:** `full` scope required + +**Content-Type:** `application/json` + +**Parameters (JSON body):** + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `content` | string | Yes | Note text, min length 1 | + +**Sample request:** + +```bash +curl -X POST http://localhost:8000/api/v1/cases/c4a1b2d9/notes \ + -H "Authorization: Bearer phk_4f8a2b9c1d3e5f60718293a4b5c6d7e8" \ + -H "Content-Type: application/json" \ + -d '{"content": "Confirmed beaconing to 198.51.100.42 matches known C2 infrastructure."}' +``` + +**Sample response — 201 Created:** + +```json +{ + "id": 17, + "content": "Confirmed beaconing to 198.51.100.42 matches known C2 infrastructure." +} +``` + +**Error responses:** + +| Status | Code | When | +|--------|------|------| +| 401 | `missing_or_malformed_auth` / `invalid_key` | Auth failure | +| 403 | `insufficient_scope` | Feed-scope key | +| 404 | `case_not_found` | Case ID does not exist | +| 422 | `validation_error` | `content` missing or empty | + +#### `GET /api/v1/cases/{case_id}/notes` + +List all notes on a case, in insertion order. Read-only and idempotent. Requires `full` scope. + +**Auth:** `full` scope required + +**Parameters:** + +| Param | In | Type | Required | Description | +|-------|----|------|----------|-------------| +| `case_id` | path | string | Yes | Case ID (8 hex chars) | + +**Sample request:** + +```bash +curl http://localhost:8000/api/v1/cases/c4a1b2d9/notes \ + -H "Authorization: Bearer phk_4f8a2b9c1d3e5f60718293a4b5c6d7e8" +``` + +**Sample response — 200 OK:** + +```json +{ + "notes": [ + { + "id": 17, + "content": "Confirmed beaconing to 198.51.100.42 matches known C2 infrastructure.", + "created_at": "2026-06-12T09:21:03.552104", + "updated_at": null + } + ] +} +``` + +**Error responses:** + +| Status | Code | When | +|--------|------|------| +| 401 | `missing_or_malformed_auth` / `invalid_key` | Auth failure | +| 403 | `insufficient_scope` | Feed-scope key | +| 404 | `case_not_found` | Case ID does not exist | + --- ### IOC Feed -All feed endpoints require `feed` scope (a `full`-scope key also works), are read-only/idempotent, and support conditional requests via ETag. All three formats accept the **same query parameters** and apply the same dedup/ordering semantics — they differ only in serialization. +All feed endpoints require `feed` scope (a `full`-scope key also works), are read-only/idempotent, and support conditional requests via ETag. The four bulk formats (`iocs.json`, `iocs.csv`, `iocs.cef`, `iocs.stix`) accept the **same query parameters** and apply the same dedup/ordering semantics — they differ only in serialization. `iocs/lookup` is the exception: it takes a single required `value` instead (see its own section below). -**Shared query parameters:** +**Shared query parameters** (`iocs.json` / `iocs.csv` / `iocs.cef` / `iocs.stix`): | Param | Type | Default | Description | |-------|------|---------|-------------| @@ -808,6 +1029,7 @@ All feed endpoints require `feed` scope (a `full`-scope key also works), are rea - **Scoring:** each indicator's `score` derives from the **worst** severity recorded across its sightings: `low` = 25, `medium` = 50, `high` = 75, `critical` = 100; `severity` is the matching label. - **Deduplication:** the same indicator appearing in multiple analyses collapses to a single row carrying that maximum severity/score; `first_seen`/`last_seen` span all sightings and `case_ids` lists every contributing case. +- **`mitre_techniques`:** the deduplicated union of MITRE ATT&CK technique IDs (e.g. `T1071.001`) from every analysis that produced the indicator, derived from each analysis's ATT&CK mapping. Empty for indicators whose contributing analyses matched no techniques — not a placeholder; it reflects real detections. - **Filtering:** `min_score` is applied in SQL (not post-filtered), so it composes correctly with `limit`/`cursor` — pages are always full up to `limit` and no matching rows are dropped at page boundaries. - **Ordering:** deterministic — `last_seen` descending, then indicator value ascending as a tie-breaker. Stable ordering makes cursor pagination reliable. - **Pagination:** `next_cursor` is non-null exactly when the page came back full (`count == limit`); pass it as `cursor` for the next page. (CSV/STIX responses don't carry a cursor — page by incrementing `cursor` by `limit` while pages stay full.) @@ -841,7 +1063,7 @@ curl "http://localhost:8000/api/v1/iocs.json?min_score=50&type=ip,domain&limit=1 "first_seen": "2026-06-10T08:02:11.402199", "last_seen": "2026-06-12T09:15:40.992103", "case_ids": ["c4a1b2d9"], - "mitre_techniques": [] + "mitre_techniques": ["T1071.001"] }, { "type": "domain", @@ -887,6 +1109,68 @@ Example `422`: } ``` +#### `GET /api/v1/iocs/lookup` + +Exact-match single-IOC lookup — the #1 SOAR enrichment pattern (SIEM alert fires on an IP/domain, playbook asks "have we seen this before, and how bad?"). Unlike the bulk feed endpoints, this normally aggregates to 0 or 1 row, so `next_cursor` is always `null` and there is no `since`/`tag`/`case_id`/`cursor` filter. Matching is **exact** (`value = ?`, not a substring or prefix match) — looking up `1.2.3.4` never also returns `1.2.3.40`. An empty `value` is rejected with `422` rather than silently returning the whole feed. Read-only and idempotent. + +**Auth:** `feed` scope (or `full`) + +**Parameters:** + +| Param | In | Type | Required | Default | Description | +|-------|----|------|----------|---------|-------------| +| `value` | query | string | Yes | | Exact IOC value to look up (min length 1) | +| `min_score` | query | integer | No | `0` | Minimum threat score, 0–100 (422 outside that range) | +| `type` | query | string | No | (none) | Comma-separated IOC types to restrict to: `ip`, `domain`, `url`, `hash` | +| `limit` | query | integer | No | `1000` | Results per page, 1–10000 (422 outside that range) — irrelevant in practice since a single value aggregates to at most one row | + +**Sample request:** + +```bash +curl "http://localhost:8000/api/v1/iocs/lookup?value=198.51.100.42" \ + -H "Authorization: Bearer phk_4f8a2b9c1d3e5f60718293a4b5c6d7e8" +``` + +**Sample response — 200 OK** (found): + +```json +{ + "iocs": [ + { + "type": "ip", + "value": "198.51.100.42", + "severity": "high", + "score": 75, + "tags": ["malware", "c2-beacon"], + "first_seen": "2026-06-10T08:02:11.402199", + "last_seen": "2026-06-12T09:15:40.992103", + "case_ids": ["c4a1b2d9"], + "mitre_techniques": ["T1071.001"] + } + ], + "count": 1, + "next_cursor": null +} +``` + +**Sample response — 200 OK** (not found — an empty result, not a `404`): + +```json +{ + "iocs": [], + "count": 0, + "next_cursor": null +} +``` + +**Error responses:** + +| Status | Code | When | +|--------|------|------| +| 401 | `missing_or_malformed_auth` / `invalid_key` | Auth failure | +| 422 | `validation_error` | `value` missing/empty, or `min_score`/`limit` outside their allowed range | +| 429 | `rate_limit_exceeded` | DB key over its per-minute limit | + #### `GET /api/v1/iocs.csv` The same feed as CSV (`Content-Type: text/csv`) — built for lookup tables: Splunk lookups, Graylog CSV adapters, Wazuh CDB lists. Same query parameters, dedup, ordering, and ETag caching as the JSON feed. List-valued fields (`tags`, `case_ids`, `mitre_techniques`) are `;`-joined inside one CSV column. Values that a spreadsheet would interpret as a formula (leading `=`, `+`, `-`, `@`) are prefixed with a single quote to prevent CSV injection. @@ -904,12 +1188,33 @@ curl "http://localhost:8000/api/v1/iocs.csv?min_score=25" \ ```csv type,value,score,severity,tags,first_seen,last_seen,case_ids,mitre_techniques -ip,198.51.100.42,75,high,malware;c2-beacon,2026-06-10T08:02:11.402199,2026-06-12T09:15:40.992103,c4a1b2d9, +ip,198.51.100.42,75,high,malware;c2-beacon,2026-06-10T08:02:11.402199,2026-06-12T09:15:40.992103,c4a1b2d9,T1071.001 domain,updates.evil-cdn.example,100,critical,malware,2026-06-11T17:44:03.215587,2026-06-11T17:44:03.215587,b91e0f2c;c4a1b2d9, ``` **Error responses:** same as `iocs.json` (401 / 422 / 429), plus `405 method_not_allowed` for non-GET methods (applies to every endpoint). +#### `GET /api/v1/iocs.cef` + +The same feed as CEF (Common Event Format) syslog lines (`Content-Type: text/plain`) — for ArcSight/QRadar/Sentinel-style SIEM ingestion. Same query parameters, dedup, ordering, and ETag caching as the JSON feed. Uses a dedicated feed-to-CEF adapter rather than the app's general-purpose correlation/beacon CEF exporter, so it faithfully emits **every** row the filters selected — including LOW-severity IOCs, which the general exporter's `priority_score >= 0.4` gate would otherwise silently drop. Reserved CEF characters (`=`, `\`, CR, LF) in IOC values are escaped per the CEF spec, so an attacker-influenced indicator value can't inject a second event into the ingesting SIEM. + +**Auth:** `feed` scope (or `full`) + +**Sample request:** + +```bash +curl "http://localhost:8000/api/v1/iocs.cef?min_score=25" \ + -H "Authorization: Bearer phk_4f8a2b9c1d3e5f60718293a4b5c6d7e8" +``` + +**Sample response — 200 OK** (`text/plain`, one line per indicator): + +``` +Jun 12 09:15:41 pcap-hunter CEF:0|PCAPHunter|ThreatWorkbench|1.0|IOC-FEED-001|IOC Feed Entry (high)|8|value=198.51.100.42 type=ip score=75 tags=malware;c2-beacon +``` + +**Error responses:** same as `iocs.json` (401 / 422 / 429). + #### `GET /api/v1/iocs.stix` The same feed as a **STIX 2.1 bundle** of `indicator` objects, for STIX-native platforms (OpenCTI, MISP, TAXII ingest scripts). Also served at the alias path **`GET /api/v1/iocs/stix`** (identical behavior). Same query parameters and caching as the other formats. Indicator IDs are deterministic (UUIDv5 of the indicator value), so re-pulls produce stable IDs; IOC types map to STIX patterns: @@ -1425,6 +1730,7 @@ Revoking a key clears its rate-limit window immediately. | `rate_limit_exceeded` | 429 | Per-key rate limit exceeded (`Retry-After` header) | | `pcap_too_large` | 413 | File exceeds `max_pcap_bytes` | | `pcap_invalid_format` | 415 | Missing valid PCAP/pcapng magic signature | +| `invalid_webhook_url` | 422 | `webhook_url` fails the SSRF guard (non-http(s) scheme, unresolvable host, or a resolved address that's private/loopback/link-local/reserved/multicast) | | `queue_full` | 503 | Job queue at capacity (`Retry-After: 60` header) | | `job_not_found` | 404 | Job ID does not exist | | `result_not_ready` | 409 | Job has not finished (or finished `failed`/`cancelled`); additive `current_status` field | @@ -1483,6 +1789,16 @@ All settings are read from environment variables at startup. Defaults are suitab | `PCAP_HUNTER_API_QUEUE_DEPTH` | `100` | Maximum active (queued + running) jobs | | `PCAP_HUNTER_API_UPLOAD_TIMEOUT_SEC` | `600` | Reserved — parsed but **not currently enforced** | +### Webhooks + +Settings for the optional completion webhook (see [Completion webhook](#completion-webhook-webhook_url) under `POST /api/v1/pcaps`). Only relevant when a submission includes `webhook_url`. + +| Variable | Default | Description | +|----------|---------|-------------| +| `PCAP_HUNTER_API_WEBHOOK_TIMEOUT_SECONDS` | `10` | Per-attempt HTTP timeout (seconds) for the completion callback POST | +| `PCAP_HUNTER_API_WEBHOOK_MAX_RETRIES` | `2` | Additional attempts after the first on a non-2xx response or request exception (default: 3 attempts total) | +| `PCAP_HUNTER_API_WEBHOOK_SECRET` | (none — signing disabled) | When set, every callback carries `X-PCAP-Hunter-Signature: sha256=`, an HMAC-SHA256 of the raw JSON body keyed with this secret | + ### Retention | Variable | Default | Description | diff --git a/docs/FEATURE-ROADMAP.md b/docs/FEATURE-ROADMAP.md index ff41de9..cd39fd2 100644 --- a/docs/FEATURE-ROADMAP.md +++ b/docs/FEATURE-ROADMAP.md @@ -1,498 +1,72 @@ # PCAP Hunter Feature Roadmap -## Overview - -This document outlines the planned features for PCAP Hunter, prioritized by implementation complexity and user value. - ---- - -## Phase 1: Quick Wins (Low Effort, High Value) - -### 1.1 CSV/JSON Export - -**Goal**: Allow users to export analysis results for external processing or archival. - -**Scope**: -- Export flow data, Zeek logs, OSINT results, and beaconing scores -- Support CSV (for spreadsheets) and JSON (for programmatic use) -- Include filtered views (respect current dashboard filters) - -**Implementation**: - -| Component | File | Changes | -|-----------|------|---------| -| Export utilities | `app/utils/export.py` | New module | -| UI buttons | `app/ui/results_tab.py` | Add export buttons per section | -| Streamlit download | `app/main.py` | Wire `st.download_button` | - -**Technical Details**: -```python -# app/utils/export.py -def export_to_csv(data: list[dict], filename: str) -> bytes: - """Convert list of dicts to CSV bytes.""" - -def export_to_json(data: Any, filename: str, indent: int = 2) -> bytes: - """Convert data to formatted JSON bytes.""" -``` - -**Dependencies**: None (uses standard library) - ---- - -### 1.2 Configuration Persistence - -**Goal**: Save user settings (API keys, thresholds, preferences) across sessions. - -**Scope**: -- Persist: LLM endpoint, API keys, analysis toggles, threshold values -- Storage: Local JSON file (`.pcap_hunter_config.json`) -- Security: Encrypt sensitive values (API keys) - -**Implementation**: - -| Component | File | Changes | -|-----------|------|---------| -| Config manager | `app/utils/config_manager.py` | New module | -| Encryption | `app/security/crypto.py` | New module (Fernet) | -| UI integration | `app/ui/sidebar.py` | Load/save buttons | - -**Technical Details**: -```python -# app/utils/config_manager.py -class ConfigManager: - def load(self) -> dict: ... - def save(self, config: dict) -> None: ... - def get(self, key: str, default: Any = None) -> Any: ... -``` - -**Dependencies**: `cryptography` (for Fernet encryption) - ---- - -### 1.3 OSINT Response Caching - -**Goal**: Cache OSINT API responses to reduce API calls and improve response time. - -**Scope**: -- Cache by IP/domain with configurable TTL (default: 24 hours) -- Storage: SQLite database (`data/osint_cache.db`) -- Manual cache invalidation UI - -**Implementation**: - -| Component | File | Changes | -|-----------|------|---------| -| Cache layer | `app/pipeline/osint_cache.py` | New module | -| OSINT integration | `app/pipeline/osint.py` | Check cache before API call | -| UI controls | `app/ui/sidebar.py` | Cache stats & clear button | - -**Technical Details**: -```python -# app/pipeline/osint_cache.py -class OSINTCache: - def __init__(self, db_path: str, ttl_hours: int = 24): ... - def get(self, indicator: str, provider: str) -> dict | None: ... - def set(self, indicator: str, provider: str, data: dict) -> None: ... - def invalidate(self, indicator: str = None) -> int: ... -``` - -**Dependencies**: `sqlite3` (standard library) - ---- - -### 1.4 JA3/JA3S Fingerprint Lookup - -**Goal**: Identify TLS client/server implementations using JA3 fingerprints. - -**Scope**: -- Calculate JA3/JA3S from TLS handshake fields -- Lookup against known fingerprint databases -- Display in SSL/TLS analysis section - -**Implementation**: - -| Component | File | Changes | -|-----------|------|---------| -| JA3 calculator | `app/pipeline/ja3.py` | New module | -| Fingerprint DB | `app/data/ja3_fingerprints.json` | Static lookup table | -| Zeek integration | `app/pipeline/zeek.py` | Parse `ssl.log` JA3 fields | -| UI display | `app/ui/results_tab.py` | Add JA3 column to SSL table | - -**Technical Details**: -```python -# app/pipeline/ja3.py -def calculate_ja3(version: str, ciphers: list, extensions: list, - curves: list, point_formats: list) -> str: - """Calculate JA3 fingerprint hash.""" - -def lookup_ja3(ja3_hash: str) -> dict | None: - """Lookup JA3 in known fingerprint database.""" -``` - -**Dependencies**: None (Zeek already extracts JA3) - ---- - -## Phase 2: Medium Effort Features - -### 2.1 DNS Query/Response Carving - -**Goal**: Extract and analyze DNS queries and responses for threat detection. - -**Scope**: -- Parse DNS packets for query names, types, and responses -- Detect suspicious patterns (DGA, DNS tunneling, fast flux) -- Visualize DNS activity timeline - -**Implementation**: - -| Component | File | Changes | -|-----------|------|---------| -| DNS carver | `app/pipeline/dns_carve.py` | New module | -| DGA detection | `app/pipeline/dns_analysis.py` | Entropy/pattern analysis | -| Zeek DNS parsing | `app/pipeline/zeek.py` | Enhanced `dns.log` parsing | -| UI visualization | `app/ui/dns_tab.py` | New tab | - -**Technical Details**: -```python -# app/pipeline/dns_analysis.py -def detect_dga(domain: str) -> float: - """Return DGA probability score (0-1).""" - -def detect_tunneling(dns_records: list[dict]) -> dict: - """Analyze for DNS tunneling indicators.""" - -def detect_fast_flux(domain: str, responses: list[dict]) -> bool: - """Check for fast-flux DNS behavior.""" -``` - -**Dependencies**: None (uses existing Zeek/PyShark) - ---- - -### 2.2 Multi-PCAP Batch Analysis - -**Goal**: Analyze multiple PCAP files together with correlation across files. - -**Scope**: -- Upload multiple PCAPs -- Correlate IPs/domains across files -- Timeline aggregation -- Merged reporting - -**Implementation**: - -| Component | File | Changes | -|-----------|------|---------| -| Batch processor | `app/pipeline/batch.py` | New module | -| Session merger | `app/pipeline/merge.py` | New module | -| UI multi-upload | `app/ui/upload_tab.py` | Multi-file upload | -| Progress tracking | `app/pipeline/state.py` | Multi-file progress | - -**Technical Details**: -```python -# app/pipeline/batch.py -class BatchProcessor: - def __init__(self, pcap_paths: list[str]): ... - def process_all(self, phase: PhaseHandle) -> dict: ... - def correlate(self) -> dict: ... -``` - -**Dependencies**: None - ---- - -### 2.3 SSL/TLS Certificate Extraction - -**Goal**: Extract and display certificate details from TLS handshakes. - -**Scope**: -- Extract X.509 certificates from PCAP -- Parse certificate fields (subject, issuer, validity, SANs) -- Certificate chain validation -- Export certificates as PEM - -**Implementation**: - -| Component | File | Changes | -|-----------|------|---------| -| Cert extractor | `app/pipeline/tls_certs.py` | New module | -| PyShark integration | `app/pipeline/pyshark_pass.py` | Add TLS parsing | -| UI display | `app/ui/tls_tab.py` | New tab or section | - -**Technical Details**: -```python -# app/pipeline/tls_certs.py -@dataclass -class Certificate: - subject: dict - issuer: dict - not_before: datetime - not_after: datetime - serial: str - sans: list[str] - fingerprint_sha256: str - -def extract_certificates(pcap_path: str) -> list[Certificate]: ... -def validate_chain(certs: list[Certificate]) -> dict: ... -``` - -**Dependencies**: `cryptography` (for X.509 parsing) - ---- - -## Phase 3: High Effort Features - -### 3.1 YARA Rule Scanning - -**Goal**: Scan carved files against YARA rules for malware detection. - -**Scope**: -- Integrate YARA engine -- Include default rule sets (malware signatures) -- Support custom rule upload -- Display matches with context - -**Implementation**: - -| Component | File | Changes | -|-----------|------|---------| -| YARA scanner | `app/pipeline/yara_scan.py` | New module | -| Rule manager | `app/utils/yara_rules.py` | Load/manage rules | -| Default rules | `app/data/yara/` | Curated rule sets | -| UI integration | `app/ui/carve_tab.py` | Scan results display | - -**Technical Details**: -```python -# app/pipeline/yara_scan.py -class YARAScanner: - def __init__(self, rules_dir: str = None): ... - def add_rules(self, rules_path: str) -> None: ... - def scan_file(self, file_path: str) -> list[YARAMatch]: ... - def scan_directory(self, dir_path: str, phase: PhaseHandle) -> dict: ... - -@dataclass -class YARAMatch: - rule: str - tags: list[str] - strings: list[tuple[int, str, bytes]] - meta: dict -``` - -**Dependencies**: `yara-python` +## Status: Phases 1-3 delivered (v1.0.0, historical) + +This document originally laid out Phases 1-3 as *planned* work on the way to +a "v1.0.0 Enterprise Ready" milestone. All of it has since shipped: CSV/JSON +export, encrypted config persistence, OSINT response caching, JA3/JA3S +fingerprinting, DNS/DGA/tunneling analysis, TLS certificate extraction, +multi-PCAP batch analysis, YARA scanning, PDF report generation, and case +management are all in production (see `CHANGELOG.md` for the full history +through `1.0.0`). The section below is kept for historical context only — +it is **not** a live plan, and the module paths it originally proposed +(`app/ui/results_tab.py`, `app/db/cases.py`, `app/models/case.py`, etc.) were +early sketches; the actual implementation landed under different paths +(`app/utils/export.py`, `app/database/`, `app/pipeline/`, `app/ui/layout.py`, +`app/reports/pdf_generator.py`, and friends — see `CLAUDE.md` for the current +architecture map). --- -### 3.2 PDF Report Generation - -**Goal**: Generate professional PDF reports for documentation and sharing. - -**Scope**: -- Executive summary -- Detailed findings with visualizations -- IOC list -- Timeline of events -- Customizable branding - -**Implementation**: - -| Component | File | Changes | -|-----------|------|---------| -| Report generator | `app/reports/pdf_report.py` | New module | -| Templates | `app/reports/templates/` | Jinja2 templates | -| Chart export | `app/utils/chart_export.py` | Export Plotly as images | -| UI button | `app/ui/report_tab.py` | Generate PDF button | - -**Technical Details**: -```python -# app/reports/pdf_report.py -class PDFReportGenerator: - def __init__(self, analysis_data: dict, template: str = "default"): ... - def generate(self, output_path: str) -> None: ... - - def _render_executive_summary(self) -> str: ... - def _render_flow_analysis(self) -> str: ... - def _render_osint_findings(self) -> str: ... - def _render_timeline(self) -> str: ... - def _render_ioc_table(self) -> str: ... -``` - -**Dependencies**: `weasyprint` or `reportlab`, `jinja2` +## Post-1.0 — recently landed + +Work completed on top of the 1.0.0 baseline: + +- **Correctness fixes** — STIX export hash-type/IPv6/JA3 handling routed + through one shared helper; JA3 attribution de-duplicated via the + authoritative `lookup_ja3`; internal/private domains filtered out of OSINT + enrichment before egress; SQLite `cases.db` runs with WAL + busy-timeout to + avoid lock errors; attack-timeline persistence and beacon-penalty tuning + fixes for highly-regular HTTPS flows. +- **MITRE ATT&CK wired end-to-end** — the mapping engine now runs in the + pipeline runner and persists to `analyses.attack_json`; it's threaded + through the UI and API caller paths, rendered on the dashboard, included in + the PDF report, and reflected in the IOC feed's `mitre_techniques` field + (including for UI-saved analyses, so feed output stays consistent + regardless of how an analysis was produced). +- **HTTP analysis stage** — a new pipeline stage (`app/pipeline/http_analysis.py`) + adds user-agent/credential/URI heuristics, feeding cleartext-credential and + suspicious-UA signals into the correlation engine and the Raw Data tab, with + HTTPS beaconing detection tuned to reduce false positives on regular, + high-confidence flows. +- **Integrations API additions (tier-1)** — case management endpoints (list, + get, patch, delete, notes) under `/api/v1/cases`, a single-IOC exact-match + lookup endpoint, a CEF-formatted IOC feed (with CRLF escaping to prevent + syslog log injection), and an SSRF-safe job-completion webhook dispatched + from the queue worker. +- **CI hardening** — a coverage floor (`--cov-fail-under=58`) enforced in the + test job, an advisory `pip-audit` job plus Dependabot config for pip and + GitHub Actions updates, and a Docker build-and-test job that builds the + `test`-stage image (with real Zeek) and runs the in-image suite as a + second, non-required signal alongside the host-based `test` job. +- **Demo capture** — a synthetic `pcaps/demo.pcap` (DNS/HTTP/beacon traffic) + plus a "load demo capture" button for a first-run experience without + needing a real PCAP on hand. --- -### 3.3 Case Management - -**Goal**: Organize analyses into cases with notes, tags, and history. - -**Scope**: -- Create/manage cases -- Link multiple PCAPs to a case -- Add analyst notes and tags -- Search across cases -- Export case archive - -**Implementation**: - -| Component | File | Changes | -|-----------|------|---------| -| Case model | `app/models/case.py` | New module | -| Database | `app/db/cases.py` | SQLite persistence | -| Case API | `app/api/cases.py` | CRUD operations | -| UI | `app/ui/cases_tab.py` | Full case management UI | - -**Technical Details**: -```python -# app/models/case.py -@dataclass -class Case: - id: str - title: str - description: str - created_at: datetime - updated_at: datetime - tags: list[str] - pcaps: list[str] - notes: list[Note] - iocs: list[IOC] - status: CaseStatus - -# app/db/cases.py -class CaseDB: - def create(self, case: Case) -> str: ... - def get(self, case_id: str) -> Case | None: ... - def update(self, case: Case) -> None: ... - def delete(self, case_id: str) -> None: ... - def search(self, query: str, tags: list[str] = None) -> list[Case]: ... -``` - -**Dependencies**: `sqlite3`, possibly `sqlalchemy` - ---- - -## Implementation Priority Matrix - -| Feature | Effort | Value | Priority | -|---------|--------|-------|----------| -| CSV/JSON Export | Low | High | P0 | -| Config Persistence | Low | High | P0 | -| OSINT Caching | Low | Medium | P1 | -| JA3 Lookup | Low | Medium | P1 | -| DNS Carving | Medium | High | P1 | -| Multi-PCAP | Medium | High | P2 | -| TLS Cert Extraction | Medium | Medium | P2 | -| YARA Scanning | High | High | P2 | -| PDF Reports | High | Medium | P3 | -| Case Management | High | High | P3 | - ---- - -## Dependencies Summary - -**New packages required**: -``` -cryptography>=41.0.0 # Config encryption, TLS cert parsing -yara-python>=4.3.0 # YARA scanning (optional) -weasyprint>=60.0 # PDF generation (optional) -jinja2>=3.1.0 # Report templates (optional) -``` - ---- - -## Architecture Considerations - -### Module Organization - -``` -app/ -├── pipeline/ -│ ├── dns_carve.py # New: DNS carving -│ ├── dns_analysis.py # New: DGA/tunneling detection -│ ├── ja3.py # New: JA3 fingerprinting -│ ├── osint_cache.py # New: OSINT caching -│ ├── tls_certs.py # New: Certificate extraction -│ ├── yara_scan.py # New: YARA scanning -│ └── batch.py # New: Multi-PCAP processing -├── utils/ -│ ├── export.py # New: CSV/JSON export -│ ├── config_manager.py # New: Config persistence -│ └── yara_rules.py # New: YARA rule management -├── reports/ -│ ├── pdf_report.py # New: PDF generation -│ └── templates/ # New: Report templates -├── models/ -│ └── case.py # New: Case data model -├── db/ -│ └── cases.py # New: Case database -└── security/ - └── crypto.py # New: Encryption utilities -``` - -### Data Flow - -``` - ┌─────────────────┐ - │ PCAP Upload │ - └────────┬────────┘ - │ - ┌────────────────┼────────────────┐ - ▼ ▼ ▼ - ┌───────────┐ ┌───────────┐ ┌───────────┐ - │ Zeek │ │ PyShark │ │ Tshark │ - └─────┬─────┘ └─────┬─────┘ └─────┬─────┘ - │ │ │ - │ ┌───────────┴───────────┐ │ - │ ▼ ▼ │ - │ ┌───────┐ ┌───────┐ │ - │ │ JA3 │ │ DNS │ │ - │ └───────┘ │ Carve │ │ - │ └───────┘ │ - │ │ - └──────────────┬──────────────────┘ - ▼ - ┌─────────────────────┐ - │ OSINT Enrichment │◄──── Cache - └──────────┬──────────┘ - │ - ┌────────────┼────────────┐ - ▼ ▼ ▼ - ┌──────────┐ ┌──────────┐ ┌──────────┐ - │ Beaconing│ │ YARA │ │ TLS │ - │ Detection│ │ Scanning│ │ Certs │ - └──────────┘ └──────────┘ └──────────┘ - │ - ▼ - ┌─────────────────────┐ - │ LLM Analysis │ - └──────────┬──────────┘ - │ - ┌────────────┼────────────┐ - ▼ ▼ ▼ - ┌──────────┐ ┌──────────┐ ┌──────────┐ - │ Export │ │ PDF │ │ Case │ - │ CSV/JSON │ │ Report │ │ Mgmt │ - └──────────┘ └──────────┘ └──────────┘ -``` - ---- - -## Version Milestones - -### v0.3.0 - Export & Persistence -- CSV/JSON Export -- Configuration Persistence -- OSINT Caching - -### v0.4.0 - Enhanced Analysis -- JA3/JA3S Fingerprinting -- DNS Carving & Analysis -- SSL/TLS Certificate Extraction - -### v0.5.0 - Advanced Features -- Multi-PCAP Batch Analysis -- YARA Rule Scanning - -### v1.0.0 - Enterprise Ready -- PDF Report Generation -- Case Management System -- Full documentation +## Future ideas (not yet started) + +Genuine candidates for future work — none of these are in progress: + +- Lateral-movement / SMB traffic detection +- JA4 / JARM fingerprinting (JA3/JA3S only today) +- IPv6-aware OSINT enrichment (current provider integrations are IPv4-centric) +- MISP and Sigma export formats (STIX 2.0/2.1, ATT&CK Navigator, and CSV/JSON + are already supported) +- Per-case configuration profiles (today's config is global, one machine-wide + `ConfigManager` instance) +- Multi-user support with role-based access control and an audit trail + (the app currently assumes a single local analyst) +- Configurable case/analysis retention policy (currently manual "clear data" + actions only; no automatic expiry) diff --git a/pcaps/_make_demo.py b/pcaps/_make_demo.py new file mode 100644 index 0000000..8b222c2 --- /dev/null +++ b/pcaps/_make_demo.py @@ -0,0 +1,308 @@ +"""Generate ``pcaps/demo.pcap`` deterministically. + +Run once locally to regenerate the fixture if it's lost or needs to evolve: + + python pcaps/_make_demo.py + +The committed ``demo.pcap`` is small (a few KB, dozens of packets) and +entirely SYNTHETIC — license-clean by construction. It exists so a first-run +user has something to load via the "Load demo capture" button instead of +facing an empty app, and so that loading it actually exercises real +detection paths in the pipeline: + +- DNS query/response pairs (UDP/53): two normal-looking domains plus one + digit-heavy, all-consonant DGA-looking domain that + ``app.pipeline.dns_analysis.detect_dga`` scores above its confirmation + threshold (via the digit-ratio / consonant-ratio / no-vowel bonuses). +- A minimal TLS ClientHello (TCP/443) with an SNI extension, for protocol + variety. +- Two HTTP GET requests (TCP/80): one with a normal browser User-Agent, one + with a scripting-tool User-Agent (``python-requests``) that + ``app.pipeline`` HTTP heuristics flag as suspicious. +- A periodic TCP beacon (13 packets, exactly 30s apart) to a well-known C2 + port (4444, see ``app.config.C2_SUSPECT_PORTS``) so + ``app.pipeline.beacon.rank_beaconing`` has a clear candidate. + +All destination IPs are from documentation/test ranges (RFC 5737 +TEST-NET-3 ``203.0.113.0/24`` and TEST-NET-2 ``198.51.100.0/24``) — publicly +reserved for exactly this purpose, never routed on the real Internet — so +the capture is obviously synthetic and safe to publish. The source is an +RFC 1918 client. Ethernet MACs and every packet timestamp are pinned (like +``tests/fixtures/_make_tiny.py``) so regenerating the fixture produces +byte-identical output, no matter where or when the script runs. + +The DGA domain resolves (in the synthetic DNS response) to the same IP the +beacon later targets, and the second "normal" domain resolves to the TLS +server IP — a small, deliberate narrative thread for anyone poking at the +demo data, not something the pipeline depends on. + +Tests must NOT import this module — they read the committed ``demo.pcap`` +file directly. This module is a developer tool, not test code. +""" + +from __future__ import annotations + +import pathlib +import struct + +import scapy.all as scapy + +FIXTURE_PATH = pathlib.Path(__file__).parent / "demo.pcap" + +# Pinned epoch for deterministic record timestamps: 2026-01-01T00:00:00 UTC +# (same anchor as tests/fixtures/_make_tiny.py). +BASE_EPOCH = 1767225600 + +# --- Deterministic L2/L3 addressing --- +CLIENT_MAC = "00:00:00:00:00:01" +SERVER_MAC = "00:00:00:00:00:02" +CLIENT_IP = "10.0.0.50" # RFC 1918 + +# Documentation/test-net destinations (RFC 5737) — never routed publicly. +DNS_SERVER_IP = "203.0.113.53" +NORMAL_HTTP_IP = "198.51.100.10" +TLS_SERVER_IP = "203.0.113.50" +SUSPICIOUS_HTTP_IP = "203.0.113.10" +BEACON_IP = "203.0.113.99" +BEACON_PORT = 4444 # in app.config.C2_SUSPECT_PORTS + +# --- Domains --- +DOMAIN_NORMAL_1 = "www.example.com" +DOMAIN_NORMAL_2 = "vault.example.net" +# Digit-heavy, all-consonant-alpha synthetic name — not a real +# registered domain. Scores well above app.pipeline.dns_analysis's DGA +# confirmation threshold (0.5) via digit-ratio + consonant-ratio + no-vowels +# heuristics. +DOMAIN_DGA = "x7k2p9qz3v.com" + + +def _eth_ip(src_ip: str, dst_ip: str): + """Return an Ether/IP layer stack with MACs pinned by direction.""" + if src_ip == CLIENT_IP: + return scapy.Ether(src=CLIENT_MAC, dst=SERVER_MAC) / scapy.IP(src=src_ip, dst=dst_ip) + return scapy.Ether(src=SERVER_MAC, dst=CLIENT_MAC) / scapy.IP(src=src_ip, dst=dst_ip) + + +def _dns_pair(ts: float, query_id: int, domain: str, resolved_ip: str, sport: int) -> list: + """Build a DNS query + response packet pair for ``domain``.""" + query = ( + _eth_ip(CLIENT_IP, DNS_SERVER_IP) + / scapy.UDP(sport=sport, dport=53) + / scapy.DNS(id=query_id, rd=1, qd=scapy.DNSQR(qname=domain)) + ) + query.time = ts + + response = ( + _eth_ip(DNS_SERVER_IP, CLIENT_IP) + / scapy.UDP(sport=53, dport=sport) + / scapy.DNS( + id=query_id, + qr=1, + aa=1, + rd=1, + ra=1, + qd=scapy.DNSQR(qname=domain), + an=scapy.DNSRR(rrname=domain, type="A", ttl=300, rdata=resolved_ip), + ) + ) + response.time = ts + 0.05 + + return [query, response] + + +def _tcp_http_exchange( + ts: float, client_ip: str, server_ip: str, sport: int, dport: int, req_bytes: bytes, resp_bytes: bytes +) -> list: + """Build a minimal SYN/SYN-ACK/ACK + PSH request/response TCP exchange.""" + cli_seq = 1000 + srv_seq = 2000 + pkts = [] + + syn = _eth_ip(client_ip, server_ip) / scapy.TCP(sport=sport, dport=dport, flags="S", seq=cli_seq) + syn.time = ts + pkts.append(syn) + + synack = _eth_ip(server_ip, client_ip) / scapy.TCP( + sport=dport, dport=sport, flags="SA", seq=srv_seq, ack=cli_seq + 1 + ) + synack.time = ts + 0.01 + pkts.append(synack) + + ack = _eth_ip(client_ip, server_ip) / scapy.TCP( + sport=sport, dport=dport, flags="A", seq=cli_seq + 1, ack=srv_seq + 1 + ) + ack.time = ts + 0.02 + pkts.append(ack) + + request = ( + _eth_ip(client_ip, server_ip) + / scapy.TCP(sport=sport, dport=dport, flags="PA", seq=cli_seq + 1, ack=srv_seq + 1) + / scapy.Raw(load=req_bytes) + ) + request.time = ts + 0.03 + pkts.append(request) + + response = ( + _eth_ip(server_ip, client_ip) + / scapy.TCP(sport=dport, dport=sport, flags="PA", seq=srv_seq + 1, ack=cli_seq + 1 + len(req_bytes)) + / scapy.Raw(load=resp_bytes) + ) + response.time = ts + 0.04 + pkts.append(response) + + final_ack = _eth_ip(client_ip, server_ip) / scapy.TCP( + sport=sport, + dport=dport, + flags="A", + seq=cli_seq + 1 + len(req_bytes), + ack=srv_seq + 1 + len(resp_bytes), + ) + final_ack.time = ts + 0.05 + pkts.append(final_ack) + + return pkts + + +def _build_tls_client_hello(sni: str) -> bytes: + """Build a minimal, well-formed TLS 1.2 ClientHello record with an SNI extension. + + Not a real handshake (no key share, only two cipher suites) — just enough + structure for a dissector to recognize it as a TLS ClientHello carrying a + server name, for protocol variety in the demo capture. + """ + sni_bytes = sni.encode() + server_name_entry = b"\x00" + struct.pack(">H", len(sni_bytes)) + sni_bytes # type=host_name(0) + server_name_list = struct.pack(">H", len(server_name_entry)) + server_name_entry + ext_server_name = struct.pack(">HH", 0x0000, len(server_name_list)) + server_name_list + extensions = ext_server_name + + cipher_suites = bytes([0xC0, 0x2F, 0xC0, 0x30, 0x00, 0x9C, 0x00, 0x9D]) + random_bytes = bytes(range(32)) # deterministic filler, not real randomness + + body = ( + b"\x03\x03" # client_version: TLS 1.2 + + random_bytes + + b"\x00" # session_id length: 0 + + struct.pack(">H", len(cipher_suites)) + + cipher_suites + + b"\x01\x00" # compression methods: length 1, method null(0) + + struct.pack(">H", len(extensions)) + + extensions + ) + handshake = b"\x01" + len(body).to_bytes(3, "big") + body # handshake type 1 = ClientHello + return b"\x16\x03\x01" + struct.pack(">H", len(handshake)) + handshake # content type 22 = Handshake + + +def _tls_hello_exchange(ts: float, client_ip: str, server_ip: str, sport: int, dport: int, sni: str) -> list: + """Build a minimal SYN/SYN-ACK/ACK + ClientHello TCP exchange.""" + cli_seq = 3000 + srv_seq = 4000 + pkts = [] + + syn = _eth_ip(client_ip, server_ip) / scapy.TCP(sport=sport, dport=dport, flags="S", seq=cli_seq) + syn.time = ts + pkts.append(syn) + + synack = _eth_ip(server_ip, client_ip) / scapy.TCP( + sport=dport, dport=sport, flags="SA", seq=srv_seq, ack=cli_seq + 1 + ) + synack.time = ts + 0.01 + pkts.append(synack) + + ack = _eth_ip(client_ip, server_ip) / scapy.TCP( + sport=sport, dport=dport, flags="A", seq=cli_seq + 1, ack=srv_seq + 1 + ) + ack.time = ts + 0.02 + pkts.append(ack) + + hello_bytes = _build_tls_client_hello(sni) + hello = ( + _eth_ip(client_ip, server_ip) + / scapy.TCP(sport=sport, dport=dport, flags="PA", seq=cli_seq + 1, ack=srv_seq + 1) + / scapy.Raw(load=hello_bytes) + ) + hello.time = ts + 0.03 + pkts.append(hello) + + return pkts + + +def _beacon_packets( + start_ts: float, count: int, interval: float, client_ip: str, server_ip: str, sport: int, dport: int +) -> list: + """Build ``count`` small TCP packets at an exact ``interval`` — a periodic beacon.""" + pkts = [] + seq = 5000 + for i in range(count): + payload = f"chk-in-{i:02d}".encode() + pkt = ( + _eth_ip(client_ip, server_ip) + / scapy.TCP(sport=sport, dport=dport, flags="PA", seq=seq, ack=1) + / scapy.Raw(load=payload) + ) + pkt.time = start_ts + i * interval + seq += len(payload) + pkts.append(pkt) + return pkts + + +def build_packets() -> list: + """Return a deterministic, chronologically ordered list of demo packets.""" + pkts: list = [] + + # DNS: two normal lookups, one DGA-looking lookup. + pkts += _dns_pair(BASE_EPOCH + 0, 1, DOMAIN_NORMAL_1, NORMAL_HTTP_IP, sport=40001) + pkts += _dns_pair(BASE_EPOCH + 1, 2, DOMAIN_NORMAL_2, TLS_SERVER_IP, sport=40002) + pkts += _dns_pair(BASE_EPOCH + 2, 3, DOMAIN_DGA, BEACON_IP, sport=40003) + + # TLS: minimal ClientHello with SNI, for protocol variety. + pkts += _tls_hello_exchange(BASE_EPOCH + 5, CLIENT_IP, TLS_SERVER_IP, sport=50001, dport=443, sni=DOMAIN_NORMAL_2) + + # HTTP: suspicious tool User-Agent hitting a hardcoded IP (no prior DNS lookup). + suspicious_req = ( + b"GET /gate.php?id=1 HTTP/1.1\r\n" + b"Host: 203.0.113.10\r\n" + b"User-Agent: python-requests/2.31.0\r\n" + b"Accept: */*\r\n" + b"Connection: close\r\n\r\n" + ) + suspicious_resp = b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 2\r\n\r\nOK" + pkts += _tcp_http_exchange( + BASE_EPOCH + 10, CLIENT_IP, SUSPICIOUS_HTTP_IP, 51000, 80, suspicious_req, suspicious_resp + ) + + # HTTP: normal browser User-Agent to the site the client actually resolved. + normal_req = ( + b"GET /index.html HTTP/1.1\r\n" + b"Host: www.example.com\r\n" + b"User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36\r\n" + b"Accept: text/html\r\n" + b"Connection: close\r\n\r\n" + ) + normal_resp = b"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: 13\r\n\r\nHello, world!" + pkts += _tcp_http_exchange(BASE_EPOCH + 15, CLIENT_IP, NORMAL_HTTP_IP, 51001, 80, normal_req, normal_resp) + + # Beacon: 13 packets, exactly 30s apart, to a well-known C2 port. + pkts += _beacon_packets( + BASE_EPOCH + 20, + count=13, + interval=30.0, + client_ip=CLIENT_IP, + server_ip=BEACON_IP, + sport=55000, + dport=BEACON_PORT, + ) + + pkts.sort(key=lambda p: p.time) + return pkts + + +def main() -> None: + pkts = build_packets() + scapy.wrpcap(str(FIXTURE_PATH), pkts) + size = FIXTURE_PATH.stat().st_size + print(f"wrote {FIXTURE_PATH} ({size} bytes, {len(pkts)} packets)") + + +if __name__ == "__main__": + main() diff --git a/pcaps/demo.pcap b/pcaps/demo.pcap new file mode 100644 index 0000000..ec84e9c Binary files /dev/null and b/pcaps/demo.pcap differ diff --git a/pyproject.toml b/pyproject.toml index e78fb42..aca347f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,3 +17,9 @@ quote-style = "double" indent-style = "space" skip-magic-trailing-comma = false line-ending = "auto" + +[tool.pytest.ini_options] +filterwarnings = ["ignore::DeprecationWarning"] + +[tool.coverage.report] +exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:", "raise NotImplementedError"] diff --git a/scripts/capture_screenshots.py b/scripts/capture_screenshots.py index 887c4f8..14a9533 100644 --- a/scripts/capture_screenshots.py +++ b/scripts/capture_screenshots.py @@ -1,10 +1,10 @@ """Capture README screenshots of the PCAP Hunter UI and redact all IPs. Drives a headless Chromium via Playwright against a running Streamlit -instance (default http://localhost:8501), uploads ``data/sample.pcap`` -by entering its path into the "type a container path" text input, -clicks Extract & Analyze, waits for the pipeline to finish, then -snapshots each tab at 1440×900. +instance (default http://localhost:8501), uploads ``pcaps/demo.pcap`` +(the committed synthetic DNS/HTTP/beacon capture) by entering its path +into the "type a container path" text input, clicks Extract & Analyze, +waits for the pipeline to finish, then snapshots each tab at 1440×900. After capture, every PNG is post-processed with Pillow to redact IPv4 addresses. We do this at the pixel level using OCR-free regex scanning @@ -32,7 +32,7 @@ REPO_ROOT = Path(__file__).resolve().parent.parent OUT_DIR = REPO_ROOT / "docs" / "images" -SAMPLE_PCAP = REPO_ROOT / "data" / "sample.pcap" +SAMPLE_PCAP = REPO_ROOT / "pcaps" / "demo.pcap" # IPv4 pattern — matches anything that looks like a.b.c.d with 0-255 octets. # We redact any IP, including RFC1918/loopback — the goal is zero IPs visible. @@ -106,7 +106,7 @@ def upload_sample_pcap(page: Page, pcap_path: str) -> None: """Fill the 'type a container path' text input to load the sample. ``pcap_path`` must be valid for the SERVER process — when the app runs in - Docker that means a container-visible path like ``data/sample.pcap``, not + Docker that means a container-visible path like ``pcaps/demo.pcap``, not the host's absolute path. """ path_input = page.get_by_label("...or type a container path (e.g., /data/capture.pcap)") @@ -139,7 +139,7 @@ def run_extract_analyze(page: Page, wait_for_llm: bool = False, timeout_s: int = if "Please upload a PCAP or provide a valid path" in page.inner_text("body"): raise RuntimeError( "the server rejected the pcap path — pass a path the SERVER can see " - "(container-relative like data/sample.pcap when using Docker)" + "(container-relative like pcaps/demo.pcap when using Docker)" ) print(f" waiting for pipeline (up to {timeout_s}s)...") deadline = time.time() + timeout_s @@ -480,7 +480,7 @@ def main() -> int: ) parser.add_argument( "--pcap", - default="data/sample.pcap", + default="pcaps/demo.pcap", help="pcap path AS THE SERVER SEES IT (container-relative when the app runs in Docker)", ) parser.add_argument("--pipeline-timeout", type=int, default=600, help="seconds to wait for the data stages") @@ -498,7 +498,7 @@ def main() -> int: return 0 if not SAMPLE_PCAP.is_file() and not args.skip_analysis: - print(f"ERROR: {SAMPLE_PCAP} not found. Copy sample.pcap → data/ first.", file=sys.stderr) + print(f"ERROR: {SAMPLE_PCAP} not found. Demo capture should be committed at pcaps/demo.pcap.", file=sys.stderr) return 1 redact = not args.keep_ips diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index b5ed6b2..5ad4fa4 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -26,6 +26,8 @@ def _settings(main: str | None = "MAIN_KEY", feed: str | None = "FEED_KEY") -> A job_ttl_days=30, require_https=False, cors_origins=[], + webhook_timeout_seconds=10, + webhook_max_retries=2, ) diff --git a/tests/api/test_cases.py b/tests/api/test_cases.py index 00cfc3b..1df833d 100644 --- a/tests/api/test_cases.py +++ b/tests/api/test_cases.py @@ -416,3 +416,246 @@ def test_api_reports_dir_env_takes_precedence(client, tmp_path, monkeypatch): assert not (legacy_reports / f"{case_id}.pdf").exists(), ( "PCAP_HUNTER_REPORTS_DIR must be ignored when PCAP_HUNTER_API_REPORTS_DIR is set" ) + + +# ==================== GET /api/v1/cases (list) ==================== + + +def test_list_cases_returns_seeded_cases(client): + from app.api.deps import get_repo + from app.database.models import Case + + repo = get_repo() + id1 = repo.create_case(Case(title="alpha")) + id2 = repo.create_case(Case(title="beta")) + + r = client.get("/api/v1/cases", headers={"Authorization": "Bearer MAIN"}) + assert r.status_code == 200 + body = r.json() + assert body["count"] == 2 + ids = {c["id"] for c in body["cases"]} + assert ids == {id1, id2} + # Light shape: list_cases() never loads analyses/notes, so the response must not + # imply case content it never fetched. + assert "analyses" not in body["cases"][0] + assert "notes" not in body["cases"][0] + + +def test_list_cases_filters_by_status(client): + from app.api.deps import get_repo + from app.database.models import Case, CaseStatus + + repo = get_repo() + repo.create_case(Case(title="open-case", status=CaseStatus.OPEN)) + closed_id = repo.create_case(Case(title="closed-case", status=CaseStatus.CLOSED)) + + r = client.get("/api/v1/cases", params={"status": "closed"}, headers={"Authorization": "Bearer MAIN"}) + assert r.status_code == 200 + body = r.json() + assert body["count"] == 1 + assert body["cases"][0]["id"] == closed_id + + +def test_list_cases_filters_by_tag(client): + from app.api.deps import get_repo + from app.database.models import Case + + repo = get_repo() + tagged_id = repo.create_case(Case(title="tagged", tags=["malware"])) + repo.create_case(Case(title="untagged")) + + r = client.get("/api/v1/cases", params={"tag": "malware"}, headers={"Authorization": "Bearer MAIN"}) + assert r.status_code == 200 + body = r.json() + assert body["count"] == 1 + assert body["cases"][0]["id"] == tagged_id + + +def test_list_cases_search_filter(client): + from app.api.deps import get_repo + from app.database.models import Case + + repo = get_repo() + match_id = repo.create_case(Case(title="beacon investigation")) + repo.create_case(Case(title="unrelated")) + + r = client.get("/api/v1/cases", params={"search": "beacon"}, headers={"Authorization": "Bearer MAIN"}) + assert r.status_code == 200 + body = r.json() + assert body["count"] == 1 + assert body["cases"][0]["id"] == match_id + + +def test_list_cases_limit_offset(client): + from app.api.deps import get_repo + from app.database.models import Case + + repo = get_repo() + for i in range(3): + repo.create_case(Case(title=f"case-{i}")) + + r = client.get("/api/v1/cases", params={"limit": 1, "offset": 0}, headers={"Authorization": "Bearer MAIN"}) + assert r.status_code == 200 + assert len(r.json()["cases"]) == 1 + + +def test_list_cases_requires_auth(client): + r = client.get("/api/v1/cases") + assert r.status_code == 401 + + +# ==================== PATCH /api/v1/cases/{id} ==================== + + +def test_patch_case_updates_fields(client): + from app.api.deps import get_repo + from app.database.models import Case + + case_id = get_repo().create_case(Case(title="orig", description="orig-desc")) + + r = client.patch( + f"/api/v1/cases/{case_id}", + json={"title": "new-title", "severity": "high", "tags": ["a", "b"]}, + headers={"Authorization": "Bearer MAIN"}, + ) + assert r.status_code == 200 + body = r.json() + assert body["title"] == "new-title" + assert body["severity"] == "high" + assert sorted(body["tags"]) == ["a", "b"] + assert body["description"] == "orig-desc" # untouched + + +def test_patch_case_status_closed_sets_closed_at(client): + from app.api.deps import get_repo + from app.database.models import Case + + case_id = get_repo().create_case(Case(title="orig")) + + r = client.patch( + f"/api/v1/cases/{case_id}", + json={"status": "closed"}, + headers={"Authorization": "Bearer MAIN"}, + ) + assert r.status_code == 200 + body = r.json() + assert body["status"] == "closed" + assert body["closed_at"] is not None + + +def test_patch_case_404_on_missing(client): + r = client.patch( + "/api/v1/cases/zzzz9999", + json={"title": "x"}, + headers={"Authorization": "Bearer MAIN"}, + ) + assert r.status_code == 404 + assert r.json()["code"] == "case_not_found" + + +def test_patch_case_invalid_status_returns_422(client): + from app.api.deps import get_repo + from app.database.models import Case + + case_id = get_repo().create_case(Case(title="orig")) + + r = client.patch( + f"/api/v1/cases/{case_id}", + json={"status": "not-a-status"}, + headers={"Authorization": "Bearer MAIN"}, + ) + assert r.status_code == 422 + + +def test_patch_case_invalid_severity_returns_422(client): + from app.api.deps import get_repo + from app.database.models import Case + + case_id = get_repo().create_case(Case(title="orig")) + + r = client.patch( + f"/api/v1/cases/{case_id}", + json={"severity": "not-a-severity"}, + headers={"Authorization": "Bearer MAIN"}, + ) + assert r.status_code == 422 + + +def test_patch_case_empty_body_is_noop(client): + from app.api.deps import get_repo + from app.database.models import Case + + case_id = get_repo().create_case(Case(title="orig", description="d")) + + r = client.patch(f"/api/v1/cases/{case_id}", json={}, headers={"Authorization": "Bearer MAIN"}) + assert r.status_code == 200 + body = r.json() + assert body["title"] == "orig" + assert body["description"] == "d" + + +# ==================== POST/GET /api/v1/cases/{id}/notes ==================== + + +def test_post_note_adds_note(client): + from app.api.deps import get_repo + from app.database.models import Case + + case_id = get_repo().create_case(Case(title="x")) + + r = client.post( + f"/api/v1/cases/{case_id}/notes", + json={"content": "analyst note"}, + headers={"Authorization": "Bearer MAIN"}, + ) + assert r.status_code == 201 + body = r.json() + assert body["content"] == "analyst note" + assert isinstance(body["id"], int) + + +def test_post_note_404_on_missing_case(client): + r = client.post( + "/api/v1/cases/zzzz9999/notes", + json={"content": "x"}, + headers={"Authorization": "Bearer MAIN"}, + ) + assert r.status_code == 404 + assert r.json()["code"] == "case_not_found" + + +def test_post_note_empty_content_returns_422(client): + from app.api.deps import get_repo + from app.database.models import Case + + case_id = get_repo().create_case(Case(title="x")) + + r = client.post( + f"/api/v1/cases/{case_id}/notes", + json={"content": ""}, + headers={"Authorization": "Bearer MAIN"}, + ) + assert r.status_code == 422 + + +def test_get_notes_lists_notes(client): + from app.api.deps import get_repo + from app.database.models import Case + + repo = get_repo() + case_id = repo.create_case(Case(title="x")) + repo.add_note(case_id, "first note") + repo.add_note(case_id, "second note") + + r = client.get(f"/api/v1/cases/{case_id}/notes", headers={"Authorization": "Bearer MAIN"}) + assert r.status_code == 200 + body = r.json() + assert len(body["notes"]) == 2 + contents = {n["content"] for n in body["notes"]} + assert contents == {"first note", "second note"} + + +def test_get_notes_404_on_missing_case(client): + r = client.get("/api/v1/cases/zzzz9999/notes", headers={"Authorization": "Bearer MAIN"}) + assert r.status_code == 404 + assert r.json()["code"] == "case_not_found" diff --git a/tests/api/test_feed_query.py b/tests/api/test_feed_query.py index 3f0ed9b..ea09679 100644 --- a/tests/api/test_feed_query.py +++ b/tests/api/test_feed_query.py @@ -109,3 +109,118 @@ def test_duplicate_ioc_takes_max_severity_score(tmp_path): row = next(r for r in rows if r["value"] == "198.51.100.7") assert row["score"] == 100 assert row["severity"] == "critical" + + +def test_query_iocs_derives_mitre_techniques(tmp_path): + """mitre_techniques must be derived from the analysis's technique IDs, not hardcoded [].""" + repo = CaseRepository(db_path=str(tmp_path / "t.db")) + repo.create_case(Case(id="case9001", title="t")) + analysis = Analysis( + case_id="case9001", + pcap_path="/tmp/x.pcap", + mitre_techniques=["T1071.001"], + iocs=[IOC(ioc_type=IOCType.IP, value="9.9.9.9", severity=Severity.HIGH)], + ) + repo.save_analysis(analysis) + + rows = query_iocs(repo, IOCFilter()) + row = next(r for r in rows if r["value"] == "9.9.9.9") + assert row["mitre_techniques"] == ["T1071.001"] + + +def test_query_iocs_no_techniques_returns_empty_list(tmp_path): + """An analysis with no mapped techniques must yield [], not crash the LEFT JOIN.""" + repo = _seed(tmp_path) + rows = query_iocs(repo, IOCFilter()) + for r in rows: + assert r["mitre_techniques"] == [] + + +def test_query_iocs_multi_technique_multi_tag_no_duplication(tmp_path): + """Fan-out guard: the LEFT JOIN analysis_techniques multiplies result rows + (one per technique), which could silently corrupt any non-DISTINCT aggregate. + Seed multiple techniques AND multiple tags on the same IOC's analysis and + confirm both tags and technique IDs come back deduped, not repeated.""" + repo = CaseRepository(db_path=str(tmp_path / "t.db")) + repo.create_case(Case(id="case9002", title="t", tags=["tag-a", "tag-b"])) + analysis = Analysis( + case_id="case9002", + pcap_path="/tmp/y.pcap", + mitre_techniques=["T1071.001", "T1059"], + iocs=[IOC(ioc_type=IOCType.IP, value="8.8.8.8", severity=Severity.HIGH)], + ) + repo.save_analysis(analysis) + + rows = query_iocs(repo, IOCFilter()) + row = next(r for r in rows if r["value"] == "8.8.8.8") + assert row["mitre_techniques"] == ["T1059", "T1071.001"] # sorted + assert sorted(row["tags"]) == ["tag-a", "tag-b"] + + +def test_query_iocs_value_filter_exact_match(tmp_path): + """An exact value filter returns only the matching IOC row.""" + repo = _seed(tmp_path) + rows = query_iocs(repo, IOCFilter(value="1.2.3.4")) + assert len(rows) == 1 + assert rows[0]["value"] == "1.2.3.4" + assert rows[0]["score"] == 75 + assert rows[0]["severity"] == "high" + + +def test_query_iocs_value_filter_unknown_returns_empty(tmp_path): + repo = _seed(tmp_path) + rows = query_iocs(repo, IOCFilter(value="9.9.9.9")) + assert rows == [] + + +def test_query_iocs_value_filter_is_exact_not_substring(tmp_path): + """value='1.2.3.4' must not also match '1.2.3.40' (substring false-positive).""" + repo = CaseRepository(db_path=str(tmp_path / "t.db")) + repo.create_case(Case(id="case9004", title="t")) + analysis = Analysis( + case_id="case9004", + pcap_path="/tmp/z.pcap", + iocs=[ + IOC(ioc_type=IOCType.IP, value="1.2.3.4", severity=Severity.HIGH), + IOC(ioc_type=IOCType.IP, value="1.2.3.40", severity=Severity.LOW), + ], + ) + repo.save_analysis(analysis) + + rows = query_iocs(repo, IOCFilter(value="1.2.3.4")) + assert len(rows) == 1 + assert rows[0]["value"] == "1.2.3.4" + + +def test_query_iocs_value_filter_composes_with_type_filter(tmp_path): + """value filter must AND with the existing type filter, not override it.""" + repo = _seed(tmp_path) + rows = query_iocs(repo, IOCFilter(value="1.2.3.4", types=["domain"])) + assert rows == [] + rows = query_iocs(repo, IOCFilter(value="1.2.3.4", types=["ip"])) + assert len(rows) == 1 + assert rows[0]["value"] == "1.2.3.4" + + +def test_query_iocs_techniques_dont_affect_pagination(tmp_path): + """LIMIT/OFFSET must still page over distinct (ioc_type, value) groups, not + over the post-join fan-out rows, when techniques are present.""" + repo = CaseRepository(db_path=str(tmp_path / "t.db")) + repo.create_case(Case(id="case9003", title="t", status=CaseStatus.IN_PROGRESS, severity=Severity.LOW)) + analysis = Analysis( + case_id="case9003", + pcap_path="x.pcap", + features={"artifacts": {}}, + mitre_techniques=["T1071.001", "T1059", "T1105"], + ) + analysis.iocs = [ + IOC(ioc_type=IOCType.IP, value="203.0.113.10", severity=Severity.HIGH), + IOC(ioc_type=IOCType.IP, value="203.0.113.11", severity=Severity.HIGH), + ] + repo.save_analysis(analysis) + + page1 = query_iocs(repo, IOCFilter(limit=1, offset=0)) + page2 = query_iocs(repo, IOCFilter(limit=1, offset=1)) + assert len(page1) == 1 + assert len(page2) == 1 + assert page1[0]["value"] != page2[0]["value"] diff --git a/tests/api/test_gc.py b/tests/api/test_gc.py index fd216e0..41c41fc 100644 --- a/tests/api/test_gc.py +++ b/tests/api/test_gc.py @@ -28,6 +28,8 @@ def _settings(tmp_path) -> APISettings: job_ttl_days=3, require_https=False, cors_origins=[], + webhook_timeout_seconds=10, + webhook_max_retries=2, ) diff --git a/tests/api/test_iocs.py b/tests/api/test_iocs.py index c24a2ab..9cff103 100644 --- a/tests/api/test_iocs.py +++ b/tests/api/test_iocs.py @@ -68,6 +68,55 @@ def test_iocs_json_filter_by_type(client): assert all(i["type"] == "ip" for i in r.json()["iocs"]) +# ── mitre_techniques derivation ───────────────────────────────────────────── + + +def test_iocs_json_includes_mitre_techniques(client): + """mitre_techniques must be derived from the analysis, not the hardcoded [].""" + from app.api.deps import get_repo + + repo = get_repo() + case = Case(id="case0007", title="mitre") + repo.create_case(case) + repo.save_analysis( + Analysis( + case_id=case.id, + pcap_path="/tmp/m.pcap", + mitre_techniques=["T1071.001"], + iocs=[IOC(ioc_type=IOCType.IP, value="9.9.9.9", severity=Severity.HIGH)], + ) + ) + + r = client.get("/api/v1/iocs.json", headers={"Authorization": "Bearer FEED"}) + assert r.status_code == 200 + row = next(i for i in r.json()["iocs"] if i["value"] == "9.9.9.9") + assert row["mitre_techniques"] == ["T1071.001"] + + +def test_iocs_json_multi_technique_multi_tag_no_tag_duplication(client): + """Fan-out guard at the API layer: multiple techniques + multiple tags on the + same IOC's analysis must not duplicate tags or technique IDs in the response.""" + from app.api.deps import get_repo + + repo = get_repo() + case = Case(id="case0008", title="multi", tags=["tag-x", "tag-y"]) + repo.create_case(case) + repo.save_analysis( + Analysis( + case_id=case.id, + pcap_path="/tmp/m2.pcap", + mitre_techniques=["T1059", "T1071.001"], + iocs=[IOC(ioc_type=IOCType.IP, value="7.7.7.7", severity=Severity.HIGH)], + ) + ) + + r = client.get("/api/v1/iocs.json", headers={"Authorization": "Bearer FEED"}) + assert r.status_code == 200 + row = next(i for i in r.json()["iocs"] if i["value"] == "7.7.7.7") + assert sorted(row["mitre_techniques"]) == ["T1059", "T1071.001"] + assert sorted(row["tags"]) == ["tag-x", "tag-y"] + + # ── ETag / 304 ────────────────────────────────────────────────────────────── @@ -137,6 +186,208 @@ def test_iocs_stix_bundle(client): assert any(o.get("type") == "indicator" for o in objs) +def test_iocs_stix_md5_hash_type(client): + """An MD5-length hash IOC must emit file:hashes.MD5, not the hardcoded SHA-256.""" + from app.api.deps import get_repo + + md5_value = "c0" * 16 # 32 hex chars -> MD5 + repo = get_repo() + case = Case(id="case0005", title="md5") + repo.create_case(case) + repo.save_analysis( + Analysis( + case_id=case.id, + pcap_path="/tmp/md5.pcap", + iocs=[IOC(ioc_type=IOCType.HASH, value=md5_value)], + ) + ) + + r = client.get("/api/v1/iocs.stix", headers={"Authorization": "Bearer FEED"}) + assert r.status_code == 200 + body = r.json() + matching = [o["pattern"] for o in body["objects"] if o.get("type") == "indicator" and md5_value in o["pattern"]] + assert matching, "expected an indicator pattern for the MD5 IOC" + assert "file:hashes.MD5" in matching[0] + assert "SHA-256" not in matching[0] + + +def test_iocs_stix_ja3_not_dropped(client): + """JA3 rows must be emitted, not silently dropped.""" + from app.api.deps import get_repo + + ja3_value = "e" * 32 + repo = get_repo() + case = Case(id="case0006", title="ja3") + repo.create_case(case) + repo.save_analysis( + Analysis( + case_id=case.id, + pcap_path="/tmp/ja3.pcap", + iocs=[IOC(ioc_type=IOCType.JA3, value=ja3_value)], + ) + ) + + r = client.get("/api/v1/iocs.stix", headers={"Authorization": "Bearer FEED"}) + assert r.status_code == 200 + body = r.json() + matching = [o["pattern"] for o in body["objects"] if o.get("type") == "indicator" and ja3_value in o["pattern"]] + assert matching, "expected a JA3 indicator pattern, but it was dropped" + assert "x509-certificate" in matching[0] + + +# ── CEF feed ──────────────────────────────────────────────────────────────── + + +def test_iocs_cef_low_ioc_not_dropped(client): + """A LOW-severity IOC (score 25) must appear in CEF output. + + The generic priority-score gate in cef_export._events_from_iocs drops + anything below 0.4 — a naive `score/100` mapping would silently drop a + LOW (25 -> 0.25) IOC from a feed pull. The dedicated feed adapter must + not apply that gate at all. + """ + from app.api.deps import get_repo + + repo = get_repo() + case = Case(id="case0010", title="cef-low") + repo.create_case(case) + repo.save_analysis( + Analysis( + case_id=case.id, + pcap_path="/tmp/low.pcap", + iocs=[IOC(ioc_type=IOCType.IP, value="66.66.66.66", severity=Severity.LOW)], + ) + ) + + r = client.get("/api/v1/iocs.cef", headers={"Authorization": "Bearer FEED"}) + assert r.status_code == 200 + assert r.headers["content-type"].startswith("text/plain") + assert "CEF:0" in r.text + # The seeded HIGH IOC (fixture) and the new LOW IOC must BOTH appear. + assert "1.2.3.4" in r.text + assert "66.66.66.66" in r.text + + +def test_iocs_cef_respects_min_score(client): + from app.api.deps import get_repo + + repo = get_repo() + case = Case(id="case0011", title="cef-minscore") + repo.create_case(case) + repo.save_analysis( + Analysis( + case_id=case.id, + pcap_path="/tmp/lo2.pcap", + iocs=[IOC(ioc_type=IOCType.IP, value="77.77.77.77", severity=Severity.LOW)], + ) + ) + + r = client.get("/api/v1/iocs.cef?min_score=50", headers={"Authorization": "Bearer FEED"}) + assert r.status_code == 200 + assert "77.77.77.77" not in r.text # LOW (25) filtered out by min_score=50 + assert "1.2.3.4" in r.text # HIGH (75) survives the >=50 filter + + +def test_iocs_cef_requires_feed_scope(client): + r = client.get("/api/v1/iocs.cef") + assert r.status_code == 401 + + +def test_iocs_cef_escaping_safe(client): + """CEF reserved characters (=, \\) in a value must be escaped, not raw.""" + from app.api.deps import get_repo + + repo = get_repo() + case = Case(id="case0013", title="cef-escape") + repo.create_case(case) + repo.save_analysis( + Analysis( + case_id=case.id, + pcap_path="/tmp/esc.pcap", + iocs=[IOC(ioc_type=IOCType.URL, value="http://evil.example/a=b")], + ) + ) + + r = client.get("/api/v1/iocs.cef", headers={"Authorization": "Bearer FEED"}) + assert r.status_code == 200 + assert "a\\=b" in r.text + assert "a=b" not in r.text.replace("a\\=b", "") + + +def test_iocs_cef_etag_round_trip(client): + r1 = client.get("/api/v1/iocs.cef", headers={"Authorization": "Bearer FEED"}) + assert r1.status_code == 200 + etag = r1.headers["ETag"] + + r2 = client.get( + "/api/v1/iocs.cef", + headers={"Authorization": "Bearer FEED", "If-None-Match": etag}, + ) + assert r2.status_code == 304 + assert r2.content == b"" + + +# ── Single-IOC lookup ─────────────────────────────────────────────────────── + + +def test_iocs_lookup_exact_match(client): + r = client.get("/api/v1/iocs/lookup?value=1.2.3.4", headers={"Authorization": "Bearer FEED"}) + assert r.status_code == 200 + body = r.json() + assert body["count"] == 1 + assert body["next_cursor"] is None + assert len(body["iocs"]) == 1 + entry = body["iocs"][0] + assert entry["value"] == "1.2.3.4" + assert entry["type"] == "ip" + assert entry["score"] == 75 + assert entry["severity"] == "high" + + +def test_iocs_lookup_unknown_value_returns_empty(client): + r = client.get("/api/v1/iocs/lookup?value=nonexistent.example", headers={"Authorization": "Bearer FEED"}) + assert r.status_code == 200 + assert r.json() == {"iocs": [], "count": 0, "next_cursor": None} + + +def test_iocs_lookup_requires_feed_scope(client): + r = client.get("/api/v1/iocs/lookup?value=1.2.3.4") + assert r.status_code == 401 + + +def test_iocs_lookup_missing_value_param_422(client): + r = client.get("/api/v1/iocs/lookup", headers={"Authorization": "Bearer FEED"}) + assert r.status_code == 422 + + +def test_iocs_lookup_empty_value_422_not_whole_feed(client): + # An empty value must be rejected, never silently dump the entire feed. + r = client.get("/api/v1/iocs/lookup?value=", headers={"Authorization": "Bearer FEED"}) + assert r.status_code == 422 + + +def test_iocs_lookup_exact_not_substring(client): + """value=1.2.3.4 must not also return 1.2.3.40.""" + from app.api.deps import get_repo + + repo = get_repo() + case = Case(id="case0009", title="substring") + repo.create_case(case) + repo.save_analysis( + Analysis( + case_id=case.id, + pcap_path="/tmp/sub.pcap", + iocs=[IOC(ioc_type=IOCType.IP, value="1.2.3.40", severity=Severity.LOW)], + ) + ) + + r = client.get("/api/v1/iocs/lookup?value=1.2.3.4", headers={"Authorization": "Bearer FEED"}) + assert r.status_code == 200 + body = r.json() + assert body["count"] == 1 + assert body["iocs"][0]["value"] == "1.2.3.4" + + # ── Pagination ────────────────────────────────────────────────────────────── diff --git a/tests/api/test_key_auth.py b/tests/api/test_key_auth.py index 5a67f59..d64f22b 100644 --- a/tests/api/test_key_auth.py +++ b/tests/api/test_key_auth.py @@ -32,6 +32,8 @@ def _settings(main="MAIN", feed="FEED") -> APISettings: job_ttl_days=3, require_https=False, cors_origins=[], + webhook_timeout_seconds=10, + webhook_max_retries=2, ) diff --git a/tests/api/test_key_repository.py b/tests/api/test_key_repository.py index 39b029c..57456ad 100644 --- a/tests/api/test_key_repository.py +++ b/tests/api/test_key_repository.py @@ -100,11 +100,14 @@ def test_increment_usage_upsert(tmp_path): repo = _repo(tmp_path) _, key = _make_key() kid = repo.create_key(key) - repo.increment_usage(kid, "2026-05-23", 5) - repo.increment_usage(kid, "2026-05-23", 3) + # Use a date well within the 30-day retrieval window so this test doesn't + # rot as "today" drifts away from a hardcoded date (see get_usage's cutoff). + day = (datetime.now() - timedelta(days=2)).strftime("%Y-%m-%d") + repo.increment_usage(kid, day, 5) + repo.increment_usage(kid, day, 3) usage = repo.get_usage(kid, days=30) # Should have one entry with 8 total - day_data = [u for u in usage if u["date"] == "2026-05-23"] + day_data = [u for u in usage if u["date"] == day] assert len(day_data) == 1 assert day_data[0]["requests"] == 8 # total_requests on key should also be 8 @@ -116,13 +119,16 @@ def test_get_usage_returns_daily_data(tmp_path): repo = _repo(tmp_path) _, key = _make_key() kid = repo.create_key(key) - repo.increment_usage(kid, "2026-05-22", 10) - repo.increment_usage(kid, "2026-05-23", 20) + # Two distinct, consecutive days, both well within the 30-day window. + day_one = (datetime.now() - timedelta(days=3)).strftime("%Y-%m-%d") + day_two = (datetime.now() - timedelta(days=2)).strftime("%Y-%m-%d") + repo.increment_usage(kid, day_one, 10) + repo.increment_usage(kid, day_two, 20) usage = repo.get_usage(kid, days=30) assert len(usage) == 2 dates = [u["date"] for u in usage] - assert "2026-05-22" in dates - assert "2026-05-23" in dates + assert day_one in dates + assert day_two in dates def test_get_usage_summary(tmp_path): @@ -131,10 +137,11 @@ def test_get_usage_summary(tmp_path): _, k2 = _make_key(name="k2") id1 = repo.create_key(k1) id2 = repo.create_key(k2) - repo.increment_usage(id1, "2026-05-23", 10) - repo.increment_usage(id2, "2026-05-23", 5) + day = (datetime.now() - timedelta(days=2)).strftime("%Y-%m-%d") + repo.increment_usage(id1, day, 10) + repo.increment_usage(id2, day, 5) summary = repo.get_usage_summary(days=30) - day_data = [s for s in summary if s["date"] == "2026-05-23"] + day_data = [s for s in summary if s["date"] == day] assert len(day_data) == 1 assert day_data[0]["requests"] == 15 diff --git a/tests/api/test_pcaps.py b/tests/api/test_pcaps.py index 23c8d21..37a8acc 100644 --- a/tests/api/test_pcaps.py +++ b/tests/api/test_pcaps.py @@ -3,6 +3,7 @@ from __future__ import annotations +import json import pathlib import pytest @@ -77,6 +78,59 @@ def test_post_with_invalid_magic_returns_415(client): assert r.status_code == 415 +# --------------------------------------------------------------------------- +# Task 4.4: SSRF-safe completion webhook on submit +# --------------------------------------------------------------------------- + + +def test_post_with_private_webhook_url_returns_422(client): + """Fail fast (before upload I/O) on a webhook_url that fails the SSRF guard.""" + r = client.post( + "/api/v1/pcaps", + headers={"Authorization": "Bearer MAIN"}, + files={"pcap": ("a.pcap", b"\x00" * 100)}, + data={"webhook_url": "http://10.0.0.5/hook"}, + ) + assert r.status_code == 422 + assert r.json()["detail"] == "invalid_webhook_url" + + +def test_post_with_loopback_webhook_url_returns_422(client): + r = client.post( + "/api/v1/pcaps", + headers={"Authorization": "Bearer MAIN"}, + files={"pcap": ("a.pcap", b"\x00" * 100)}, + data={"webhook_url": "http://127.0.0.1:9000/hook"}, + ) + assert r.status_code == 422 + assert r.json()["detail"] == "invalid_webhook_url" + + +@pytest.mark.skipif(not FIXTURE.exists(), reason="fixture missing") +def test_post_with_public_webhook_url_returns_202_and_persists_option(client, monkeypatch): + """A public https webhook_url must be accepted and carried into the job's options_json.""" + monkeypatch.setattr("app.api.routers.pcaps.is_safe_webhook_url", lambda url: True) + + with FIXTURE.open("rb") as f: + r = client.post( + "/api/v1/pcaps", + headers={"Authorization": "Bearer MAIN"}, + files={"pcap": ("tiny.pcap", f, "application/vnd.tcpdump.pcap")}, + data={ + "osint_enabled": "false", + "llm_enabled": "false", + "webhook_url": "https://example.com/hook", + }, + ) + assert r.status_code == 202, r.text + job_id = r.json()["job_id"] + + from app.api.deps import get_repo + + job = get_repo().get_job(job_id) + assert json.loads(job.options_json)["webhook_url"] == "https://example.com/hook" + + @pytest.mark.skipif(not FIXTURE.exists(), reason="fixture missing") def test_post_returns_503_when_queue_full(client, monkeypatch): monkeypatch.setenv("PCAP_HUNTER_API_QUEUE_DEPTH", "1") diff --git a/tests/api/test_queue.py b/tests/api/test_queue.py index cc9985b..883899f 100644 --- a/tests/api/test_queue.py +++ b/tests/api/test_queue.py @@ -303,6 +303,97 @@ def fake_run_pipeline(pcap_path, case_id, options, progress, heartbeat=None): assert persisted.features["beacon_records"] == records +def test_worker_persists_attack_mapping(tmp_path, monkeypatch): + """attack_mapping must round-trip from the runner result into the persisted + Analysis, and mitre_techniques must flow through the job result blob.""" + import app.pipeline.runner as runner_mod + from app.pipeline.runner import PipelineResult + + mapping = { + "techniques": ["T1071.001"], + "tactics": {"command-and-control": ["T1071.001"]}, + } + techniques = ["T1071.001"] + + def fake_run_pipeline(pcap_path, case_id, options, progress, heartbeat=None): + return PipelineResult( + case_id=case_id, + packet_count=1, + features={ + "flows": [{"src": "10.0.0.1", "dst": "8.8.8.8", "proto": "TCP", "count": 9}], + "artifacts": {"ips": ["10.0.0.1", "8.8.8.8"], "domains": [], "urls": [], "hashes": [], "ja3": []}, + }, + attack_mapping=dict(mapping), + mitre_techniques=list(techniques), + ) + + # _worker_run imports run_pipeline from app.pipeline.runner at call time, + # so the patch must target the source module, not queue_mod. + monkeypatch.setattr(runner_mod, "run_pipeline", fake_run_pipeline) + + fake_pcap = tmp_path / "fake.pcap" + fake_pcap.write_bytes(b"\xd4\xc3\xb2\xa1" + b"\x00" * 20) + + db = str(tmp_path / "t.db") + repo = CaseRepository(db_path=db) + repo.create_case(Case(id="cafe0030", title="t", status=CaseStatus.IN_PROGRESS, severity=Severity.LOW)) + job_id = repo.create_job(Job(case_id="cafe0030", pcap_path=str(fake_pcap), options_json="{}")) + + _worker_run(job_id, db, str(fake_pcap), {"osint_enabled": False, "llm_enabled": False}) + + result = json.loads(repo.get_job(job_id).result_json) + assert result["mitre_techniques"] == techniques, "mitre_techniques must flow into the job result blob" + assert result["analysis_id"], "persistence must succeed with the faked pipeline result" + + persisted = repo.get_analysis(result["analysis_id"]) + assert persisted.attack_mapping == mapping, "attack_mapping must be persisted on the Analysis row" + + +def test_worker_http_analysis_round_trip(tmp_path, monkeypatch): + """http_analysis has no dedicated Analysis column — it must be stashed in + features['http_analysis'] (mirroring the beacon_records precedent) and + survive persistence.""" + import app.pipeline.runner as runner_mod + from app.pipeline.runner import PipelineResult + + http_analysis = { + "total_requests": 5, + "unique_hosts": 3, + "alerts": {"cleartext_cred_count": 1, "suspicious_ua_count": 0, "suspicious_uri_count": 0}, + "cleartext_credentials": [{"host": "10.0.0.5", "uri": "/admin", "username": "admin"}], + } + + def fake_run_pipeline(pcap_path, case_id, options, progress, heartbeat=None): + return PipelineResult( + case_id=case_id, + packet_count=1, + features={ + "flows": [{"src": "10.0.0.1", "dst": "8.8.8.8", "proto": "TCP", "count": 9}], + "artifacts": {"ips": ["10.0.0.1", "8.8.8.8"], "domains": [], "urls": [], "hashes": [], "ja3": []}, + }, + http_analysis=dict(http_analysis), + ) + + # _worker_run imports run_pipeline from app.pipeline.runner at call time, + # so the patch must target the source module, not queue_mod. + monkeypatch.setattr(runner_mod, "run_pipeline", fake_run_pipeline) + + fake_pcap = tmp_path / "fake.pcap" + fake_pcap.write_bytes(b"\xd4\xc3\xb2\xa1" + b"\x00" * 20) + + db = str(tmp_path / "t.db") + repo = CaseRepository(db_path=db) + repo.create_case(Case(id="cafe0031", title="t", status=CaseStatus.IN_PROGRESS, severity=Severity.LOW)) + job_id = repo.create_job(Job(case_id="cafe0031", pcap_path=str(fake_pcap), options_json="{}")) + + _worker_run(job_id, db, str(fake_pcap), {"osint_enabled": False, "llm_enabled": False}) + + result = json.loads(repo.get_job(job_id).result_json) + assert result["analysis_id"], "persistence must succeed with the faked pipeline result" + persisted = repo.get_analysis(result["analysis_id"]) + assert persisted.features["http_analysis"] == http_analysis + + # --------------------------------------------------------------------------- # Task 3: progress reconciliation on completion # --------------------------------------------------------------------------- diff --git a/tests/api/test_settings.py b/tests/api/test_settings.py index 681449c..145a7eb 100644 --- a/tests/api/test_settings.py +++ b/tests/api/test_settings.py @@ -50,6 +50,28 @@ def test_env_overrides(monkeypatch): assert s.require_https is True +def test_webhook_settings_defaults(monkeypatch): + monkeypatch.setenv("PCAP_HUNTER_API_KEY", "x") + monkeypatch.delenv("PCAP_HUNTER_API_WEBHOOK_TIMEOUT_SECONDS", raising=False) + monkeypatch.delenv("PCAP_HUNTER_API_WEBHOOK_MAX_RETRIES", raising=False) + monkeypatch.delenv("PCAP_HUNTER_API_WEBHOOK_SECRET", raising=False) + s = APISettings.from_env() + assert s.webhook_timeout_seconds == 10 + assert s.webhook_max_retries == 2 + assert s.webhook_secret is None + + +def test_webhook_settings_env_overrides(monkeypatch): + monkeypatch.setenv("PCAP_HUNTER_API_KEY", "x") + monkeypatch.setenv("PCAP_HUNTER_API_WEBHOOK_TIMEOUT_SECONDS", "5") + monkeypatch.setenv("PCAP_HUNTER_API_WEBHOOK_MAX_RETRIES", "4") + monkeypatch.setenv("PCAP_HUNTER_API_WEBHOOK_SECRET", "shh") + s = APISettings.from_env() + assert s.webhook_timeout_seconds == 5 + assert s.webhook_max_retries == 4 + assert s.webhook_secret == "shh" + + def test_create_app_refuses_start_without_any_auth(monkeypatch, tmp_path): """create_app() raises NoKeysConfiguredError when no auth sources exist.""" monkeypatch.delenv("PCAP_HUNTER_API_KEY", raising=False) diff --git a/tests/api/test_webhook.py b/tests/api/test_webhook.py new file mode 100644 index 0000000..6850696 --- /dev/null +++ b/tests/api/test_webhook.py @@ -0,0 +1,301 @@ +# tests/api/test_webhook.py +"""Tests for the completion-webhook dispatch (Task 4.4, app/api/queue.py). + +`_dispatch_webhook` is unit-tested directly (patching `hardened_session` at +its source module so the worker's lazy `from app.security.opsec import +hardened_session` picks up the fake). `_worker_run` integration tests confirm +it is invoked from both terminal branches, guarded by `options["webhook_url"]`. +""" + +from __future__ import annotations + +import json +import pathlib +from unittest.mock import MagicMock + +import pytest + +import app.api.queue as queue_mod +from app.api.queue import _dispatch_webhook +from app.database.models import Case, CaseStatus, Job, JobStatus, Severity +from app.database.repository import CaseRepository + +FIXTURE_PCAP = pathlib.Path(__file__).parent.parent / "fixtures" / "tiny.pcap" + + +@pytest.fixture(autouse=True) +def _no_sleep(monkeypatch): + """Keep the in-loop retry sleep from slowing down the suite.""" + monkeypatch.setattr(queue_mod.time, "sleep", lambda *_a, **_kw: None) + + +@pytest.fixture(autouse=True) +def _always_safe_url(monkeypatch): + """Most tests here exercise dispatch/retry logic, not the SSRF guard itself.""" + monkeypatch.setattr("app.utils.network_utils.is_safe_webhook_url", lambda url: True) + + +# --------------------------------------------------------------------------- +# _dispatch_webhook unit tests +# --------------------------------------------------------------------------- + + +def test_dispatch_webhook_posts_expected_envelope(monkeypatch): + captured = {} + + def fake_hardened_session(timeout): + captured["timeout"] = timeout + session = MagicMock() + + def post(url, data=None, headers=None, **kwargs): + captured["url"] = url + captured["data"] = data + captured["headers"] = headers + return MagicMock(status_code=200) + + session.post = post + return session + + monkeypatch.setattr("app.security.opsec.hardened_session", fake_hardened_session) + + payload = {"job_id": "j_abc12345", "case_id": "cafe0001", "status": "done", "analysis_id": "an_1"} + _dispatch_webhook("https://example.com/hook", payload, timeout=7, max_retries=2) + + assert captured["url"] == "https://example.com/hook" + assert captured["timeout"] == 7 + assert json.loads(captured["data"]) == payload + assert captured["headers"]["Content-Type"] == "application/json" + + +def test_dispatch_webhook_does_not_follow_redirects(monkeypatch): + """A 302 to an internal host must NOT be followed -- the SSRF guard only + validated the original URL, so redirects would reopen the SSRF hole. + The POST must pass allow_redirects=False.""" + captured = {} + calls = {"n": 0} + + def fake_hardened_session(timeout): + session = MagicMock() + + def post(url, data=None, headers=None, allow_redirects=None, **kwargs): + calls["n"] += 1 + captured["allow_redirects"] = allow_redirects + captured["url"] = url + # A malicious receiver tries to bounce us to the cloud metadata endpoint. + return MagicMock(status_code=302, headers={"Location": "http://169.254.169.254/latest/meta-data/"}) + + session.post = post + return session + + monkeypatch.setattr("app.security.opsec.hardened_session", fake_hardened_session) + + _dispatch_webhook("https://example.com/hook", {"a": 1}, timeout=5, max_retries=0) + + assert captured["allow_redirects"] is False, "webhook POST must disable redirect-following" + # Only the original (validated) URL was ever requested; the 302 Location was not chased. + assert calls["n"] == 1 + assert captured["url"] == "https://example.com/hook" + + +def test_dispatch_webhook_retries_up_to_max_retries_on_failure(monkeypatch): + calls = {"n": 0} + + def fake_hardened_session(timeout): + session = MagicMock() + + def post(url, data=None, headers=None, **kwargs): + calls["n"] += 1 + return MagicMock(status_code=500) + + session.post = post + return session + + monkeypatch.setattr("app.security.opsec.hardened_session", fake_hardened_session) + + _dispatch_webhook("https://example.com/hook", {"a": 1}, timeout=5, max_retries=2) + + assert calls["n"] == 3 # 1 initial attempt + 2 retries + + +def test_dispatch_webhook_stops_retrying_once_a_2xx_is_seen(monkeypatch): + calls = {"n": 0} + + def fake_hardened_session(timeout): + session = MagicMock() + + def post(url, data=None, headers=None, **kwargs): + calls["n"] += 1 + return MagicMock(status_code=200) + + session.post = post + return session + + monkeypatch.setattr("app.security.opsec.hardened_session", fake_hardened_session) + + _dispatch_webhook("https://example.com/hook", {"a": 1}, timeout=5, max_retries=3) + + assert calls["n"] == 1 + + +def test_dispatch_webhook_never_raises_when_session_construction_throws(monkeypatch): + def fake_hardened_session(timeout): + raise RuntimeError("network is down") + + monkeypatch.setattr("app.security.opsec.hardened_session", fake_hardened_session) + + # Must not raise -- a webhook failure must never affect the job's own status. + _dispatch_webhook("https://example.com/hook", {"a": 1}, timeout=5, max_retries=1) + + +def test_dispatch_webhook_never_raises_when_post_throws_every_attempt(monkeypatch): + def fake_hardened_session(timeout): + session = MagicMock() + session.post.side_effect = ConnectionError("boom") + return session + + monkeypatch.setattr("app.security.opsec.hardened_session", fake_hardened_session) + + _dispatch_webhook("https://example.com/hook", {"a": 1}, timeout=5, max_retries=2) + + +def test_dispatch_webhook_never_raises_when_lazy_import_fails(monkeypatch): + """The lazy `from app.security.opsec import ...` lives INSIDE the try, so an + import failure is swallowed too. If it escaped, it would propagate into + _worker_run's `except` and flip an already-DONE job to FAILED.""" + import sys + + # Poisoning sys.modules makes `from app.security.opsec import ...` raise + # ModuleNotFoundError -- the very first statement inside _dispatch_webhook's try. + monkeypatch.setitem(sys.modules, "app.security.opsec", None) + + # Must return, not raise, even though the import blew up. + _dispatch_webhook("https://example.com/hook", {"a": 1}, timeout=5, max_retries=1) + + +def test_dispatch_webhook_refuses_unsafe_url_without_posting(monkeypatch): + monkeypatch.setattr("app.utils.network_utils.is_safe_webhook_url", lambda url: False) + + session = MagicMock() + monkeypatch.setattr("app.security.opsec.hardened_session", lambda timeout: session) + + _dispatch_webhook("http://10.0.0.5/hook", {"a": 1}, timeout=5, max_retries=2) + + session.post.assert_not_called() + + +# --------------------------------------------------------------------------- +# _worker_run integration: webhook fires on both terminal branches +# --------------------------------------------------------------------------- + + +def _fake_pipeline_result(case_id): + from app.pipeline.runner import PipelineResult + + return PipelineResult( + case_id=case_id, + packet_count=1, + features={"flows": [], "artifacts": {"ips": [], "domains": [], "urls": [], "hashes": [], "ja3": []}}, + ) + + +def test_worker_fires_webhook_on_success(tmp_path, monkeypatch): + import app.pipeline.runner as runner_mod + + captured = {} + + def fake_dispatch(url, payload, timeout, max_retries): + captured.update(url=url, payload=payload, timeout=timeout, max_retries=max_retries) + + monkeypatch.setattr(queue_mod, "_dispatch_webhook", fake_dispatch) + monkeypatch.setenv("PCAP_HUNTER_API_WEBHOOK_TIMEOUT_SECONDS", "3") + monkeypatch.setenv("PCAP_HUNTER_API_WEBHOOK_MAX_RETRIES", "1") + monkeypatch.setattr( + runner_mod, + "run_pipeline", + lambda pcap_path, case_id, options, progress, heartbeat=None: _fake_pipeline_result(case_id), + ) + + fake_pcap = tmp_path / "fake.pcap" + fake_pcap.write_bytes(b"\xd4\xc3\xb2\xa1" + b"\x00" * 20) + + db = str(tmp_path / "t.db") + repo = CaseRepository(db_path=db) + repo.create_case(Case(id="cafe0099", title="t", status=CaseStatus.IN_PROGRESS, severity=Severity.LOW)) + job_id = repo.create_job(Job(case_id="cafe0099", pcap_path=str(fake_pcap), options_json="{}")) + + queue_mod._worker_run( + job_id, + db, + str(fake_pcap), + {"osint_enabled": False, "llm_enabled": False, "webhook_url": "https://example.com/hook"}, + ) + + job = repo.get_job(job_id) + assert job.status == JobStatus.DONE + assert captured["url"] == "https://example.com/hook" + assert captured["payload"]["job_id"] == job_id + assert captured["payload"]["case_id"] == "cafe0099" + assert captured["payload"]["status"] == "done" + assert captured["timeout"] == 3 + assert captured["max_retries"] == 1 + + +def test_worker_fires_webhook_on_failure(tmp_path, monkeypatch): + import app.pipeline.runner as runner_mod + + captured = {} + + def fake_dispatch(url, payload, timeout, max_retries): + captured["payload"] = payload + + monkeypatch.setattr(queue_mod, "_dispatch_webhook", fake_dispatch) + + def boom(*a, **kw): + raise RuntimeError("pipeline exploded") + + monkeypatch.setattr(runner_mod, "run_pipeline", boom) + + fake_pcap = tmp_path / "fake.pcap" + fake_pcap.write_bytes(b"\xd4\xc3\xb2\xa1" + b"\x00" * 20) + + db = str(tmp_path / "t.db") + repo = CaseRepository(db_path=db) + repo.create_case(Case(id="cafe0098", title="t", status=CaseStatus.IN_PROGRESS, severity=Severity.LOW)) + job_id = repo.create_job(Job(case_id="cafe0098", pcap_path=str(fake_pcap), options_json="{}")) + + queue_mod._worker_run( + job_id, + db, + str(fake_pcap), + {"osint_enabled": False, "llm_enabled": False, "webhook_url": "https://example.com/hook"}, + ) + + job = repo.get_job(job_id) + assert job.status == JobStatus.FAILED + assert captured["payload"]["status"] == "failed" + assert captured["payload"]["analysis_id"] is None + assert captured["payload"]["case_id"] == "cafe0098" + + +def test_worker_does_not_fire_webhook_when_not_configured(tmp_path, monkeypatch): + import app.pipeline.runner as runner_mod + + called = {"n": 0} + monkeypatch.setattr(queue_mod, "_dispatch_webhook", lambda *a, **kw: called.__setitem__("n", called["n"] + 1)) + monkeypatch.setattr( + runner_mod, + "run_pipeline", + lambda pcap_path, case_id, options, progress, heartbeat=None: _fake_pipeline_result(case_id), + ) + + fake_pcap = tmp_path / "fake.pcap" + fake_pcap.write_bytes(b"\xd4\xc3\xb2\xa1" + b"\x00" * 20) + + db = str(tmp_path / "t.db") + repo = CaseRepository(db_path=db) + repo.create_case(Case(id="cafe0097", title="t", status=CaseStatus.IN_PROGRESS, severity=Severity.LOW)) + job_id = repo.create_job(Job(case_id="cafe0097", pcap_path=str(fake_pcap), options_json="{}")) + + queue_mod._worker_run(job_id, db, str(fake_pcap), {"osint_enabled": False, "llm_enabled": False}) + + assert called["n"] == 0 diff --git a/tests/test_attack_mapping.py b/tests/test_attack_mapping.py new file mode 100644 index 0000000..3cd4024 --- /dev/null +++ b/tests/test_attack_mapping.py @@ -0,0 +1,105 @@ +"""Tests for the ATT&CK mapping engine (app/threat_intel/attack_mapping.py). + +The mapper was built but never instantiated at runtime until it was wired +into app/pipeline/runner.py — these tests use production-shape output from +the real analysis stages (app/pipeline/tls_certs.py, app/pipeline/dns_analysis.py, +app/pipeline/beacon.py) rather than "looks reasonable" dicts, since a shape +mismatch here previously escaped notice precisely because the mapper was dead +code (see the `_check_tls` fix in this same change). +""" + +from __future__ import annotations + +from app.threat_intel import ATTACKMapper + + +class TestCheckTLS: + """Regression coverage for the tls_analysis shape consumed by _check_tls.""" + + def test_self_signed_certificate_is_detected(self): + """analyze_certificates() reports per-cert flags in `certificates`, not a + list of {"type": ..., "cert": ...} alert objects — must not raise. + """ + tls_analysis = { + "total_certificates": 1, + "certificates": [ + { + "subject_cn": "evil.example", + "is_self_signed": True, + "is_expired": False, + } + ], + "alerts": {"self_signed_count": 1, "expired_count": 0, "high_risk_count": 0}, + } + mapping = ATTACKMapper().map_analysis(tls_analysis=tls_analysis) + ids = {t.technique_id for t in mapping.techniques} + assert "T1587.003" in ids + assert "T1573.002" in ids + evidence = [e for t in mapping.techniques for e in t.evidence] + assert any("evil.example" in e for e in evidence) + + def test_expired_certificate_is_detected(self): + tls_analysis = { + "total_certificates": 1, + "certificates": [ + { + "subject_cn": "stale.example", + "is_self_signed": False, + "is_expired": True, + } + ], + "alerts": {"self_signed_count": 0, "expired_count": 1, "high_risk_count": 0}, + } + mapping = ATTACKMapper().map_analysis(tls_analysis=tls_analysis) + ids = {t.technique_id for t in mapping.techniques} + assert "T1573.002" in ids + + def test_clean_certificates_produce_no_techniques(self): + tls_analysis = { + "total_certificates": 1, + "certificates": [ + { + "subject_cn": "clean.example", + "is_self_signed": False, + "is_expired": False, + } + ], + "alerts": {"self_signed_count": 0, "expired_count": 0, "high_risk_count": 0}, + } + mapping = ATTACKMapper().map_analysis(tls_analysis=tls_analysis) + assert mapping.techniques == [] + + def test_missing_certificates_key_does_not_raise(self): + """Older/partial tls_analysis dicts without a certificates key are tolerated.""" + mapping = ATTACKMapper().map_analysis(tls_analysis={"alerts": {}}) + assert mapping.techniques == [] + + +class TestMapAnalysisEndToEnd: + """map_analysis() called the way app/pipeline/runner.py calls it (no yara/osint).""" + + def test_combined_production_shape_inputs_produce_a_mapping(self): + features = { + "flows": [{"dst": "1.2.3.4", "count": 5, "bytes": 20_000_000}], + "artifacts": {"ips": [], "domains": [], "urls": [], "hashes": [], "ja3": []}, + } + dns_analysis = {"alerts": {"dga_count": 2, "tunneling_count": 0, "fast_flux_count": 0}} + tls_analysis = { + "certificates": [{"subject_cn": "c2.example", "is_self_signed": True, "is_expired": False}], + "alerts": {"self_signed_count": 1, "expired_count": 0, "high_risk_count": 0}, + } + beacon_results = [{"dst": "1.2.3.4", "score": 0.85}] + + mapping = ATTACKMapper().map_analysis( + features=features, + dns_analysis=dns_analysis, + tls_analysis=tls_analysis, + beacon_results=beacon_results, + ) + ids = {t.technique_id for t in mapping.techniques} + assert "T1071.001" in ids # beaconing + assert "T1587.003" in ids # self-signed cert + assert any(tid.startswith("T1568") for tid in ids) # DGA + assert mapping.overall_severity in {"low", "medium", "high", "critical"} + d = mapping.to_dict() + assert "techniques" in d and "tactics_summary" in d diff --git a/tests/test_batch.py b/tests/test_batch.py index 8d3fc3b..0267108 100644 --- a/tests/test_batch.py +++ b/tests/test_batch.py @@ -6,6 +6,7 @@ BatchProcessor, PCAPResult, aggregate_dns_analysis, + aggregate_http_analysis, aggregate_tls_analysis, correlate_results, merge_beacon_candidates, @@ -162,6 +163,31 @@ def test_carved_items_accepts_records(self): result = PCAPResult(path="/data/test.pcap", filename="test.pcap", carved_items=carved) assert result.carved_items == carved + def test_attack_mapping_defaults_empty(self): + result = PCAPResult(path="/data/test.pcap", filename="test.pcap") + assert result.attack_mapping == {} + + def test_attack_mapping_accepts_mapping(self): + mapping = { + "techniques": ["T1071.001"], + "tactics": {"command-and-control": ["T1071.001"]}, + } + result = PCAPResult(path="/data/test.pcap", filename="test.pcap", attack_mapping=mapping) + assert result.attack_mapping == mapping + + def test_http_analysis_defaults_empty(self): + result = PCAPResult(path="/data/test.pcap", filename="test.pcap") + assert result.http_analysis == {} + + def test_http_analysis_accepts_dict(self): + http_analysis = { + "total_requests": 5, + "unique_hosts": 3, + "alerts": {"cleartext_cred_count": 1, "suspicious_ua_count": 0, "suspicious_uri_count": 0}, + } + result = PCAPResult(path="/data/test.pcap", filename="test.pcap", http_analysis=http_analysis) + assert result.http_analysis == http_analysis + class TestMergeZeekTables: """Test merging Zeek tables from multiple PCAPs.""" @@ -315,6 +341,80 @@ def test_deduplicate_by_fingerprint(self): assert aggregated["total_certificates"] == 2 # Deduplicated +class TestAggregateHTTP: + """Test aggregating HTTP request analysis results (FIX 1: batch mode must + not under-report intel from files 2..N).""" + + def test_empty_results(self): + aggregated = aggregate_http_analysis([]) + assert aggregated["total_requests"] == 0 + assert aggregated["cleartext_credentials"] == [] + + def test_skips_errored_and_skipped_files(self): + results = [ + PCAPResult(path="/data/1.pcap", filename="1.pcap", features={}, error="boom"), + PCAPResult(path="/data/2.pcap", filename="2.pcap", features={}, http_analysis={"skipped": True}), + PCAPResult( + path="/data/3.pcap", + filename="3.pcap", + features={}, + http_analysis={"error": "No HTTP log data", "records": 0}, + ), + ] + aggregated = aggregate_http_analysis(results) + assert aggregated["total_requests"] == 0 + + def test_aggregate_merges_findings_from_every_file(self): + # Regression for the bug where st.session_state["http_analysis"] only + # reflected the FIRST successful file in batch mode — this asserts the + # aggregate carries findings from file 2 as well. + results = [ + PCAPResult( + path="/data/1.pcap", + filename="1.pcap", + features={}, + http_analysis={ + "total_requests": 10, + "unique_hosts": 3, + "methods": {"GET": 8, "POST": 2}, + "status_codes": {"200": 10}, + "suspicious_user_agents": [], + "cleartext_credentials": [{"host": "1.2.3.4", "uri": "/login", "username": "admin"}], + "suspicious_uris": [], + "alerts": {"suspicious_ua_count": 0, "cleartext_cred_count": 1, "suspicious_uri_count": 0}, + }, + ), + PCAPResult( + path="/data/2.pcap", + filename="2.pcap", + features={}, + http_analysis={ + "total_requests": 5, + "unique_hosts": 2, + "methods": {"GET": 5}, + "status_codes": {"200": 4, "404": 1}, + "suspicious_user_agents": [ + {"host": "evil.example", "user_agent": "sqlmap/1.6", "uri": "/", "reason": "known tool"} + ], + "cleartext_credentials": [], + "suspicious_uris": [], + "alerts": {"suspicious_ua_count": 1, "cleartext_cred_count": 0, "suspicious_uri_count": 0}, + }, + ), + ] + aggregated = aggregate_http_analysis(results) + assert aggregated["total_requests"] == 15 + assert aggregated["methods"]["GET"] == 13 + assert aggregated["status_codes"]["200"] == 14 + # File 1's cred finding AND file 2's UA finding must both be present. + assert len(aggregated["cleartext_credentials"]) == 1 + assert aggregated["cleartext_credentials"][0]["host"] == "1.2.3.4" + assert len(aggregated["suspicious_user_agents"]) == 1 + assert aggregated["suspicious_user_agents"][0]["host"] == "evil.example" + assert aggregated["alerts"]["cleartext_cred_count"] == 1 + assert aggregated["alerts"]["suspicious_ua_count"] == 1 + + class TestBatchProcessor: """Test BatchProcessor class.""" diff --git a/tests/test_beacon.py b/tests/test_beacon.py index 74f931b..11dbacd 100644 --- a/tests/test_beacon.py +++ b/tests/test_beacon.py @@ -1,4 +1,6 @@ +from app.config import BEACON_SCORE_THRESHOLD from app.pipeline.beacon import jitter_score, periodicity_score, rank_beaconing +from app.threat_intel.attack_mapping import DETECTION_RULES, ATTACKMapper def test_periodicity_score_empty(): @@ -100,3 +102,132 @@ def test_jitter_score_random(): # Random traffic should score lower than periodic periodic = jitter_score([float(i) for i in range(1, 21)]) assert res["jitter_score"] < periodic["jitter_score"] + + +def test_rank_beaconing_regular_443_clears_threshold(): + # A genuinely periodic, SMALL-PAYLOAD HTTPS beacon (raw score ~1.0) must + # survive the port-443 benign-service penalty and still clear + # BEACON_SCORE_THRESHOLD. Before the conditional softening, the blanket + # 0.15 multiplier crushed this down to ~0.15 — real HTTPS C2 was invisible + # to the pipeline. Small payloads (80 bytes) are what real C2 beacons look + # like, and are required for softening. + flows = [ + { + "src": "10.0.0.3", + "dst": "203.0.113.5", + "sport": "51000", + "dport": "443", + "proto": "tcp", + "pkt_times": [float(i) for i in range(1, 60)], # perfectly periodic + "pkt_lens": [80] * 59, # small, C2-like payloads + } + ] + df = rank_beaconing(flows, top_n=10) + assert len(df) == 1 + assert df.iloc[0]["score"] > BEACON_SCORE_THRESHOLD + + +def test_rank_beaconing_regular_443_large_payload_not_softened(): + # A machine-regular flow on 443 with LARGE payloads (like a CDN heartbeat) + # is indistinguishable from C2 by timing+jitter alone — it must NOT be + # softened. Only small-payload flows qualify; this keeps the full 0.15 + # penalty and stays well below threshold. Locks in the small-payload + # distinction that guards tests/test_integration.py::test_https_cdn_not_flagged. + flows = [ + { + "src": "10.0.0.5", + "dst": "13.224.0.9", # CDN-like IP + "sport": "51002", + "dport": "443", + "proto": "tcp", + "pkt_times": [float(i) for i in range(1, 60)], # perfectly periodic + "pkt_lens": [1200] * 59, # large payloads → NOT C2-like + } + ] + df = rank_beaconing(flows, top_n=10) + assert len(df) == 1 + # Full 0.15 penalty applied (raw ~1.0 * 0.15 = 0.15), no softening. + assert df.iloc[0]["score"] <= 0.15 + assert df.iloc[0]["score"] < BEACON_SCORE_THRESHOLD + + +def test_rank_beaconing_regular_443_no_pkt_lens_not_softened(): + # Conservative default: without pkt_lens we cannot confirm the flow is + # C2-like (small payloads), so we do NOT soften. Keeps the full penalty. + flows = [ + { + "src": "10.0.0.6", + "dst": "203.0.113.7", + "sport": "51003", + "dport": "443", + "proto": "tcp", + "pkt_times": [float(i) for i in range(1, 60)], # perfectly periodic + # no pkt_lens + } + ] + df = rank_beaconing(flows, top_n=10) + assert len(df) == 1 + assert df.iloc[0]["score"] <= 0.15 + assert df.iloc[0]["score"] < BEACON_SCORE_THRESHOLD + + +def test_rank_beaconing_softened_443_score_decoupled_from_attack_mapper_threshold(): + # Locks in the FIX 3 invariant: a softened-443 beacon must clear the + # BEACON_SCORE_THRESHOLD candidate floor (so genuine HTTPS beacons still + # surface) but must NOT, on timing alone, reach the ATT&CK mapper's + # beacon->C2 rule threshold (score >= 0.7, which auto-sets HIGH + # severity with a T1071.001 false positive for a benign perfectly + # periodic small-payload flow). See app/pipeline/beacon.py's + # SOFTENED_PENALTY comment for the full rationale. + flows = [ + { + "src": "10.0.0.3", + "dst": "203.0.113.5", + "sport": "51000", + "dport": "443", + "proto": "tcp", + "pkt_times": [float(i) for i in range(1, 60)], # perfectly periodic + "pkt_lens": [80] * 59, # small, C2-like payloads + } + ] + df = rank_beaconing(flows, top_n=10) + assert len(df) == 1 + score = df.iloc[0]["score"] + + # Still a beacon candidate... + assert score > BEACON_SCORE_THRESHOLD + # ...but decoupled from the ATT&CK mapper's C2 threshold. + beacon_rule_threshold = DETECTION_RULES["beacon_score"]["threshold"] + assert score < beacon_rule_threshold + + # End-to-end: feeding this exact beacon candidate into the mapper must not + # produce the T1071.001 C2 technique (and therefore not force HIGH severity + # from timing alone). + mapping = ATTACKMapper().map_analysis(beacon_results=df.to_dict("records")) + ids = {t.technique_id for t in mapping.techniques} + assert "T1071.001" not in ids + + +def test_rank_beaconing_jittery_443_stays_suppressed(): + # An ordinary HTTPS keep-alive with real-world jitter must NOT be + # promoted by the softening — only extremely regular, high-confidence + # signals qualify. This flow's raw score drops well below the 0.85 + # softening gate once jitter is introduced, so it keeps the full 0.15 + # penalty and stays suppressed below threshold. + import random + + random.seed(42) + ts = sorted(10.0 * i + random.uniform(-2, 2) for i in range(30)) + flows = [ + { + "src": "10.0.0.4", + "dst": "203.0.113.6", + "sport": "51001", + "dport": "443", + "proto": "tcp", + "pkt_times": ts, + } + ] + df = rank_beaconing(flows, top_n=10) + assert len(df) == 1 + assert df.iloc[0]["score"] < BEACON_SCORE_THRESHOLD diff --git a/tests/test_case_management.py b/tests/test_case_management.py index 9c9c117..da768e3 100644 --- a/tests/test_case_management.py +++ b/tests/test_case_management.py @@ -148,6 +148,7 @@ def test_default_values(self): assert analysis.packet_count == 0 assert analysis.features == {} assert analysis.iocs == [] + assert analysis.mitre_techniques == [] def test_to_dict(self): analysis = Analysis( @@ -373,6 +374,19 @@ def test_clear_all_reaps_jobs(self, repo): finally: conn.close() + def test_clear_all_removes_analysis_techniques(self, repo): + """clear_all must delete analysis_techniques rows along with the case data.""" + case_id = repo.create_case(Case(title="With Techniques")) + repo.save_analysis(Analysis(case_id=case_id, pcap_path="/p.pcap", mitre_techniques=["T1071.001"])) + + assert repo.clear_all() is True + + conn = repo._get_conn() + try: + assert conn.execute("SELECT COUNT(*) FROM analysis_techniques").fetchone()[0] == 0 + finally: + conn.close() + def test_save_analysis_to_case(self, repo): """Test saving analysis to case.""" case_id = repo.create_case(Case(title="With Analysis")) @@ -404,6 +418,91 @@ def test_get_analysis(self, repo): assert retrieved is not None assert retrieved.packet_count == 500 + def test_save_analysis_round_trips_attack_mapping(self, repo): + """attack_mapping (ATT&CK technique matches) must survive save -> get.""" + case_id = repo.create_case(Case(title="With ATT&CK Mapping")) + attack_mapping = { + "techniques": [ + { + "technique_id": "T1071.001", + "technique_name": "Application Layer Protocol: Web Protocols", + "tactic": "command-and-control", + "confidence": 0.8, + "evidence": ["beacon score 0.91"], + } + ], + "tactics_summary": {"command-and-control": 1}, + "kill_chain_phase": "command-and-control", + "overall_severity": "high", + } + analysis = Analysis( + case_id=case_id, + pcap_path="/test.pcap", + attack_mapping=attack_mapping, + ) + analysis_id = repo.save_analysis(analysis) + + retrieved = repo.get_analysis(analysis_id) + assert retrieved is not None + assert retrieved.attack_mapping == attack_mapping + + # Also verify it round-trips via the case-level fetch path. + restored_case = repo.get_case(case_id) + assert restored_case.analyses[0].attack_mapping == attack_mapping + + def test_get_analysis_defaults_attack_mapping_to_empty_dict(self, repo): + """Old rows / analyses saved without a mapping must not crash on read.""" + case_id = repo.create_case(Case(title="No ATT&CK Mapping")) + analysis = Analysis(case_id=case_id, pcap_path="/test.pcap") + analysis_id = repo.save_analysis(analysis) + + retrieved = repo.get_analysis(analysis_id) + assert retrieved is not None + assert retrieved.attack_mapping == {} + + def test_save_analysis_round_trips_mitre_techniques(self, repo): + """mitre_techniques (technique IDs) must survive save -> get, dedup by table UNIQUE.""" + case_id = repo.create_case(Case(title="With MITRE Techniques")) + analysis = Analysis( + case_id=case_id, + pcap_path="/test.pcap", + mitre_techniques=["T1071.001", "T1059"], + ) + analysis_id = repo.save_analysis(analysis) + + retrieved = repo.get_analysis(analysis_id) + assert retrieved is not None + assert sorted(retrieved.mitre_techniques) == ["T1059", "T1071.001"] + + # Also verify it round-trips via the case-level fetch path. + restored_case = repo.get_case(case_id) + assert sorted(restored_case.analyses[0].mitre_techniques) == ["T1059", "T1071.001"] + + def test_get_analysis_defaults_mitre_techniques_to_empty_list(self, repo): + """Analyses saved without techniques must not crash on read.""" + case_id = repo.create_case(Case(title="No MITRE Techniques")) + analysis = Analysis(case_id=case_id, pcap_path="/test.pcap") + analysis_id = repo.save_analysis(analysis) + + retrieved = repo.get_analysis(analysis_id) + assert retrieved is not None + assert retrieved.mitre_techniques == [] + + def test_save_analysis_resaved_mitre_techniques_replaces_not_accumulates(self, repo): + """Re-saving the same analysis_id with a different technique list must fully + replace the prior set, not accumulate across saves (mirrors full-replace + semantics of attack_json/features_json, unlike the accumulating iocs table).""" + case_id = repo.create_case(Case(title="Resaved Techniques")) + analysis = Analysis(case_id=case_id, pcap_path="/test.pcap", mitre_techniques=["T1071.001"]) + analysis_id = repo.save_analysis(analysis) + + analysis.id = analysis_id + analysis.mitre_techniques = ["T1059"] + repo.save_analysis(analysis) + + retrieved = repo.get_analysis(analysis_id) + assert retrieved.mitre_techniques == ["T1059"] + def test_save_analysis_with_iocs(self, repo): """Test saving analysis with IOCs.""" case_id = repo.create_case(Case(title="With IOCs")) @@ -478,6 +577,18 @@ def test_get_statistics(self, repo): assert stats["by_status"].get("open", 0) == 1 assert stats["by_status"].get("closed", 0) == 1 + def test_repo_uses_wal_and_busy_timeout(self, tmp_path): + """Concurrent writers (pool workers, API thread, Streamlit UI) must not + hit 'database is locked' immediately -- WAL + a busy_timeout give SQLite + room to retry instead of failing fast.""" + repo = CaseRepository(db_path=str(tmp_path / "t.db")) + conn = repo._get_conn() + try: + assert conn.execute("PRAGMA journal_mode").fetchone()[0].lower() == "wal" + assert conn.execute("PRAGMA busy_timeout").fetchone()[0] == 30000 + finally: + conn.close() + class TestCascadeDeletion: """Deleting a case must remove every related row — the FK pragma is off, so @@ -488,7 +599,13 @@ def test_delete_case_cascades_all_related_rows(self, tmp_path): repo.create_case( Case(id="del00001", title="t", status=CaseStatus.IN_PROGRESS, severity=Severity.LOW, tags=["x"]) ) - analysis = Analysis(case_id="del00001", pcap_path="x.pcap", packet_count=1, features={"artifacts": {}}) + analysis = Analysis( + case_id="del00001", + pcap_path="x.pcap", + packet_count=1, + features={"artifacts": {}}, + mitre_techniques=["T1071.001"], + ) analysis.iocs = [IOC(ioc_type=IOCType.IP, value="203.0.113.9", context="t", severity=Severity.HIGH)] aid = repo.save_analysis(analysis) repo.add_note("del00001", "investigation note", analysis_id=aid) @@ -501,6 +618,7 @@ def test_delete_case_cascades_all_related_rows(self, tmp_path): for table, where, arg in [ ("analyses", "id = ?", aid), ("iocs", "analysis_id = ?", aid), + ("analysis_techniques", "analysis_id = ?", aid), ("jobs", "id = ?", job_id), ("case_tags", "case_id = ?", "del00001"), ("notes", "case_id = ?", "del00001"), diff --git a/tests/test_case_restore.py b/tests/test_case_restore.py index 001d0fb..cac84f4 100644 --- a/tests/test_case_restore.py +++ b/tests/test_case_restore.py @@ -152,6 +152,21 @@ def test_none_dns_and_tls_passed_through(self): assert st.session_state["dns_analysis"] is None assert st.session_state["tls_analysis"] is None + def test_http_analysis_restored_from_features_stash(self): + """http_analysis has no dedicated Analysis column — it's stashed inside + features (mirroring how beacon_records is stashed, see app/api/queue.py) + and must be pulled back out on restore.""" + analysis = _make_analysis(features={"flows": [], "http_analysis": {"total_requests": 3}}) + _restore_analysis_to_session(analysis) + assert st.session_state["http_analysis"] == {"total_requests": 3} + + def test_http_analysis_none_when_absent_from_features(self): + """A case saved before http_analysis existed (its features dict has no + such key) must not surface stale/wrong data.""" + analysis = _make_analysis() # default features has no http_analysis key + _restore_analysis_to_session(analysis) + assert st.session_state["http_analysis"] is None + class TestDetailViewAutoRestoreGuard: """The case-detail auto-restore must be a one-shot keyed on restored_analysis_id. diff --git a/tests/test_cases_tab.py b/tests/test_cases_tab.py new file mode 100644 index 0000000..5f182dd --- /dev/null +++ b/tests/test_cases_tab.py @@ -0,0 +1,158 @@ +"""Tests for app/ui/cases_tab.py. + +Focused on two session-state-derived helpers used by the UI manual-save paths +(``_quick_save_analysis`` and ``_add_current_analysis_to_case``), both of +which build an ``Analysis(...)`` from ``st.session_state`` rather than a +``PipelineResult`` (the API path's source, see ``app/api/queue.py``): + +- ``_session_mitre_techniques`` derives ``Analysis.mitre_techniques`` from + ``st.session_state["attack_mapping"]``. Without this, analyses saved + through the UI ship a permanently-empty ``mitre_techniques`` field even + though the API path populates it from ``result.mitre_techniques`` -- both + share the same ``data/cases.db``, so the IOC feed's ``mitre_techniques`` + column would be inconsistent depending on which path saved the case. + +- ``_session_analysis_features`` builds the ``features`` dict passed to + ``Analysis(...)``, stashing ``http_analysis`` and ``beacon_records`` into + it exactly as ``app/api/queue.py:_persist_analysis`` stashes them onto + ``result.features`` (neither has a dedicated ``Analysis`` column). Without + this, a case saved through the UI loses its HTTP findings (cleartext + creds / suspicious UA / suspicious URIs) and beacon records on restore, + while the same case saved via the API keeps them. + +Uses Streamlit's AppTest harness (see ``tests/test_config_ui.py`` for the +established pattern) since the helpers read ``st.session_state`` directly. +""" + +import pandas as pd +import streamlit as st +from streamlit.testing.v1 import AppTest + +from app.ui.cases_tab import _session_analysis_features + + +def _mitre_helper_app(): + import streamlit as st + + from app.ui.cases_tab import _session_mitre_techniques + + st.session_state["__result"] = _session_mitre_techniques() + + +def _make_app() -> AppTest: + return AppTest.from_function(_mitre_helper_app, default_timeout=30) + + +class TestSessionMitreTechniques: + def test_derives_technique_ids_from_attack_mapping(self): + at = _make_app() + at.session_state["attack_mapping"] = { + "techniques": [ + {"technique_id": "T1071.001", "tactic": "command-and-control"}, + {"technique_id": "T1059", "tactic": "execution"}, + ] + } + at.run() + + assert at.session_state["__result"] == ["T1071.001", "T1059"] + + def test_empty_attack_mapping_yields_empty_list(self): + at = _make_app() + at.session_state["attack_mapping"] = {} + at.run() + + assert at.session_state["__result"] == [] + + def test_missing_attack_mapping_yields_empty_list(self): + at = _make_app() + at.run() + + assert at.session_state["__result"] == [] + + def test_skips_techniques_missing_technique_id(self): + at = _make_app() + at.session_state["attack_mapping"] = { + "techniques": [ + {"tactic": "execution"}, + {"technique_id": "T1105"}, + ] + } + at.run() + + assert at.session_state["__result"] == ["T1105"] + + +class TestSessionAnalysisFeatures: + """``_session_analysis_features`` builds the ``features`` dict for both UI + manual-save paths, stashing ``http_analysis``/``beacon_records`` exactly as + ``app/api/queue.py:_persist_analysis`` stashes them onto ``result.features`` + (see module docstring). Uses bare ``st.session_state`` directly (no AppTest + script-run context needed), matching ``tests/test_case_restore.py``. + """ + + def setup_method(self): + st.session_state.clear() + + def test_stashes_http_analysis_from_session(self): + st.session_state["features"] = {"flows": []} + st.session_state["http_analysis"] = {"cleartext_creds": ["user:pass@1.2.3.4"]} + + result = _session_analysis_features() + + assert result["http_analysis"] == {"cleartext_creds": ["user:pass@1.2.3.4"]} + + def test_http_analysis_none_when_absent_from_session(self): + st.session_state["features"] = {"flows": []} + + result = _session_analysis_features() + + assert result["http_analysis"] is None + + def test_stashes_beacon_records_from_beacon_df(self): + st.session_state["features"] = {"flows": []} + st.session_state["beacon_df"] = pd.DataFrame([{"src": "10.0.0.5", "dst": "203.0.113.9", "score": 0.87}]) + + result = _session_analysis_features() + + assert result["beacon_records"] == [{"src": "10.0.0.5", "dst": "203.0.113.9", "score": 0.87}] + + def test_beacon_records_empty_when_beacon_df_missing(self): + st.session_state["features"] = {"flows": []} + + result = _session_analysis_features() + + assert result["beacon_records"] == [] + + def test_beacon_records_empty_when_beacon_df_empty(self): + st.session_state["features"] = {"flows": []} + st.session_state["beacon_df"] = pd.DataFrame() + + result = _session_analysis_features() + + assert result["beacon_records"] == [] + + def test_preserves_other_feature_keys(self): + st.session_state["features"] = {"flows": [{"src": "1.1.1.1"}]} + + result = _session_analysis_features() + + assert result["flows"] == [{"src": "1.1.1.1"}] + + def test_does_not_mutate_live_session_features_dict(self): + """The live ``st.session_state["features"]`` dict must not gain the + stash keys in place -- it may be read elsewhere in the same rerun, and + an in-place mutation would leak the stash into unrelated readers.""" + original = {"flows": []} + st.session_state["features"] = original + + _session_analysis_features() + + assert "http_analysis" not in original + assert "beacon_records" not in original + assert st.session_state["features"] is original + + def test_missing_features_yields_stash_only(self): + result = _session_analysis_features() + + assert result["http_analysis"] is None + assert result["beacon_records"] == [] diff --git a/tests/test_cef_export.py b/tests/test_cef_export.py index d5b34d4..2dddb4a 100644 --- a/tests/test_cef_export.py +++ b/tests/test_cef_export.py @@ -5,6 +5,7 @@ from app.utils.cef_export import ( CEFEvent, export_cef_text, + feed_rows_to_cef, format_syslog, generate_cef_events, ) @@ -42,6 +43,28 @@ def test_empty_extensions(self): cef = event.to_cef() assert cef.endswith("|1|") + def test_extension_value_newline_escaping(self): + """A newline/CR embedded in an attacker-influenced extension value must not + be able to forge a second CEF/syslog line.""" + event = CEFEvent( + signature_id="T-1", + name="Test", + severity=1, + extensions={"cs1": "evil\nCEF:0|Fake|Fake|1.0|999|Forged Event|10|src=9.9.9.9\r"}, + ) + cef = event.to_cef() + assert "\n" not in cef + assert "\r" not in cef + assert "cs1=evil\\nCEF:0|Fake|Fake|1.0|999|Forged Event|10|src\\=9.9.9.9\\r" in cef + + def test_name_newline_does_not_split_header(self): + """A newline embedded in the name field must not split the CEF header + into a second forged line.""" + event = CEFEvent(signature_id="T-1", name="Evil\nCEF:0|Fake|Fake|1.0|999|Forged|10|", severity=1) + cef = event.to_cef() + assert "\n" not in cef + assert "\r" not in cef + class TestFormatSyslog: def test_syslog_header(self): @@ -148,3 +171,77 @@ def test_full_export(self): def test_empty_export(self): text = export_cef_text() assert text == "" + + +class TestFeedRowsToCEF: + """feed_rows_to_cef renders feed-query rows (score 25-100, no priority_score) + directly — it must NOT apply the 0.4 priority-score gate used by + _events_from_iocs, or a LOW-severity row (score 25) would be silently + dropped from a feed pull.""" + + def test_low_row_not_dropped(self): + rows = [{"type": "ip", "value": "1.2.3.4", "score": 25, "severity": "low", "tags": []}] + text = feed_rows_to_cef(rows) + assert "CEF:0" in text + assert "1.2.3.4" in text + + def test_one_event_per_row(self): + rows = [ + {"type": "ip", "value": "1.1.1.1", "score": 25, "severity": "low", "tags": []}, + {"type": "ip", "value": "2.2.2.2", "score": 100, "severity": "critical", "tags": []}, + ] + text = feed_rows_to_cef(rows) + lines = [ln for ln in text.strip().splitlines() if ln] + assert len(lines) == 2 + + def test_severity_mapping_reuses_severity_map(self): + rows = [{"type": "ip", "value": "9.9.9.9", "score": 100, "severity": "critical", "tags": []}] + text = feed_rows_to_cef(rows) + # _SEVERITY_MAP["critical"] == 10 + assert "|10|" in text + + def test_unknown_severity_defaults_sanely(self): + rows = [{"type": "ip", "value": "8.8.8.8", "score": 50, "severity": "bogus", "tags": []}] + text = feed_rows_to_cef(rows) + assert "CEF:0" in text # doesn't raise; renders with a fallback severity + + def test_escaping_of_special_chars_in_value_and_tags(self): + rows = [ + { + "type": "url", + "value": "http://evil.example/a=b", + "score": 50, + "severity": "medium", + "tags": ["tag=1", "back\\slash"], + } + ] + text = feed_rows_to_cef(rows) + assert "a\\=b" in text + assert "tag\\=1" in text + assert "back\\\\slash" in text + + def test_empty_rows(self): + assert feed_rows_to_cef([]) == "" + + def test_hostname_in_syslog_header(self): + rows = [{"type": "ip", "value": "1.2.3.4", "score": 75, "severity": "high", "tags": []}] + text = feed_rows_to_cef(rows, hostname="feedhost") + assert "feedhost" in text + + def test_hostile_value_with_embedded_newline_yields_one_line_per_event(self): + """IOC values are attacker-influenced (derived from PCAP data). A value + containing an embedded newline must not be able to inject a forged + second CEF/syslog line into the SIEM feed.""" + rows = [ + { + "type": "domain", + "value": "evil.example\nCEF:0|Fake|Fake|1.0|999|Forged Event|10|src=9.9.9.9", + "score": 100, + "severity": "critical", + "tags": [], + }, + {"type": "ip", "value": "2.2.2.2", "score": 50, "severity": "medium", "tags": []}, + ] + text = feed_rows_to_cef(rows) + lines = [ln for ln in text.strip().splitlines() if ln] + assert len(lines) == 2 diff --git a/tests/test_correlation.py b/tests/test_correlation.py index f9d757a..eea7cc5 100644 --- a/tests/test_correlation.py +++ b/tests/test_correlation.py @@ -203,3 +203,122 @@ def test_correlate_shodan_transport_error_dict_ignored(): } results = correlate_indicators(features=features, osint=osint) assert results == [] + + +def test_correlate_http_cleartext_cred_lights_up_ip_indicator(): + """A cleartext-credential finding whose Host header is a raw IP should + attribute an http_cleartext_cred signal to that IP indicator.""" + features = { + "artifacts": {"ips": ["1.2.3.4"], "domains": []}, + "flows": [], + } + http_analysis = { + "cleartext_credentials": [{"host": "1.2.3.4", "uri": "/login", "username": "admin"}], + "suspicious_user_agents": [], + } + results = correlate_indicators(features=features, http_analysis=http_analysis) + assert len(results) == 1 + assert results[0].indicator == "1.2.3.4" + assert results[0].indicator_type == "ip" + sig = next(s for s in results[0].signals if s.name == "http_cleartext_cred") + assert sig.source == "http" + assert 0.0 < sig.score <= 1.0 + + +def test_correlate_http_cleartext_cred_lights_up_domain_indicator(): + """The same finding, but with a domain Host header, should attribute the + signal to the domain indicator instead (host may be IP OR domain).""" + features = { + "artifacts": {"ips": [], "domains": ["portal.evil-corp.test"]}, + "flows": [], + } + http_analysis = { + "cleartext_credentials": [{"host": "portal.evil-corp.test", "uri": "/login", "username": "admin"}], + "suspicious_user_agents": [], + } + results = correlate_indicators(features=features, http_analysis=http_analysis) + assert len(results) == 1 + assert results[0].indicator == "portal.evil-corp.test" + assert results[0].indicator_type == "domain" + assert any(s.name == "http_cleartext_cred" and s.source == "http" for s in results[0].signals) + + +def test_correlate_http_cleartext_cred_host_with_port_normalized(): + """A Host header carrying an explicit port (e.g. "example.com:8443") + must still attach its signal to the bare domain/IP indicator instead of + being silently dropped because the exact strings don't match.""" + features = { + "artifacts": {"ips": [], "domains": ["example.com"]}, + "flows": [], + } + http_analysis = { + "cleartext_credentials": [{"host": "example.com:8443", "uri": "/login", "username": "admin"}], + "suspicious_user_agents": [], + } + results = correlate_indicators(features=features, http_analysis=http_analysis) + assert len(results) == 1 + assert results[0].indicator == "example.com" + sig = next(s for s in results[0].signals if s.name == "http_cleartext_cred") + assert sig.source == "http" + + +def test_normalize_http_host_is_ipv6_safe(): + """Stripping the :port suffix must not mangle a bracketed IPv6 literal + (a naive rsplit on ':' would split one of the address's own colons).""" + from app.analysis.correlation import _normalize_http_host + + assert _normalize_http_host("example.com:8443") == "example.com" + assert _normalize_http_host("203.0.113.5:8443") == "203.0.113.5" + assert _normalize_http_host("example.com") == "example.com" + # Bracketed IPv6, with and without an explicit port — the address is preserved. + assert _normalize_http_host("[::1]") == "[::1]" + assert _normalize_http_host("[2001:db8::1]:8443") == "[2001:db8::1]" + assert _normalize_http_host("") == "" + + +def test_correlate_http_suspicious_ua_signal(): + features = { + "artifacts": {"ips": ["5.6.7.8"], "domains": []}, + "flows": [], + } + http_analysis = { + "cleartext_credentials": [], + "suspicious_user_agents": [ + {"host": "5.6.7.8", "user_agent": "sqlmap/1.6", "uri": "/", "reason": "known tool user-agent (sqlmap)"} + ], + } + results = correlate_indicators(features=features, http_analysis=http_analysis) + assert len(results) == 1 + sig = next(s for s in results[0].signals if s.name == "http_suspicious_ua") + assert sig.source == "http" + assert 0.0 < sig.score <= 1.0 + + +def test_correlate_http_cleartext_cred_scores_higher_than_suspicious_ua(): + """Cleartext creds are a stronger indicator than a suspicious UA alone.""" + features = {"artifacts": {"ips": ["9.9.9.9"], "domains": []}, "flows": []} + http_analysis = { + "cleartext_credentials": [{"host": "9.9.9.9", "uri": "/login", "username": "admin"}], + "suspicious_user_agents": [{"host": "9.9.9.9", "user_agent": "sqlmap/1.6", "uri": "/", "reason": "known tool"}], + } + results = correlate_indicators(features=features, http_analysis=http_analysis) + cred_sig = next(s for s in results[0].signals if s.name == "http_cleartext_cred") + ua_sig = next(s for s in results[0].signals if s.name == "http_suspicious_ua") + assert cred_sig.score > ua_sig.score + + +def test_correlate_without_http_analysis_has_no_http_signals(): + """Omitting http_analysis (the pre-3.5 default) must not change behaviour.""" + features = {"artifacts": {"ips": ["8.8.8.8"], "domains": []}, "flows": []} + osint = {"ips": {"8.8.8.8": {"greynoise": {"classification": "malicious"}}}, "domains": {}} + results = correlate_indicators(features=features, osint=osint) + assert len(results) == 1 + assert not any(s.name.startswith("http_") for s in results[0].signals) + + +def test_correlate_http_analysis_none_explicit_is_noop(): + features = {"artifacts": {"ips": ["8.8.8.8"], "domains": []}, "flows": []} + osint = {"ips": {"8.8.8.8": {"greynoise": {"classification": "malicious"}}}, "domains": {}} + results = correlate_indicators(features=features, osint=osint, http_analysis=None) + assert len(results) == 1 + assert not any(s.name.startswith("http_") for s in results[0].signals) diff --git a/tests/test_fixtures.py b/tests/test_fixtures.py index d383178..9f50857 100644 --- a/tests/test_fixtures.py +++ b/tests/test_fixtures.py @@ -10,6 +10,7 @@ import pathlib FIXTURE_DIR = pathlib.Path(__file__).parent / "fixtures" +DEMO_PCAP = pathlib.Path(__file__).parent.parent / "pcaps" / "demo.pcap" def test_tiny_pcap_exists(): @@ -30,3 +31,21 @@ def test_tiny_pcap_has_valid_magic(): b"\x0a\x0d\x0d\x0a", # pcapng } assert head in valid_magics, f"unexpected magic bytes: {head!r}" + + +def test_demo_pcap_exists(): + assert DEMO_PCAP.exists(), f"missing fixture: {DEMO_PCAP}. Regenerate via: python pcaps/_make_demo.py" + assert DEMO_PCAP.stat().st_size > 100, "demo pcap is suspiciously small; regenerate" + + +def test_demo_pcap_has_valid_magic(): + """First 4 bytes match a known pcap or pcapng magic.""" + head = DEMO_PCAP.read_bytes()[:4] + valid_magics = { + b"\xd4\xc3\xb2\xa1", # pcap classic, little-endian + b"\xa1\xb2\xc3\xd4", # pcap classic, byte-swapped + b"\x4d\x3c\xb2\xa1", # pcap nanosecond + b"\xa1\xb2\x3c\x4d", # pcap nanosecond, byte-swapped + b"\x0a\x0d\x0d\x0a", # pcapng + } + assert head in valid_magics, f"unexpected magic bytes: {head!r}" diff --git a/tests/test_http_analysis.py b/tests/test_http_analysis.py new file mode 100644 index 0000000..171a3e9 --- /dev/null +++ b/tests/test_http_analysis.py @@ -0,0 +1,403 @@ +"""Tests for HTTP analysis module.""" + +from unittest.mock import patch + +import pandas as pd +import pytest + +from app.pipeline.http_analysis import ( + HTTPRequest, + analyze_http, + detect_cleartext_credentials, + detect_suspicious_ua, + detect_suspicious_uri, + parse_http_log, +) + + +def _req(**overrides) -> HTTPRequest: + """Build an HTTPRequest with sane defaults, overridden per test.""" + defaults = dict( + ts=1234567890.0, + src="192.168.1.10", + dst="93.184.216.34", + host="www.example.com", + uri="/index.html", + method="GET", + user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64)", + username="", + status_code=200, + ) + defaults.update(overrides) + return HTTPRequest(**defaults) + + +class TestParseHTTPLog: + """Test parsing Zeek http.log DataFrame.""" + + def test_empty_dataframe(self): + df = pd.DataFrame() + records = parse_http_log(df) + assert records == [] + + def test_basic_parsing_dotted_columns(self): + df = pd.DataFrame( + [ + { + "ts": "1234567890.0", + "id.orig_h": "192.168.1.10", + "id.resp_h": "93.184.216.34", + "host": "www.example.com", + "uri": "/index.html", + "method": "GET", + "user_agent": "Mozilla/5.0", + "username": "-", + "password": "-", + "status_code": "200", + } + ] + ) + records = parse_http_log(df) + assert len(records) == 1 + r = records[0] + assert r.src == "192.168.1.10" + assert r.dst == "93.184.216.34" + assert r.host == "www.example.com" + assert r.uri == "/index.html" + assert r.method == "GET" + assert r.user_agent == "Mozilla/5.0" + assert r.username == "" # "-" treated as missing + assert r.status_code == 200 + + def test_underscore_column_fallback(self): + df = pd.DataFrame( + [ + { + "ts": "1234567890.0", + "id_orig_h": "10.0.0.5", + "id_resp_h": "10.0.0.1", + "host": "internal.local", + "uri": "/", + "method": "GET", + "user_agent": "-", + "status_code": "-", + } + ] + ) + records = parse_http_log(df) + assert len(records) == 1 + assert records[0].src == "10.0.0.5" + assert records[0].dst == "10.0.0.1" + assert records[0].user_agent == "" + assert records[0].status_code == 0 + + def test_missing_optional_fields_default_empty(self): + df = pd.DataFrame([{"ts": "1.0", "id.orig_h": "1.2.3.4"}]) + records = parse_http_log(df) + assert len(records) == 1 + assert records[0].host == "" + assert records[0].uri == "" + assert records[0].username == "" + + +class TestSuspiciousUA: + """Test suspicious User-Agent detection.""" + + def test_missing_ua_with_host_is_suspicious(self): + req = _req(user_agent="", host="www.example.com") + reason = detect_suspicious_ua(req) + assert reason is not None + assert "user-agent" in reason.lower() + + def test_missing_ua_without_host_is_not_flagged(self): + req = _req(user_agent="", host="") + assert detect_suspicious_ua(req) is None + + @pytest.mark.parametrize( + "ua", + [ + "python-requests/2.28.0", + "curl/7.79.1", + "Wget/1.21", + "PowerShell/7.2", + "Go-http-client/1.1", + "Nmap Scripting Engine", + "sqlmap/1.6", + ], + ) + def test_known_tool_ua_is_suspicious(self, ua): + req = _req(user_agent=ua) + reason = detect_suspicious_ua(req) + assert reason is not None + + def test_normal_browser_ua_is_not_flagged(self): + req = _req(user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15") + assert detect_suspicious_ua(req) is None + + +class TestCleartextCredentials: + """Test cleartext credential detection.""" + + def test_username_present_is_flagged(self): + req = _req(username="admin", uri="/login") + cred = detect_cleartext_credentials(req) + assert cred == {"host": req.host, "uri": "/login", "username": "admin"} + + def test_username_missing_is_not_flagged(self): + req = _req(username="") + assert detect_cleartext_credentials(req) is None + + def test_password_value_never_emitted(self): + req = _req(username="admin") + cred = detect_cleartext_credentials(req) + assert "password" not in cred + + +class TestSuspiciousURI: + """Test suspicious URI detection.""" + + @pytest.mark.parametrize("ext", [".exe", ".dll", ".ps1", ".scr", ".bin"]) + def test_risky_extension_is_flagged(self, ext): + req = _req(uri=f"/downloads/payload{ext}") + reason = detect_suspicious_uri(req) + assert reason is not None + + def test_long_uri_is_flagged(self): + req = _req(uri="/path?" + "a" * 600) + reason = detect_suspicious_uri(req) + assert reason is not None + assert "long" in reason.lower() + + def test_raw_ip_host_file_download_is_flagged(self): + req = _req(host="203.0.113.5", uri="/payload.dll") + reason = detect_suspicious_uri(req) + assert reason is not None + + def test_clean_uri_is_not_flagged(self): + req = _req(uri="/index.html") + assert detect_suspicious_uri(req) is None + + def test_empty_uri_is_not_flagged(self): + req = _req(uri="") + assert detect_suspicious_uri(req) is None + + +class TestAnalyzeHTTP: + """Test comprehensive HTTP analysis.""" + + def test_empty_zeek_tables_is_skipped(self): + result = analyze_http({}) + assert result == {"skipped": True} + + def test_empty_http_log_is_error(self): + result = analyze_http({"http.log": pd.DataFrame()}) + assert result.get("error") == "No HTTP log data" + assert result.get("records") == 0 + + def test_basic_analysis_all_heuristics(self): + df = pd.DataFrame( + [ + { + "ts": "1.0", + "id.orig_h": "192.168.1.10", + "id.resp_h": "93.184.216.34", + "host": "www.example.com", + "uri": "/index.html", + "method": "GET", + "user_agent": "Mozilla/5.0", + "username": "-", + "status_code": "200", + }, + { + "ts": "2.0", + "id.orig_h": "192.168.1.11", + "id.resp_h": "93.184.216.34", + "host": "www.example.com", + "uri": "/api/data", + "method": "GET", + "user_agent": "-", + "username": "-", + "status_code": "200", + }, + { + "ts": "3.0", + "id.orig_h": "192.168.1.12", + "id.resp_h": "198.51.100.9", + "host": "cdn.example.net", + "uri": "/pkg/tool.tar.gz", + "method": "GET", + "user_agent": "python-requests/2.28.0", + "username": "-", + "status_code": "200", + }, + { + "ts": "4.0", + "id.orig_h": "192.168.1.13", + "id.resp_h": "10.0.0.5", + "host": "10.0.0.5", + "uri": "/admin", + "method": "POST", + "user_agent": "Mozilla/5.0", + "username": "admin", + "password": "hunter2", + "status_code": "401", + }, + { + "ts": "5.0", + "id.orig_h": "192.168.1.14", + "id.resp_h": "198.51.100.10", + "host": "files.example.org", + "uri": "/downloads/update.exe", + "method": "GET", + "user_agent": "Mozilla/5.0", + "username": "-", + "status_code": "200", + }, + { + "ts": "6.0", + "id.orig_h": "192.168.1.15", + "id.resp_h": "203.0.113.5", + "host": "203.0.113.5", + "uri": "/malware.dll", + "method": "GET", + "user_agent": "Mozilla/5.0", + "username": "-", + "status_code": "200", + }, + { + "ts": "7.0", + "id.orig_h": "192.168.1.16", + "id.resp_h": "198.51.100.11", + "host": "files.example.org", + "uri": "/q?" + "x" * 600, + "method": "GET", + "user_agent": "Mozilla/5.0", + "username": "-", + "status_code": "200", + }, + ] + ) + result = analyze_http({"http.log": df}) + + assert result["total_requests"] == 7 + assert result["unique_hosts"] == 5 + assert result["methods"] == {"GET": 6, "POST": 1} + assert result["status_codes"]["200"] == 6 + assert result["status_codes"]["401"] == 1 + + assert result["alerts"]["suspicious_ua_count"] == 2 # missing UA + python-requests + assert result["alerts"]["cleartext_cred_count"] == 1 + assert result["alerts"]["suspicious_uri_count"] == 3 # .exe, .dll(+raw-ip), long query + + assert len(result["suspicious_user_agents"]) == 2 + assert len(result["cleartext_credentials"]) == 1 + assert result["cleartext_credentials"][0] == {"host": "10.0.0.5", "uri": "/admin", "username": "admin"} + # Password must never be emitted even though the log carried one. + assert all("password" not in c for c in result["cleartext_credentials"]) + assert len(result["suspicious_uris"]) == 3 + + def test_clean_traffic_yields_empty_alerts(self): + df = pd.DataFrame( + [ + { + "ts": "1.0", + "id.orig_h": "192.168.1.10", + "id.resp_h": "93.184.216.34", + "host": "www.example.com", + "uri": "/index.html", + "method": "GET", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)", + "username": "-", + "status_code": "200", + }, + { + "ts": "2.0", + "id.orig_h": "192.168.1.11", + "id.resp_h": "93.184.216.34", + "host": "www.example.com", + "uri": "/style.css", + "method": "GET", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)", + "username": "-", + "status_code": "200", + }, + ] + ) + result = analyze_http({"http.log": df}) + + assert result["total_requests"] == 2 + assert result["suspicious_user_agents"] == [] + assert result["cleartext_credentials"] == [] + assert result["suspicious_uris"] == [] + assert result["alerts"] == { + "suspicious_ua_count": 0, + "cleartext_cred_count": 0, + "suspicious_uri_count": 0, + } + + def test_http_log_path_used_over_capped_table(self): + """When http_log_path is given, the full uncapped log wins over zeek_tables.""" + capped_df = pd.DataFrame( + [ + { + "ts": "1.0", + "id.orig_h": "192.168.1.10", + "id.resp_h": "93.184.216.34", + "host": "capped.example.com", + "uri": "/", + "method": "GET", + "user_agent": "Mozilla/5.0", + "status_code": "200", + } + ] + ) + full_df = pd.DataFrame( + [ + { + "ts": "1.0", + "id.orig_h": "192.168.1.10", + "id.resp_h": "93.184.216.34", + "host": "full.example.com", + "uri": "/", + "method": "GET", + "user_agent": "Mozilla/5.0", + "status_code": "200", + }, + { + "ts": "2.0", + "id.orig_h": "192.168.1.11", + "id.resp_h": "93.184.216.34", + "host": "full.example.com", + "uri": "/extra", + "method": "GET", + "user_agent": "Mozilla/5.0", + "status_code": "200", + }, + ] + ) + with patch("app.pipeline.http_analysis.load_zeek_any", return_value=full_df) as mock_load: + result = analyze_http({"http.log": capped_df}, http_log_path="/fake/path/http.log") + + mock_load.assert_called_once_with("/fake/path/http.log") + assert result["total_requests"] == 2 + assert result["unique_hosts"] == 1 + + def test_http_log_path_load_failure_falls_back_to_zeek_tables(self): + df = pd.DataFrame( + [ + { + "ts": "1.0", + "id.orig_h": "192.168.1.10", + "id.resp_h": "93.184.216.34", + "host": "www.example.com", + "uri": "/", + "method": "GET", + "user_agent": "Mozilla/5.0", + "status_code": "200", + } + ] + ) + with patch("app.pipeline.http_analysis.load_zeek_any", side_effect=OSError("boom")): + result = analyze_http({"http.log": df}, http_log_path="/does/not/exist/http.log") + + assert result["total_requests"] == 1 diff --git a/tests/test_jobs_schema.py b/tests/test_jobs_schema.py index b9859e5..15ed8d1 100644 --- a/tests/test_jobs_schema.py +++ b/tests/test_jobs_schema.py @@ -26,6 +26,39 @@ def test_cases_has_source_column(tmp_path): assert "source" in cols, "cases.source column should exist for ui/api distinction" +def test_analyses_has_attack_json_column(tmp_path): + repo = CaseRepository(db_path=str(tmp_path / "test.db")) + conn = repo._get_conn() + try: + cols = [r[1] for r in conn.execute("PRAGMA table_info(analyses)").fetchall()] + finally: + conn.close() + assert "attack_json" in cols, "analyses.attack_json column should exist for persisted ATT&CK mappings" + + +def test_analysis_techniques_table_exists(tmp_path): + repo = CaseRepository(db_path=str(tmp_path / "test.db")) + conn = repo._get_conn() + try: + row = conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name='analysis_techniques'" + ).fetchone() + cols = [r[1] for r in conn.execute("PRAGMA table_info(analysis_techniques)").fetchall()] + finally: + conn.close() + assert row is not None, "analysis_techniques table should exist for feed mitre_techniques derivation" + assert set(cols) == {"analysis_id", "technique_id"} + + +def test_analysis_techniques_table_idempotent_recreate(tmp_path): + """_init_schema runs on every CaseRepository construction; the CREATE TABLE + IF NOT EXISTS / CREATE INDEX IF NOT EXISTS statements must not raise on a + second open against the same database file.""" + db_path = str(tmp_path / "test.db") + CaseRepository(db_path=db_path) + CaseRepository(db_path=db_path) # must not raise + + def test_jobs_table_columns(tmp_path): """Verify the jobs table has all expected columns.""" repo = CaseRepository(db_path=str(tmp_path / "test.db")) diff --git a/tests/test_layout.py b/tests/test_layout.py index decd501..07aa954 100644 --- a/tests/test_layout.py +++ b/tests/test_layout.py @@ -2,11 +2,168 @@ from __future__ import annotations +from streamlit.testing.v1 import AppTest + from app.ui.layout import resolve_logo_path LIGHT = "logo-256.png" DARK = "logo-dark-256.png" +# Production-shape dict, as stored in st.session_state["attack_mapping"] +# (AttackMapping.to_dict() output from the ATT&CK mapping pipeline stage). +PROD_ATTACK_MAPPING_DICT = { + "techniques": [ + { + "technique_id": "T1071.001", + "technique_name": "Application Layer Protocol: Web Protocols", + "tactic": "command-and-control", + "confidence": 0.85, + "evidence": ["Beaconing detected with score 0.85 to 203.0.113.5"], + }, + { + "technique_id": "T1568.002", + "technique_name": "Dynamic Resolution: Domain Generation Algorithms", + "tactic": "command-and-control", + "confidence": 0.6, + "evidence": ["DGA domains detected: xk3jd9a.example.com"], + }, + ], + "tactics_summary": {"command-and-control": 2}, + "kill_chain_phase": "command-and-control", + "overall_severity": "high", +} + + +def _render_attack_mapping_app(): + import streamlit as st + + from app.threat_intel import AttackMapping + from app.ui.layout import render_attack_mapping + + attack_mapping = st.session_state.get("attack_mapping", {}) + render_attack_mapping(st.container(), AttackMapping.from_dict(attack_mapping)) + + +class TestRenderAttackMappingFromDict: + def test_renders_without_exception_and_shows_mitre_attck(self): + at = AppTest.from_function(_render_attack_mapping_app, default_timeout=30) + at.session_state["attack_mapping"] = PROD_ATTACK_MAPPING_DICT + at.run() + + assert not at.exception + expanders = [e for e in at.expander if "MITRE ATT&CK" in (e.label or "")] + assert expanders, "Dashboard should render a MITRE ATT&CK Mapping expander" + + def test_empty_dict_renders_without_exception(self): + at = AppTest.from_function(_render_attack_mapping_app, default_timeout=30) + at.session_state["attack_mapping"] = {} + at.run() + + assert not at.exception + + +# Production-shape dict, as stored in st.session_state["http_analysis"] +# (analyze_http() output from app/pipeline/http_analysis.py). +PROD_HTTP_ANALYSIS_DICT = { + "total_requests": 7, + "unique_hosts": 5, + "methods": {"GET": 6, "POST": 1}, + "status_codes": {"200": 6, "401": 1}, + "suspicious_user_agents": [ + {"host": "10.0.0.5", "user_agent": "", "uri": "/", "reason": "missing user-agent"}, + { + "host": "10.0.0.6", + "user_agent": "python-requests/2.31", + "uri": "/api", + "reason": "known tool user-agent (python-requests)", + }, + ], + "cleartext_credentials": [ + {"host": "10.0.0.5", "uri": "/admin", "username": "admin"}, + ], + "suspicious_uris": [ + {"host": "10.0.0.7", "uri": "/payload.exe", "reason": "risky file extension (.exe)"}, + { + "host": "203.0.113.9", + "uri": "/files/tool.dll", + "reason": "risky file extension (.dll); raw-IP host serving file download", + }, + {"host": "10.0.0.8", "uri": "/" + "a" * 600, "reason": "long URI (605 chars)"}, + ], + "alerts": { + "suspicious_ua_count": 2, + "cleartext_cred_count": 1, + "suspicious_uri_count": 3, + }, +} + + +def _render_http_analysis_app(): + import streamlit as st + + from app.ui.layout import render_http_analysis + + render_http_analysis(st.container(), st.session_state.get("http_analysis")) + + +class TestRenderHttpAnalysis: + def test_production_shape_dict_renders_without_exception(self): + at = AppTest.from_function(_render_http_analysis_app, default_timeout=30) + at.session_state["http_analysis"] = PROD_HTTP_ANALYSIS_DICT + at.run() + + assert not at.exception + expanders = [e for e in at.expander if "HTTP Analysis" in (e.label or "")] + assert expanders, "Raw Data tab should render an HTTP Analysis expander" + assert expanders[0].proto.expanded, "alerts present -> expander should default to expanded" + + def test_none_shows_info_and_returns(self): + at = AppTest.from_function(_render_http_analysis_app, default_timeout=30) + at.session_state["http_analysis"] = None + at.run() + + assert not at.exception + assert at.info, "None http_analysis should render an info message, not crash" + + def test_skipped_shows_info(self): + at = AppTest.from_function(_render_http_analysis_app, default_timeout=30) + at.session_state["http_analysis"] = {"skipped": True} + at.run() + + assert not at.exception + assert at.info + + def test_error_shows_info(self): + at = AppTest.from_function(_render_http_analysis_app, default_timeout=30) + at.session_state["http_analysis"] = {"error": "No HTTP log data", "records": 0} + at.run() + + assert not at.exception + assert at.info + + def test_clean_run_no_alerts_not_expanded_and_shows_captions(self): + at = AppTest.from_function(_render_http_analysis_app, default_timeout=30) + at.session_state["http_analysis"] = { + "total_requests": 3, + "unique_hosts": 2, + "methods": {"GET": 3}, + "status_codes": {"200": 3}, + "suspicious_user_agents": [], + "cleartext_credentials": [], + "suspicious_uris": [], + "alerts": {"suspicious_ua_count": 0, "cleartext_cred_count": 0, "suspicious_uri_count": 0}, + } + at.run() + + assert not at.exception + expanders = [e for e in at.expander if "HTTP Analysis" in (e.label or "")] + assert expanders + assert not expanders[0].proto.expanded, "no alerts -> expander should default to collapsed" + captions = [c.value for c in at.caption] + assert any("No cleartext credentials" in c for c in captions) + assert any("No suspicious user-agents" in c for c in captions) + assert any("No suspicious URIs" in c for c in captions) + def _make_assets(tmp_path, *names): for name in names: diff --git a/tests/test_network_utils.py b/tests/test_network_utils.py index 2084124..96630f7 100644 --- a/tests/test_network_utils.py +++ b/tests/test_network_utils.py @@ -4,10 +4,14 @@ from unittest.mock import patch +import pytest + from app.utils.network_utils import ( _validate_domain, bulk_resolve_ips, + is_enrichable_domain, is_public_ipv4, + is_safe_webhook_url, pick_top_public_ips, resolve_ip, ) @@ -74,6 +78,46 @@ def test_hyphen_ok(self): assert _validate_domain("my-host.example.com") is True +class TestIsEnrichableDomain: + """is_enrichable_domain must keep internal/private hostnames out of third-party + OSINT submissions (VirusTotal, OTX) while still allowing public domains through.""" + + @pytest.mark.parametrize("d", ["evil.com", "sub.example.org", "cdn.cloudflare.net"]) + def test_accepts_public_domains(self, d): + assert is_enrichable_domain(d) is True + + @pytest.mark.parametrize( + "d", + [ + "dc01.internal.corp", # private TLD + "printer.local", # private TLD + "host.lan", # private TLD + "server.home", # private TLD + "box.intranet", # private TLD + "vm.test", # private TLD + "site.invalid", # private TLD + "1.2.3.4", # IP-shaped, not a domain + "10.0.0.1.in-addr.arpa", # reverse-lookup junk + "1.0.0.0.ip6.arpa", # reverse-lookup junk (IPv6) + "localhost", # no dot, private TLD + "workstation", # single-label name, no dot + "_dmarc", # underscore + no dot + ], + ) + def test_rejects_internal_and_malformed(self, d): + assert is_enrichable_domain(d) is False + + def test_rejects_underscore_domain(self): + # Underscored labels (e.g. DKIM/DMARC records) are not enrichable hostnames. + assert is_enrichable_domain("under_score.example.com") is False + + def test_rejects_empty(self): + assert is_enrichable_domain("") is False + + def test_rejects_none(self): + assert is_enrichable_domain(None) is False + + class TestResolveIP: @patch("app.utils.network_utils.socket.gethostbyaddr") def test_success(self, mock_gethostbyaddr): @@ -126,3 +170,111 @@ def test_pick_top_public_ips_ranks_by_packet_volume(): assert pick_top_public_ips(features, 1) == ["8.8.8.8"] # n <= 0 -> all public ips from artifacts assert set(pick_top_public_ips(features, 0)) == {"8.8.8.8", "1.1.1.1"} + + +class TestIsSafeWebhookUrl: + """SSRF guard for the API v2 completion-webhook feature (Task 4.4). + + hardened_session does NOT block private IPs, so this guard is the only + SSRF protection — the "public URL" and "unresolvable host" cases mock + socket.getaddrinfo so the suite never depends on real DNS/network access. + """ + + def test_rejects_non_http_scheme(self): + assert is_safe_webhook_url("ftp://example.com/hook") is False + + def test_rejects_missing_host(self): + assert is_safe_webhook_url("http:///hook") is False + + def test_rejects_empty_string(self): + assert is_safe_webhook_url("") is False + + def test_rejects_private_10(self): + assert is_safe_webhook_url("http://10.0.0.5/hook") is False + + def test_rejects_loopback(self): + assert is_safe_webhook_url("http://127.0.0.1/hook") is False + + def test_rejects_link_local(self): + assert is_safe_webhook_url("http://169.254.169.254/hook") is False + + def test_rejects_localhost_hostname(self): + # "localhost" resolves via the hosts file/nsswitch without hitting + # the network, so this is safe to assert without mocking. + assert is_safe_webhook_url("http://localhost/hook") is False + + def test_rejects_unresolvable_host(self, monkeypatch): + import socket as socket_mod + + def boom(host, port, *args, **kwargs): + raise socket_mod.gaierror("nodename nor servname provided") + + monkeypatch.setattr("app.utils.network_utils.socket.getaddrinfo", boom) + assert is_safe_webhook_url("https://does-not-exist.invalid/hook") is False + + def test_accepts_public_https_url(self, monkeypatch): + def fake_getaddrinfo(host, port, *args, **kwargs): + return [(2, 1, 6, "", ("93.184.216.34", 0))] + + monkeypatch.setattr("app.utils.network_utils.socket.getaddrinfo", fake_getaddrinfo) + assert is_safe_webhook_url("https://example.com/hook") is True + + def test_rejects_when_any_resolved_ip_is_private(self, monkeypatch): + """DNS-rebinding-style bypass: if ANY resolved address is private, refuse.""" + + def fake_getaddrinfo(host, port, *args, **kwargs): + return [ + (2, 1, 6, "", ("93.184.216.34", 0)), + (2, 1, 6, "", ("10.0.0.9", 0)), + ] + + monkeypatch.setattr("app.utils.network_utils.socket.getaddrinfo", fake_getaddrinfo) + assert is_safe_webhook_url("https://rebinding.example/hook") is False + + def test_rejects_ipv4_multicast(self, monkeypatch): + """ipaddress.is_global is (surprisingly) True for IPv4 multicast — must be rejected explicitly.""" + + def fake_getaddrinfo(host, port, *args, **kwargs): + return [(2, 1, 6, "", ("239.255.255.250", 0))] + + monkeypatch.setattr("app.utils.network_utils.socket.getaddrinfo", fake_getaddrinfo) + assert is_safe_webhook_url("http://multicast.example/hook") is False + + def test_rejects_ipv6_link_local_multicast(self, monkeypatch): + """ipaddress.is_global is also True for some IPv6 multicast ranges (e.g. ff02::1).""" + + def fake_getaddrinfo(host, port, *args, **kwargs): + return [(10, 1, 6, "", ("ff02::1", 0, 0, 0))] + + monkeypatch.setattr("app.utils.network_utils.socket.getaddrinfo", fake_getaddrinfo) + assert is_safe_webhook_url("http://[ff02::1]/hook") is False + + def test_rejects_ipv6_unspecified(self, monkeypatch): + def fake_getaddrinfo(host, port, *args, **kwargs): + return [(10, 1, 6, "", ("::", 0, 0, 0))] + + monkeypatch.setattr("app.utils.network_utils.socket.getaddrinfo", fake_getaddrinfo) + assert is_safe_webhook_url("http://[::]/hook") is False + + def test_accepts_public_ipv6_url(self, monkeypatch): + def fake_getaddrinfo(host, port, *args, **kwargs): + return [(10, 1, 6, "", ("2606:4700:4700::1111", 0, 0, 0))] + + monkeypatch.setattr("app.utils.network_utils.socket.getaddrinfo", fake_getaddrinfo) + assert is_safe_webhook_url("https://public-v6.example/hook") is True + + @pytest.mark.parametrize( + "url", + [ + "http://[::ffff:127.0.0.1]/", # IPv4-mapped IPv6 loopback + "http://2130706433/", # decimal-encoded 127.0.0.1 + "http://0x7f000001/", # hex-encoded 127.0.0.1 + "http://100.64.0.1/", # CGNAT shared address space (100.64.0.0/10) + "http://240.0.0.1/", # reserved (240.0.0.0/4) + ], + ) + def test_rejects_known_ssrf_bypass_vectors(self, url): + """Lock the guard against classic SSRF encoding/range bypasses across + Python versions -- these use the real resolver (getaddrinfo handles the + IP literals / integer forms without network access).""" + assert is_safe_webhook_url(url) is False diff --git a/tests/test_osint.py b/tests/test_osint.py index 0ec1dbe..4711cc4 100644 --- a/tests/test_osint.py +++ b/tests/test_osint.py @@ -20,6 +20,7 @@ PROBE_RESULT_UNREACHABLE, _cached_query, _j, + enrich, probe_providers, provider_status, ) @@ -303,3 +304,35 @@ def test_probe_passes_timeout(self): with _patched_session(get): probe_providers(ALL_KEYS, timeout=5.0) assert all(call.kwargs.get("timeout") == 5.0 for call in get.call_args_list) + + +class TestEnrichDomainFiltering: + """enrich() must not leak internal/private domain names to third-party OSINT + providers (VirusTotal, OTX) — only public, routable domains get queried.""" + + def _run(self, artifacts: dict) -> tuple[dict, list[str]]: + queried: list[str] = [] + + def fake_query_providers(indicator, providers, keys): + queried.append(indicator) + # cache_hits=1 so enrich() doesn't sleep(throttle) between calls. + return {"_raw": "ok"}, 1 + + cache = MagicMock() + with ( + patch("app.pipeline.osint._get_cache", return_value=cache), + patch("app.pipeline.osint._query_providers", side_effect=fake_query_providers), + ): + result = enrich(artifacts, keys={}) + return result, queried + + def test_internal_domain_is_never_queried(self): + result, queried = self._run({"ips": [], "domains": ["evil.com", "dc01.internal.corp"]}) + assert queried == ["evil.com"] + assert "evil.com" in result["domains"] + assert "dc01.internal.corp" not in result["domains"] + + def test_all_internal_domains_means_no_queries(self): + result, queried = self._run({"ips": [], "domains": ["printer.local", "host.lan"]}) + assert queried == [] + assert result["domains"] == {} diff --git a/tests/test_pdf_generator.py b/tests/test_pdf_generator.py index c62b12e..dcb2fec 100644 --- a/tests/test_pdf_generator.py +++ b/tests/test_pdf_generator.py @@ -247,13 +247,20 @@ def test_cover_page_timestamp_is_timezone_aware(self): """The cover 'Generated:' stamp must carry a timezone abbreviation.""" gen = PDFReportGenerator() html = gen._render_cover_page(case_info=None) - assert re.search(r"Generated: \d{4}-\d{2}-\d{2} \d{2}:\d{2} [A-Z]{2,5}", html) + # %Z renders as an alpha abbreviation (e.g. "UTC") on some hosts/CI and + # as a numeric offset (e.g. "+07") on others (e.g. macOS) — either + # satisfies "carries a timezone indicator". + assert re.search( + r"Generated: \d{4}-\d{2}-\d{2} \d{2}:\d{2} (?:[A-Z]{2,5}|[+-]\d{2}(?::?\d{2})?)", html + ) def test_appendix_timestamp_is_timezone_aware(self): """The appendix 'Generated:' stamp must carry a timezone abbreviation.""" gen = PDFReportGenerator() html = gen._render_appendix({"flows": [], "artifacts": {}}) - assert re.search(r"Generated: \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [A-Z]{2,5}", html) + # See test_cover_page_timestamp_is_timezone_aware: accept both an alpha + # tz abbreviation and a numeric UTC offset. + assert re.search(r"Generated: \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} (?:[A-Z]{2,5}|[+-]\d{2}(?::?\d{2})?)", html) def test_generate_without_weasyprint(self): """Test generation when weasyprint is not available.""" diff --git a/tests/test_pdf_integration.py b/tests/test_pdf_integration.py index fdfb472..7dd037a 100644 --- a/tests/test_pdf_integration.py +++ b/tests/test_pdf_integration.py @@ -21,6 +21,7 @@ from app.analysis.correlation import CorrelationResult, CorrelationSignal from app.reports.pdf_generator import WEASYPRINT_AVAILABLE, PDFReportGenerator, ReportConfig +from app.threat_intel.attack_mapping import AttackMapping, TechniqueMatch # --------------------------------------------------------------------------- # Realistic fixtures — shaped exactly like live-pipeline output @@ -215,6 +216,35 @@ def realistic_geoip_data() -> list[dict]: ] +@pytest.fixture +def realistic_attack_mapping() -> dict: + """attack_mapping dict shape matching AttackMapping.to_dict() — the form + persisted in st.session_state["attack_mapping"] and passed to the PDF + generator, not a simplified dict with similar-looking keys.""" + mapping = AttackMapping( + techniques=[ + TechniqueMatch( + technique_id="T1071.001", + technique_name="Application Layer Protocol: Web Protocols", + tactic="command-and-control", + confidence=0.85, + evidence=["Beacon to 34.12.37.224 over HTTPS at regular intervals"], + ), + TechniqueMatch( + technique_id="T1568.002", + technique_name="Dynamic Resolution: Domain Generation Algorithms", + tactic="command-and-control", + confidence=0.92, + evidence=["DGA domain malicious.dga.xyz detected"], + ), + ], + tactics_summary={"command-and-control": 2}, + kill_chain_phase="command-and-control", + overall_severity="high", + ) + return mapping.to_dict() + + @pytest.fixture def full_report_html( realistic_features, @@ -225,6 +255,7 @@ def full_report_html( realistic_tls_analysis, realistic_yara_results, realistic_geoip_data, + realistic_attack_mapping, ) -> str: """Complete report HTML built from the production-shape fixtures above. @@ -245,6 +276,7 @@ def full_report_html( beacon_df=realistic_beacon_df, correlations=realistic_correlations, geoip_data=realistic_geoip_data, + attack_mapping=realistic_attack_mapping, ) @@ -267,6 +299,7 @@ def test_full_pdf_with_all_sections( realistic_tls_analysis, realistic_yara_results, realistic_geoip_data, + realistic_attack_mapping, ): """Full report with every section populated and every data type realistic. @@ -297,6 +330,7 @@ def test_full_pdf_with_all_sections( beacon_df=realistic_beacon_df, correlations=realistic_correlations, geoip_data=realistic_geoip_data, + attack_mapping=realistic_attack_mapping, ) assert pdf is not None, "PDF generation returned None — see error logs" @@ -316,6 +350,7 @@ def test_html_contains_every_expected_section( realistic_tls_analysis, realistic_yara_results, realistic_geoip_data, + realistic_attack_mapping, ): """Verify section IDs and key tokens appear in the generated HTML.""" gen = PDFReportGenerator(ReportConfig(title="Section Coverage Test")) @@ -330,12 +365,14 @@ def test_html_contains_every_expected_section( beacon_df=realistic_beacon_df, correlations=realistic_correlations, geoip_data=realistic_geoip_data, + attack_mapping=realistic_attack_mapping, ) # Section anchors assert 'id="summary"' in html assert 'id="charts"' in html assert 'id="correlations"' in html + assert 'id="attack"' in html assert 'id="iocs"' in html assert 'id="osint"' in html assert 'id="dns"' in html @@ -359,6 +396,10 @@ def test_html_contains_every_expected_section( # YARA match rendering assert "CobaltStrike_Beacon" in html + # ATT&CK technique table rendering + assert "T1071.001" in html + assert "command-and-control" in html + def test_pdf_with_charts_embeds_png_data_uris( self, realistic_features, @@ -487,7 +528,12 @@ def test_numbering_stays_sequential_when_conditional_sections_absent(self, reali assert toc_ids == ["summary", "iocs", "flows", "appendix"] def test_report_timestamp_is_timezone_aware(self, full_report_html): - assert re.search(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}(:\d{2})? [A-Z]{2,5}", full_report_html) + # %Z renders as an alpha abbreviation (e.g. "UTC") on some hosts/CI and + # as a numeric offset (e.g. "+07") on others (e.g. macOS) — either + # satisfies "carries a timezone indicator". + assert re.search( + r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}(:\d{2})? (?:[A-Z]{2,5}|[+-]\d{2}(?::?\d{2})?)", full_report_html + ) class TestLLMMarkdownTableRendering: diff --git a/tests/test_phase4.py b/tests/test_phase4.py index f84316d..7b2a460 100644 --- a/tests/test_phase4.py +++ b/tests/test_phase4.py @@ -74,6 +74,17 @@ def test_to_dict(self): assert d["confidence"] == 0.7 assert len(d["evidence"]) == 2 + def test_from_dict_round_trip(self): + tech = TechniqueMatch( + technique_id="T1071.001", + technique_name="Application Layer Protocol: Web Protocols", + tactic="command-and-control", + confidence=0.85, + evidence=["HTTP beaconing detected", "second signal"], + ) + rebuilt = TechniqueMatch.from_dict(tech.to_dict()) + assert rebuilt == tech + class TestAttackMapping: """Test AttackMapping dataclass.""" @@ -114,6 +125,40 @@ def test_to_dict(self): assert d["kill_chain_phase"] == "command-and-control" assert d["overall_severity"] == "high" + def test_from_dict_round_trip(self): + mapping = AttackMapping( + techniques=[ + TechniqueMatch("T1071", "Test1", "command-and-control", 0.8, ["e1"]), + TechniqueMatch("T1095", "Test2", "command-and-control", 0.7, []), + TechniqueMatch("T1041", "Test3", "exfiltration", 0.6, ["e2", "e3"]), + ], + tactics_summary={"command-and-control": 2, "exfiltration": 1}, + kill_chain_phase="exfiltration", + overall_severity="critical", + ) + rebuilt = AttackMapping.from_dict(mapping.to_dict()) + assert rebuilt == mapping + assert [t.technique_id for t in rebuilt.techniques] == [t.technique_id for t in mapping.techniques] + assert rebuilt.tactics_summary == mapping.tactics_summary + assert rebuilt.kill_chain_phase == mapping.kill_chain_phase + assert rebuilt.overall_severity == mapping.overall_severity + + def test_from_dict_empty_dict_returns_valid_empty_mapping(self): + mapping = AttackMapping.from_dict({}) + assert mapping == AttackMapping() + assert mapping.techniques == [] + assert mapping.tactics_summary == {} + assert mapping.kill_chain_phase == "unknown" + assert mapping.overall_severity == "low" + + def test_from_dict_none_like_partial_dict_does_not_crash(self): + # Mapper-failure default is {}; also guard a partial dict missing keys. + mapping = AttackMapping.from_dict({"kill_chain_phase": "execution"}) + assert mapping.techniques == [] + assert mapping.tactics_summary == {} + assert mapping.kill_chain_phase == "execution" + assert mapping.overall_severity == "low" + class TestATTACKMapper: """Test ATTACKMapper class.""" @@ -146,7 +191,12 @@ def test_dns_tunneling_detection(self): def test_self_signed_cert_detection(self): mapper = ATTACKMapper() - tls_analysis = {"alerts": [{"type": "self_signed", "cert": "test.com"}]} + # Production shape from app.pipeline.tls_certs.analyze_certificates(): per-cert + # boolean flags in "certificates", not a list of {"type": ..., "cert": ...} objects. + tls_analysis = { + "certificates": [{"subject_cn": "test.com", "is_self_signed": True, "is_expired": False}], + "alerts": {"self_signed_count": 1, "expired_count": 0, "high_risk_count": 0}, + } mapping = mapper.map_analysis(tls_analysis=tls_analysis) technique_ids = [t.technique_id for t in mapping.techniques] assert "T1573.002" in technique_ids @@ -178,6 +228,39 @@ def test_kill_chain_phase(self): # DNS tunneling maps to command-and-control and exfiltration assert mapping.kill_chain_phase in ["command-and-control", "exfiltration"] + def test_ja3_uses_authoritative_db(self): + mapper = ATTACKMapper() + m = mapper.map_analysis( + osint={"ja3": {"a0e9f5d64349fb13191bc781f81f42e1": {"malware": True, "client": "Cobalt Strike"}}} + ) + # technique evidence should reference the ja3.py client, and the hash must resolve + from app.pipeline.ja3 import lookup_ja3 + + assert lookup_ja3("a0e9f5d64349fb13191bc781f81f42e1")["client"] == "Cobalt Strike" + technique_ids = [t.technique_id for t in m.techniques] + assert technique_ids # sanity: osint ja3 path still produces a technique + + def test_ja3_from_features_uses_authoritative_db(self): + """Regression test: attack_mapping._check_ja3_from_features previously carried its + own inline `known_malware_ja3` dict that contradicted app.pipeline.ja3's + KNOWN_JA3_FINGERPRINTS — the same hash was attributed to two different malware + families. This hash is authoritatively "Cobalt Strike" per ja3.py, NOT "TrickBot" + (the old inline dict's wrong label for this hash). + """ + from app.pipeline.ja3 import lookup_ja3 + + ja3_hash = "a0e9f5d64349fb13191bc781f81f42e1" + authoritative = lookup_ja3(ja3_hash) + assert authoritative["client"] == "Cobalt Strike" + + mapper = ATTACKMapper() + features = {"artifacts": {"ja3": [ja3_hash]}} + mapping = mapper.map_analysis(features=features) + + evidence_text = " ".join(e for t in mapping.techniques for e in t.evidence) + assert "Cobalt Strike" in evidence_text + assert "TrickBot" not in evidence_text + # ============================================================================= # IOC Scorer Tests @@ -453,6 +536,34 @@ def test_export_txt(self, sample_features): assert "1.2.3.4" in lines assert "5.6.7.8" in lines + def test_ioc_to_stix_pattern_sha512_hash(self): + exporter = IOCExporter() + record = IOCRecord(ioc_type="hash", value="e" * 128) + pattern = exporter._ioc_to_stix_pattern(record) + assert pattern == "[file:hashes.'SHA-512' = '" + "e" * 128 + "']" + + def test_ioc_to_stix_pattern_ipv6(self): + exporter = IOCExporter() + record = IOCRecord(ioc_type="ip", value="2001:db8::1") + pattern = exporter._ioc_to_stix_pattern(record) + assert pattern.startswith("[ipv6-addr") + + def test_ioc_to_stix_pattern_ja3_is_emitted(self): + exporter = IOCExporter() + record = IOCRecord(ioc_type="ja3", value="769,47-53-5-10,0-23-35,23-24,0") + pattern = exporter._ioc_to_stix_pattern(record) + assert pattern is not None + assert "x509-certificate:hashes.'JA3'" in pattern + + def test_export_stix_basic_includes_ja3_indicator(self): + features = {"artifacts": {"ja3": ["abc123"]}} + exporter = IOCExporter(features) + stix_bytes = exporter._export_stix_basic() + data = json.loads(stix_bytes.decode("utf-8")) + indicators = [o for o in data["objects"] if o["type"] == "indicator"] + assert len(indicators) == 1 + assert "JA3" in indicators[0]["pattern"] + def test_generate_ioc_filename(): filename = generate_ioc_filename("csv") @@ -586,6 +697,57 @@ def test_generate_stix_filename(): assert filename.endswith(".json") +# ============================================================================= +# Module-level ioc_to_stix_pattern / stix_hash_property (shared helper) +# ============================================================================= + + +def test_ioc_to_stix_pattern_hash_types(): + from app.utils.stix_export import ioc_to_stix_pattern + + assert ioc_to_stix_pattern("hash", "a" * 32) == "[file:hashes.MD5 = '" + "a" * 32 + "']" + assert ioc_to_stix_pattern("hash", "b" * 40).startswith("[file:hashes.'SHA-1'") + assert ioc_to_stix_pattern("hash", "c" * 64).startswith("[file:hashes.'SHA-256'") + assert ioc_to_stix_pattern("hash", "d" * 128).startswith("[file:hashes.'SHA-512'") + + +def test_ioc_to_stix_pattern_hash_unknown_length_is_none(): + from app.utils.stix_export import ioc_to_stix_pattern + + assert ioc_to_stix_pattern("hash", "deadbeef") is None + + +def test_ioc_to_stix_pattern_ipv6_and_ja3(): + from app.utils.stix_export import ioc_to_stix_pattern + + assert ioc_to_stix_pattern("ip", "2001:db8::1").startswith("[ipv6-addr") + assert ioc_to_stix_pattern("ip", "1.2.3.4").startswith("[ipv4-addr") + assert ioc_to_stix_pattern("ja3", "d" * 32).startswith("[x509-certificate") + + +def test_ioc_to_stix_pattern_escapes_backslash_before_quote(): + from app.utils.stix_export import ioc_to_stix_pattern + + pattern = ioc_to_stix_pattern("domain", r"weird\'value") + assert pattern == r"[domain-name:value = 'weird\\\'value']" + + +def test_ioc_to_stix_pattern_unknown_type_is_none(): + from app.utils.stix_export import ioc_to_stix_pattern + + assert ioc_to_stix_pattern("bogus", "x") is None + + +def test_stix_hash_property(): + from app.utils.stix_export import stix_hash_property + + assert stix_hash_property("a" * 32) == "MD5" + assert stix_hash_property("a" * 40) == "'SHA-1'" + assert stix_hash_property("a" * 64) == "'SHA-256'" + assert stix_hash_property("a" * 128) == "'SHA-512'" + assert stix_hash_property("a" * 10) is None + + # ============================================================================= # Q&A Tests # ============================================================================= diff --git a/tests/test_pipeline_runner.py b/tests/test_pipeline_runner.py index 8f349cf..2c90b08 100644 --- a/tests/test_pipeline_runner.py +++ b/tests/test_pipeline_runner.py @@ -48,6 +48,12 @@ def test_pipeline_result_to_dict_is_json_serializable(): dns_analysis={"dga_count": 3, "tunneling": []}, tls_analysis={"certs": [{"subject": "evil.example"}]}, beacon_df_records=[{"src": "10.0.0.1", "dst": "1.2.3.4", "score": 0.91}], + attack_mapping={ + "techniques": [{"technique_id": "T1071.001", "technique_name": "Web Protocols"}], + "tactics_summary": {"command-and-control": 1}, + "kill_chain_phase": "command-and-control", + "overall_severity": "high", + }, ) serialized = json.dumps(result.to_dict()) restored = json.loads(serialized) @@ -57,6 +63,8 @@ def test_pipeline_result_to_dict_is_json_serializable(): assert restored["dns_analysis"]["dga_count"] == 3 assert restored["tls_analysis"]["certs"][0]["subject"] == "evil.example" assert restored["beacon_df_records"][0]["src"] == "10.0.0.1" + assert restored["attack_mapping"]["techniques"][0]["technique_id"] == "T1071.001" + assert restored["attack_mapping"]["overall_severity"] == "high" def test_run_pipeline_executes_all_stages_against_fixture(): @@ -108,6 +116,12 @@ def test_run_pipeline_executes_all_stages_against_fixture(): # Some progress events should have fired assert any(e.kind == "phase_start" for e in events) assert any(e.kind == "phase_done" for e in events) + # ATT&CK mapping runs post-fan-out and must always populate a shape, even + # when tiny.pcap yields no technique matches. + assert isinstance(result.attack_mapping, dict) + assert "techniques" in result.attack_mapping + assert isinstance(result.mitre_techniques, list) + assert all(isinstance(t, str) for t in result.mitre_techniques) def test_run_pipeline_skips_disabled_stages(): @@ -167,11 +181,11 @@ def test_streamlit_to_options_mapping(): def test_stages_4_to_7_run_concurrently(monkeypatch, tmp_path): - """Stages 4 (DNS), 5 (TLS), 6 (beacon), 7 (carve) must overlap in time. + """Stages 4 (DNS), 5 (TLS), HTTP, 6 (beacon), 7 (carve) must overlap in time. All upstream stages are stubbed so only the post-parse fan-out is exercised. - Each of the four stage functions increments a lock-protected counter, sleeps, - then decrements — if they run sequentially the observed max concurrency is 1. + Each stage function increments a lock-protected counter, sleeps, then + decrements — if they run sequentially the observed max concurrency is 1. """ import threading import time @@ -224,6 +238,7 @@ def _fn(*args, **kwargs): monkeypatch.setattr(R, "analyze_dns", _tracked({})) monkeypatch.setattr(R, "analyze_certificates", _tracked({})) + monkeypatch.setattr(R, "analyze_http", _tracked({})) monkeypatch.setattr(R, "rank_beaconing", _tracked(pd.DataFrame())) monkeypatch.setattr(R, "carve_http_payloads", _tracked([])) @@ -234,17 +249,19 @@ def _fn(*args, **kwargs): progress=CallbackProgress(callback=lambda _e: None, total_phases=0), ) - for stage in ("dns_analysis", "tls_certs", "beacon", "carve"): + for stage in ("dns_analysis", "tls_certs", "http_analysis", "beacon", "carve"): assert stage in result.stages_run, f"{stage} did not run" assert state["max"] >= 2, f"stages 4-7 ran sequentially (max concurrency observed: {state['max']})" def test_stage_order_deterministic_and_carve_hashes_backfilled(monkeypatch, tmp_path): - """stages_run keeps the canonical dns→tls→beacon→carve order and carve hashes land in features. + """stages_run keeps the canonical dns→tls→http→beacon→carve order and carve hashes land in features. Regression guard for the fan-out: workers record outcomes per stage, the main thread assembles stages_run in canonical order and backfills carved sha256 - hashes into features["artifacts"]["hashes"] after the join. + hashes into features["artifacts"]["hashes"] after the join. HTTP analysis runs + on the same gate as DNS/TLS (Zeek ran + non-empty zeek_tables), so it appears + in the canonical order even though this fixture's zeek_tables carries no http.log. """ import pandas as pd @@ -279,6 +296,7 @@ def test_stage_order_deterministic_and_carve_hashes_backfilled(monkeypatch, tmp_ monkeypatch.setattr(R, "merge_zeek_dns", lambda zt, f: f) monkeypatch.setattr(R, "analyze_dns", lambda *a, **kw: {}) monkeypatch.setattr(R, "analyze_certificates", lambda *a, **kw: {}) + monkeypatch.setattr(R, "analyze_http", lambda *a, **kw: {}) monkeypatch.setattr(R, "rank_beaconing", lambda *a, **kw: pd.DataFrame()) monkeypatch.setattr(R, "carve_http_payloads", lambda *a, **kw: [{"sha256": carved_sha, "path": "x"}]) @@ -289,12 +307,191 @@ def test_stage_order_deterministic_and_carve_hashes_backfilled(monkeypatch, tmp_ progress=CallbackProgress(callback=lambda _e: None, total_phases=0), ) - canonical = ["dns_analysis", "tls_certs", "beacon", "carve"] + canonical = ["dns_analysis", "tls_certs", "http_analysis", "beacon", "carve"] observed = [s for s in result.stages_run if s in set(canonical)] assert observed == canonical, f"stage order not deterministic: {result.stages_run}" assert carved_sha in result.features["artifacts"]["hashes"], "carve sha256 backfill missing post-join" +def test_http_analysis_stage_wired_into_runner(monkeypatch, tmp_path): + """The HTTP analysis stage runs when Zeek ran and its result lands on + PipelineResult.http_analysis, stages_run, and to_dict(). It reads the uncapped + on-disk http.log path (zeek_log_paths["http.log"]), not the capped table.""" + import pandas as pd + + import app.pipeline.runner as R + from app.pipeline.progress import CallbackProgress + from app.pipeline.runner import PipelineOptions, run_pipeline + + pcap = tmp_path / "fake.pcap" + pcap.write_bytes(b"") + + http_log_path = str(tmp_path / "http.log") + seen_paths: list[str | None] = [] + + def _fake_analyze_http(zeek_tables, http_log_path=None, phase=None): + seen_paths.append(http_log_path) + return {"total_requests": 3} + + monkeypatch.setattr(R, "run_zeek", lambda p, d, phase=None: {"http.log": http_log_path}) + monkeypatch.setattr(R, "load_zeek_any", lambda p: pd.DataFrame({"host": ["evil.example"]})) + monkeypatch.setattr(R, "merge_zeek_dns", lambda zt, f: f) + monkeypatch.setattr(R, "analyze_http", _fake_analyze_http) + + result = run_pipeline( + pcap_path=str(pcap), + case_id="http_test", + options=PipelineOptions( + osint_enabled=False, + llm_enabled=False, + do_pyshark=False, + do_zeek=True, + do_carve=False, + do_yara=False, + pre_count=False, + ), + progress=CallbackProgress(callback=lambda _e: None, total_phases=0), + ) + + assert "http_analysis" in result.stages_run + assert result.http_analysis == {"total_requests": 3} + assert seen_paths == [http_log_path], "analyze_http must receive the uncapped on-disk http.log path" + + serialized = result.to_dict() + assert serialized["http_analysis"] == {"total_requests": 3} + + +def test_http_analysis_stage_runs_and_skips_when_no_http_log(monkeypatch, tmp_path): + """On an all-HTTPS / DNS-only capture (Zeek ran, no http.log), the HTTP stage + still runs — mirroring DNS/TLS — so the UI phase always starts and completes. + + The gate is `do_zeek and zeek_tables` (identical to DNS/TLS), NOT + `"http.log" in zeek_tables`; a tighter gate would leave the registered + "HTTP Analysis" phase uncreated and the Overall progress bar stuck below 100%. + analyze_http runs here with http_log_path=None (no http.log key) and returns + {"skipped": True}, so http_analysis appears in stages_run with a skipped result. + """ + import pandas as pd + + import app.pipeline.runner as R + from app.pipeline.progress import CallbackProgress + from app.pipeline.runner import PipelineOptions, run_pipeline + + pcap = tmp_path / "fake.pcap" + pcap.write_bytes(b"") + + seen_paths: list[str | None] = [] + + def _fake_analyze_http(zeek_tables, http_log_path=None, phase=None): + seen_paths.append(http_log_path) + return {"skipped": True} + + monkeypatch.setattr(R, "run_zeek", lambda p, d, phase=None: {"dns.log": str(tmp_path / "d.log")}) + monkeypatch.setattr(R, "load_zeek_any", lambda p: pd.DataFrame({"query": ["a.com"]})) + monkeypatch.setattr(R, "merge_zeek_dns", lambda zt, f: f) + monkeypatch.setattr(R, "analyze_dns", lambda *a, **kw: {}) + monkeypatch.setattr(R, "analyze_http", _fake_analyze_http) + + result = run_pipeline( + pcap_path=str(pcap), + case_id="http_skip_test", + options=PipelineOptions( + osint_enabled=False, + llm_enabled=False, + do_pyshark=False, + do_zeek=True, + do_carve=False, + do_yara=False, + pre_count=False, + ), + progress=CallbackProgress(callback=lambda _e: None, total_phases=0), + ) + + assert "http_analysis" in result.stages_run, "HTTP phase must run whenever Zeek ran (DNS/TLS parity)" + assert result.http_analysis == {"skipped": True} + assert seen_paths == [None], "analyze_http should receive http_log_path=None when no http.log was produced" + + +def test_http_analysis_stage_not_scheduled_without_zeek_tables(monkeypatch, tmp_path): + """The HTTP stage must NOT run when the gate is false — i.e. Zeek produced no + logs at all (empty zeek_tables). It shares DNS/TLS's gate, so it is dormant in + exactly the same conditions they are.""" + import app.pipeline.runner as R + from app.pipeline.progress import CallbackProgress + from app.pipeline.runner import PipelineOptions, run_pipeline + + pcap = tmp_path / "fake.pcap" + pcap.write_bytes(b"") + + def _boom(*args, **kwargs): + raise AssertionError("analyze_http must not run when zeek_tables is empty") + + # Zeek runs but yields no logs → empty zeek_tables → dns/tls/http all dormant. + monkeypatch.setattr(R, "run_zeek", lambda p, d, phase=None: {}) + monkeypatch.setattr(R, "merge_zeek_dns", lambda zt, f: f) + monkeypatch.setattr(R, "analyze_http", _boom) + + result = run_pipeline( + pcap_path=str(pcap), + case_id="http_gate_test", + options=PipelineOptions( + osint_enabled=False, + llm_enabled=False, + do_pyshark=False, + do_zeek=True, + do_carve=False, + do_yara=False, + pre_count=False, + ), + progress=CallbackProgress(callback=lambda _e: None, total_phases=0), + ) + + assert "http_analysis" not in result.stages_run + assert "dns_analysis" not in result.stages_run # same gate — both dormant + assert result.http_analysis == {} + assert result.to_dict()["http_analysis"] == {} + + +def test_http_analysis_failure_recorded_as_warning(monkeypatch, tmp_path): + """An exception inside the HTTP stage must be recorded as a warning, not raised.""" + import pandas as pd + + import app.pipeline.runner as R + from app.pipeline.progress import CallbackProgress + from app.pipeline.runner import PipelineOptions, run_pipeline + + pcap = tmp_path / "fake.pcap" + pcap.write_bytes(b"") + + monkeypatch.setattr(R, "run_zeek", lambda p, d, phase=None: {"http.log": str(tmp_path / "http.log")}) + monkeypatch.setattr(R, "load_zeek_any", lambda p: pd.DataFrame({"host": ["evil.example"]})) + monkeypatch.setattr(R, "merge_zeek_dns", lambda zt, f: f) + + def _raise(*args, **kwargs): + raise RuntimeError("boom") + + monkeypatch.setattr(R, "analyze_http", _raise) + + result = run_pipeline( + pcap_path=str(pcap), + case_id="http_fail_test", + options=PipelineOptions( + osint_enabled=False, + llm_enabled=False, + do_pyshark=False, + do_zeek=True, + do_carve=False, + do_yara=False, + pre_count=False, + ), + progress=CallbackProgress(callback=lambda _e: None, total_phases=0), + ) + + assert "http_analysis" not in result.stages_run + assert R.WARNING_HTTP_ANALYSIS_FAILED in result.warnings + assert result.http_analysis == {} + + def _stub_stage_dirs(monkeypatch, tmp_path, zeek_logs=None): """Redirect config dirs to tmp_path and stub zeek/carve to capture their out-dir args. @@ -327,6 +524,7 @@ def _fake_carve(p, d, phase=None): monkeypatch.setattr(R, "merge_zeek_dns", lambda zt, f: f) monkeypatch.setattr(R, "analyze_dns", lambda *a, **kw: {}) monkeypatch.setattr(R, "analyze_certificates", lambda *a, **kw: {}) + monkeypatch.setattr(R, "analyze_http", lambda *a, **kw: {}) monkeypatch.setattr(R, "carve_http_payloads", _fake_carve) return captured @@ -449,6 +647,36 @@ def test_run_pipeline_records_zeek_log_paths(monkeypatch, tmp_path): assert PipelineResult().zeek_log_paths == {} +def test_zeek_tables_capped_and_warned(monkeypatch, tmp_path): + """A busy capture's Zeek table is capped at C.ZEEK_TABLE_MAX_ROWS and the + truncation is surfaced via WARNING_ZEEK_TRUNCATED so callers aren't silently + handed a partial table.""" + import pandas as pd + + import app.pipeline.runner as R + from app.pipeline.progress import CallbackProgress + from app.pipeline.runner import run_pipeline + + big = pd.DataFrame({"query": [f"d{i}.com" for i in range(R.C.ZEEK_TABLE_MAX_ROWS + 10)]}) + log_path = str(tmp_path / "dns.log") + captured = _stub_stage_dirs(monkeypatch, tmp_path, zeek_logs={"dns.log": log_path}) + assert captured is not None + monkeypatch.setattr(R, "load_zeek_any", lambda p: big) + + pcap = tmp_path / "fake.pcap" + pcap.write_bytes(b"") + + result = run_pipeline( + pcap_path=str(pcap), + case_id="cap_test", + options=_zeek_carve_only_options(), + progress=CallbackProgress(callback=lambda _e: None, total_phases=0), + ) + + assert len(result.zeek_tables["dns.log"]) == R.C.ZEEK_TABLE_MAX_ROWS + assert R.WARNING_ZEEK_TRUNCATED in result.warnings + + def test_prune_stale_run_dirs_removes_only_old_dirs(tmp_path): """Run dirs older than the retention window are removed; fresh dirs and loose files stay.""" import os