From b2aa065e8387a6c74a31c32f0bc97267972274ad Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Mon, 13 Jul 2026 19:53:12 +0700 Subject: [PATCH 01/35] fix: cap zeek tables via config constant and warn on truncation --- CLAUDE.md | 1 + app/config.py | 8 ++++++++ app/pipeline/runner.py | 7 ++++++- tests/test_pipeline_runner.py | 30 ++++++++++++++++++++++++++++++ 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 2cab302..dcfc326 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -198,6 +198,7 @@ GitHub Actions (`.github/workflows/ci.yml`): - Default PyShark limit: 200,000 packets - OSINT top IPs default: 50 - `MAX_FLOW_SAMPLES`: 5,000 per-flow packet timestamps/lengths (true totals kept in `count`/`first_ts`/`last_ts`) +- `ZEEK_TABLE_MAX_ROWS`: 50,000 per-log row cap on in-memory Zeek tables (DNS analysis + UI preview read these capped frames; JA3 reads the full uncapped log via `zeek_log_paths`); truncation appends `WARNING_ZEEK_TRUNCATED` to `PipelineResult.warnings` - `RUN_DIR_RETENTION_SECONDS`: 7 days — per-run `data/zeek|carved//` dirs pruned on the next run - Subprocess timeouts: `ZEEK_TIMEOUT_SECONDS` 600, `PCAP_COUNT_TIMEOUT_SECONDS` 120, `CARVE_TIMEOUT_SECONDS` 300, `TLS_EXTRACT_TIMEOUT_SECONDS` 300, `LLM_PROBE_TIMEOUT_SECONDS` 15 diff --git a/app/config.py b/app/config.py index 4b83c4b..a709b7d 100644 --- a/app/config.py +++ b/app/config.py @@ -62,6 +62,14 @@ # below this; keep-first preserves true inter-arrival deltas (sampling would not). MAX_FLOW_SAMPLES = 5000 +# Per-log row cap on Zeek tables held in memory/session state. DNS analysis +# and the UI preview read these capped frames, so DNS/TLS fidelity is bounded +# by this value; JA3 extraction reads the full uncapped log via zeek_log_paths. +# Raised from the old hardcoded 2000 (which silently truncated busy captures); +# still bounded to protect session-state size. A WARNING_ZEEK_TRUNCATED is +# emitted when any log exceeds it. +ZEEK_TABLE_MAX_ROWS = 50000 + # Reverse DNS RDNS_CACHE_TTL_HOURS = 168 # 7 days RDNS_MAX_WORKERS = 10 # Concurrent rDNS lookups diff --git a/app/pipeline/runner.py b/app/pipeline/runner.py index 814e230..386d8b6 100644 --- a/app/pipeline/runner.py +++ b/app/pipeline/runner.py @@ -58,6 +58,7 @@ WARNING_TLS_CERTS_FAILED = "tls_certs_failed" WARNING_BEACON_FAILED = "beacon_failed" WARNING_CARVE_FAILED = "carve_failed" +WARNING_ZEEK_TRUNCATED = "zeek_tables_truncated" def _derive_run_id(case_id: str) -> str: @@ -260,7 +261,11 @@ def _run_zeek(h) -> None: df = load_zeek_any(log_path) except Exception: df = pd.DataFrame() - zeek_tables[name] = df.head(2000) + if len(df) > C.ZEEK_TABLE_MAX_ROWS: + df = df.head(C.ZEEK_TABLE_MAX_ROWS) + if WARNING_ZEEK_TRUNCATED not in warnings: + warnings.append(WARNING_ZEEK_TRUNCATED) + zeek_tables[name] = df stages_run.append("zeek") h.done("Zeek logs loaded.") else: diff --git a/tests/test_pipeline_runner.py b/tests/test_pipeline_runner.py index 8f349cf..dbc28a3 100644 --- a/tests/test_pipeline_runner.py +++ b/tests/test_pipeline_runner.py @@ -449,6 +449,36 @@ def test_run_pipeline_records_zeek_log_paths(monkeypatch, tmp_path): assert PipelineResult().zeek_log_paths == {} +def test_zeek_tables_capped_and_warned(monkeypatch, tmp_path): + """A busy capture's Zeek table is capped at C.ZEEK_TABLE_MAX_ROWS and the + truncation is surfaced via WARNING_ZEEK_TRUNCATED so callers aren't silently + handed a partial table.""" + import pandas as pd + + import app.pipeline.runner as R + from app.pipeline.progress import CallbackProgress + from app.pipeline.runner import run_pipeline + + big = pd.DataFrame({"query": [f"d{i}.com" for i in range(R.C.ZEEK_TABLE_MAX_ROWS + 10)]}) + log_path = str(tmp_path / "dns.log") + captured = _stub_stage_dirs(monkeypatch, tmp_path, zeek_logs={"dns.log": log_path}) + assert captured is not None + monkeypatch.setattr(R, "load_zeek_any", lambda p: big) + + pcap = tmp_path / "fake.pcap" + pcap.write_bytes(b"") + + result = run_pipeline( + pcap_path=str(pcap), + case_id="cap_test", + options=_zeek_carve_only_options(), + progress=CallbackProgress(callback=lambda _e: None, total_phases=0), + ) + + assert len(result.zeek_tables["dns.log"]) == R.C.ZEEK_TABLE_MAX_ROWS + assert R.WARNING_ZEEK_TRUNCATED in result.warnings + + def test_prune_stale_run_dirs_removes_only_old_dirs(tmp_path): """Run dirs older than the retention window are removed; fresh dirs and loose files stay.""" import os From e6466af8fae2e959476dba1f7299402d9455d2f7 Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Mon, 13 Jul 2026 20:02:34 +0700 Subject: [PATCH 02/35] fix: correct stix hash-type/ipv6/ja3 in ioc feed via shared helper --- app/api/routers/iocs.py | 14 +++-------- app/utils/stix_export.py | 50 +++++++++++++++++++-------------------- tests/api/test_iocs.py | 49 ++++++++++++++++++++++++++++++++++++++ tests/test_phase4.py | 51 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 127 insertions(+), 37 deletions(-) diff --git a/app/api/routers/iocs.py b/app/api/routers/iocs.py index f50f478..6f7b545 100644 --- a/app/api/routers/iocs.py +++ b/app/api/routers/iocs.py @@ -218,14 +218,6 @@ def _to_stix_bundle(rows: list[dict]) -> dict: def _row_to_stix_pattern(r: dict) -> str | None: - t = r["type"] - v = r["value"].replace("'", "\\'") - if t == "ip": - return f"[ipv4-addr:value = '{v}']" - if t == "domain": - return f"[domain-name:value = '{v}']" - if t == "url": - return f"[url:value = '{v}']" - if t == "hash": - return f"[file:hashes.'SHA-256' = '{v}']" - return None + from app.utils.stix_export import ioc_to_stix_pattern + + return ioc_to_stix_pattern(r["type"], r["value"]) diff --git a/app/utils/stix_export.py b/app/utils/stix_export.py index 98ce611..07d030b 100644 --- a/app/utils/stix_export.py +++ b/app/utils/stix_export.py @@ -19,6 +19,29 @@ def _escape_stix_value(value: str) -> str: return value.replace("\\", "\\\\").replace("'", "\\'") +def stix_hash_property(value: str) -> str | None: + """STIX hash property name from hex-digest length. None if unrecognized.""" + return {32: "MD5", 40: "'SHA-1'", 64: "'SHA-256'", 128: "'SHA-512'"}.get(len(value)) + + +def ioc_to_stix_pattern(ioc_type: str, value: str) -> str | None: + """Build a STIX 2.1 comparison-expression pattern for an IOC. None if unmappable.""" + escaped = _escape_stix_value(value) + if ioc_type == "ip": + kind = "ipv6-addr" if ":" in value else "ipv4-addr" + return f"[{kind}:value = '{escaped}']" + if ioc_type == "domain": + return f"[domain-name:value = '{escaped}']" + if ioc_type == "url": + return f"[url:value = '{escaped}']" + if ioc_type == "hash": + prop = stix_hash_property(value) + return f"[file:hashes.{prop} = '{escaped}']" if prop else None + if ioc_type == "ja3": + return f"[x509-certificate:hashes.'JA3' = '{escaped}']" + return None + + # Try to import stix2 library try: import stix2 @@ -82,32 +105,7 @@ def _create_identity(self) -> dict: def _ioc_to_pattern(self, ioc: "IOCRecord") -> str | None: """Convert IOC to STIX pattern.""" - escaped = _escape_stix_value(ioc.value) - if ioc.ioc_type == "ip": - # Check if IPv6 - if ":" in ioc.value: - return f"[ipv6-addr:value = '{escaped}']" - return f"[ipv4-addr:value = '{escaped}']" - elif ioc.ioc_type == "domain": - return f"[domain-name:value = '{escaped}']" - elif ioc.ioc_type == "hash": - # Determine hash type by length - hash_len = len(ioc.value) - if hash_len == 32: - return f"[file:hashes.MD5 = '{escaped}']" - elif hash_len == 40: - return f"[file:hashes.'SHA-1' = '{escaped}']" - elif hash_len == 64: - return f"[file:hashes.'SHA-256' = '{escaped}']" - elif hash_len == 128: - return f"[file:hashes.'SHA-512' = '{escaped}']" - elif ioc.ioc_type == "url": - return f"[url:value = '{escaped}']" - elif ioc.ioc_type == "ja3": - # JA3 as x509 extension (non-standard but useful) - return f"[x509-certificate:hashes.'JA3' = '{escaped}']" - - return None + return ioc_to_stix_pattern(ioc.ioc_type, ioc.value) def _get_indicator_labels(self, ioc: "IOCRecord") -> list[str]: """Get indicator labels based on IOC data.""" diff --git a/tests/api/test_iocs.py b/tests/api/test_iocs.py index c24a2ab..034ad4e 100644 --- a/tests/api/test_iocs.py +++ b/tests/api/test_iocs.py @@ -137,6 +137,55 @@ def test_iocs_stix_bundle(client): assert any(o.get("type") == "indicator" for o in objs) +def test_iocs_stix_md5_hash_type(client): + """An MD5-length hash IOC must emit file:hashes.MD5, not the hardcoded SHA-256.""" + from app.api.deps import get_repo + + md5_value = "c0" * 16 # 32 hex chars -> MD5 + repo = get_repo() + case = Case(id="case0005", title="md5") + repo.create_case(case) + repo.save_analysis( + Analysis( + case_id=case.id, + pcap_path="/tmp/md5.pcap", + iocs=[IOC(ioc_type=IOCType.HASH, value=md5_value)], + ) + ) + + r = client.get("/api/v1/iocs.stix", headers={"Authorization": "Bearer FEED"}) + assert r.status_code == 200 + body = r.json() + matching = [o["pattern"] for o in body["objects"] if o.get("type") == "indicator" and md5_value in o["pattern"]] + assert matching, "expected an indicator pattern for the MD5 IOC" + assert "file:hashes.MD5" in matching[0] + assert "SHA-256" not in matching[0] + + +def test_iocs_stix_ja3_not_dropped(client): + """JA3 rows must be emitted, not silently dropped.""" + from app.api.deps import get_repo + + ja3_value = "e" * 32 + repo = get_repo() + case = Case(id="case0006", title="ja3") + repo.create_case(case) + repo.save_analysis( + Analysis( + case_id=case.id, + pcap_path="/tmp/ja3.pcap", + iocs=[IOC(ioc_type=IOCType.JA3, value=ja3_value)], + ) + ) + + r = client.get("/api/v1/iocs.stix", headers={"Authorization": "Bearer FEED"}) + assert r.status_code == 200 + body = r.json() + matching = [o["pattern"] for o in body["objects"] if o.get("type") == "indicator" and ja3_value in o["pattern"]] + assert matching, "expected a JA3 indicator pattern, but it was dropped" + assert "x509-certificate" in matching[0] + + # ── Pagination ────────────────────────────────────────────────────────────── diff --git a/tests/test_phase4.py b/tests/test_phase4.py index f84316d..b36fa69 100644 --- a/tests/test_phase4.py +++ b/tests/test_phase4.py @@ -586,6 +586,57 @@ def test_generate_stix_filename(): assert filename.endswith(".json") +# ============================================================================= +# Module-level ioc_to_stix_pattern / stix_hash_property (shared helper) +# ============================================================================= + + +def test_ioc_to_stix_pattern_hash_types(): + from app.utils.stix_export import ioc_to_stix_pattern + + assert ioc_to_stix_pattern("hash", "a" * 32) == "[file:hashes.MD5 = '" + "a" * 32 + "']" + assert ioc_to_stix_pattern("hash", "b" * 40).startswith("[file:hashes.'SHA-1'") + assert ioc_to_stix_pattern("hash", "c" * 64).startswith("[file:hashes.'SHA-256'") + assert ioc_to_stix_pattern("hash", "d" * 128).startswith("[file:hashes.'SHA-512'") + + +def test_ioc_to_stix_pattern_hash_unknown_length_is_none(): + from app.utils.stix_export import ioc_to_stix_pattern + + assert ioc_to_stix_pattern("hash", "deadbeef") is None + + +def test_ioc_to_stix_pattern_ipv6_and_ja3(): + from app.utils.stix_export import ioc_to_stix_pattern + + assert ioc_to_stix_pattern("ip", "2001:db8::1").startswith("[ipv6-addr") + assert ioc_to_stix_pattern("ip", "1.2.3.4").startswith("[ipv4-addr") + assert ioc_to_stix_pattern("ja3", "d" * 32).startswith("[x509-certificate") + + +def test_ioc_to_stix_pattern_escapes_backslash_before_quote(): + from app.utils.stix_export import ioc_to_stix_pattern + + pattern = ioc_to_stix_pattern("domain", r"weird\'value") + assert pattern == r"[domain-name:value = 'weird\\\'value']" + + +def test_ioc_to_stix_pattern_unknown_type_is_none(): + from app.utils.stix_export import ioc_to_stix_pattern + + assert ioc_to_stix_pattern("bogus", "x") is None + + +def test_stix_hash_property(): + from app.utils.stix_export import stix_hash_property + + assert stix_hash_property("a" * 32) == "MD5" + assert stix_hash_property("a" * 40) == "'SHA-1'" + assert stix_hash_property("a" * 64) == "'SHA-256'" + assert stix_hash_property("a" * 128) == "'SHA-512'" + assert stix_hash_property("a" * 10) is None + + # ============================================================================= # Q&A Tests # ============================================================================= From 4ac42a2a648ab1f618e0cccf845ffe5267bf439b Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Mon, 13 Jul 2026 20:08:12 +0700 Subject: [PATCH 03/35] fix: enable wal + busy_timeout on cases.db to prevent lock errors cases.db is the hottest SQLite store in the app (pool workers, API thread, and Streamlit UI all write concurrently) yet was the only store without journal/timeout pragmas, unlike key_repository.py, osint_cache.py, and rdns_cache.py. Sets WAL once in _init_schema and busy_timeout=30000 per-connection in _get_conn, matching the existing convention. foreign_keys remains OFF -- delete_case/clear_all rely on explicit manual cascade. --- app/database/repository.py | 5 +++++ tests/test_case_management.py | 12 ++++++++++++ 2 files changed, 17 insertions(+) diff --git a/app/database/repository.py b/app/database/repository.py index 7ec6b0f..82e6902 100644 --- a/app/database/repository.py +++ b/app/database/repository.py @@ -38,12 +38,17 @@ def _get_conn(self) -> sqlite3.Connection: """Get database connection.""" conn = sqlite3.connect(str(self._db_path)) conn.row_factory = sqlite3.Row + conn.execute("PRAGMA busy_timeout=30000") return conn def _init_schema(self): """Initialize database schema.""" conn = self._get_conn() try: + # Persisted once per database file; lets concurrent readers/writers + # (pool workers, API thread, Streamlit UI) coexist instead of + # failing immediately with "database is locked". + conn.execute("PRAGMA journal_mode=WAL") conn.executescript( """ -- Cases table diff --git a/tests/test_case_management.py b/tests/test_case_management.py index 9c9c117..d563f10 100644 --- a/tests/test_case_management.py +++ b/tests/test_case_management.py @@ -478,6 +478,18 @@ def test_get_statistics(self, repo): assert stats["by_status"].get("open", 0) == 1 assert stats["by_status"].get("closed", 0) == 1 + def test_repo_uses_wal_and_busy_timeout(self, tmp_path): + """Concurrent writers (pool workers, API thread, Streamlit UI) must not + hit 'database is locked' immediately -- WAL + a busy_timeout give SQLite + room to retry instead of failing fast.""" + repo = CaseRepository(db_path=str(tmp_path / "t.db")) + conn = repo._get_conn() + try: + assert conn.execute("PRAGMA journal_mode").fetchone()[0].lower() == "wal" + assert conn.execute("PRAGMA busy_timeout").fetchone()[0] == 30000 + finally: + conn.close() + class TestCascadeDeletion: """Deleting a case must remove every related row — the FK pragma is off, so From e00c7c1caace6f1f7f2d610378cc1babfa9c4f1f Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Mon, 13 Jul 2026 20:21:58 +0700 Subject: [PATCH 04/35] fix: persist attack timeline to session state so pdf report includes it --- app/main.py | 66 ++++++++++++++++++++++++++++++--------------- app/ui/cases_tab.py | 1 + 2 files changed, 45 insertions(+), 22 deletions(-) diff --git a/app/main.py b/app/main.py index fceee9c..79e6900 100644 --- a/app/main.py +++ b/app/main.py @@ -467,6 +467,7 @@ def _run_single_pcap_pipeline( "correlations": None, "flow_asymmetry": None, "port_anomalies": None, + "attack_timeline": [], "__batch_result": None, } ) @@ -667,6 +668,23 @@ def _run_single_pcap_pipeline( except Exception as e: logger.warning("Post-analysis failed: %s", e) + try: + from app.analysis.narrator import AttackNarrator + + _timeline = AttackNarrator().create_timeline( + features=st.session_state.get("features"), + dns_analysis=st.session_state.get("dns_analysis"), + yara_results=st.session_state.get("yara_results"), + beacon_results=( + get_df_state("beacon_df").to_dict("records") if not get_df_state("beacon_df").empty else [] + ), + tls_analysis=st.session_state.get("tls_analysis"), + ) + st.session_state["attack_timeline"] = [e.to_dict() for e in _timeline] + except Exception as e: + logger.debug("timeline precompute failed: %s", e) + st.session_state["attack_timeline"] = [] + batch_tracker.finish_all( f"Batch complete: {batch_result.summary['successful']}/{batch_result.summary['total_files']} files." ) @@ -741,6 +759,23 @@ def _run_single_pcap_pipeline( except Exception as e: logger.warning("Post-analysis failed: %s", e) + try: + from app.analysis.narrator import AttackNarrator + + _timeline = AttackNarrator().create_timeline( + features=st.session_state.get("features"), + dns_analysis=st.session_state.get("dns_analysis"), + yara_results=st.session_state.get("yara_results"), + beacon_results=( + get_df_state("beacon_df").to_dict("records") if not get_df_state("beacon_df").empty else [] + ), + tls_analysis=st.session_state.get("tls_analysis"), + ) + st.session_state["attack_timeline"] = [e.to_dict() for e in _timeline] + except Exception as e: + logger.debug("timeline precompute failed: %s", e) + st.session_state["attack_timeline"] = [] + # ---- LLM REPORT (shared for single & batch) ---- features = st.session_state.get("features") or {} zeek_tables = st.session_state.get("zeek_tables") or {} @@ -1249,29 +1284,16 @@ def _run_single_pcap_pipeline( st.plotly_chart(fig, use_container_width=True) render_chart_hint("Node size = connections. Color: blue=low, red=high threat.") - # Attack timeline (full-width, if available) - try: - from app.analysis.narrator import AttackNarrator - - narrator = AttackNarrator() - timeline = narrator.create_timeline( - features=feats, - dns_analysis=st.session_state.get("dns_analysis"), - yara_results=st.session_state.get("yara_results"), - beacon_results=( - get_df_state("beacon_df").to_dict("records") if not get_df_state("beacon_df").empty else [] - ), - tls_analysis=st.session_state.get("tls_analysis"), + # Attack timeline (full-width, if available) — stored at pipeline-completion + # time (see post-analysis blocks above) so it survives without needing to + # recompute here, and so the PDF report can reuse the same data. + timeline_dicts = st.session_state.get("attack_timeline") or [] + if timeline_dicts: + st.plotly_chart( + plot_attack_timeline(timeline_dicts), + use_container_width=True, ) - if timeline: - timeline_dicts = [e.to_dict() for e in timeline] - st.plotly_chart( - plot_attack_timeline(timeline_dicts), - use_container_width=True, - ) - render_chart_hint("Diamond markers show events by severity and time.") - except Exception as e: - logger.debug("chart rendering failed: %s", e) + render_chart_hint("Diamond markers show events by severity and time.") # --- Traffic profiling charts --- if filtered_flows: diff --git a/app/ui/cases_tab.py b/app/ui/cases_tab.py index 09f1033..c2d5c1f 100644 --- a/app/ui/cases_tab.py +++ b/app/ui/cases_tab.py @@ -82,6 +82,7 @@ def _restore_analysis_to_session(analysis: Analysis) -> None: st.session_state["correlations"] = None st.session_state["flow_asymmetry"] = None st.session_state["port_anomalies"] = None + st.session_state["attack_timeline"] = [] st.session_state["rdns_map"] = {} st.session_state["filter_ips"] = set() st.session_state["filter_protos"] = set() From 479ad35c9aec8e07895583c4b026d75a44008a77 Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Mon, 13 Jul 2026 20:34:52 +0700 Subject: [PATCH 05/35] fix: filter internal/private domains out of osint enrichment to prevent leaks --- app/pipeline/osint.py | 3 ++- app/utils/network_utils.py | 47 +++++++++++++++++++++++++++++++++++++ tests/test_network_utils.py | 43 +++++++++++++++++++++++++++++++++ tests/test_osint.py | 33 ++++++++++++++++++++++++++ 4 files changed, 125 insertions(+), 1 deletion(-) diff --git a/app/pipeline/osint.py b/app/pipeline/osint.py index 05e4cf5..b06183d 100644 --- a/app/pipeline/osint.py +++ b/app/pipeline/osint.py @@ -13,6 +13,7 @@ from app.pipeline.state import PhaseHandle from app.security.opsec import hardened_session from app.utils.common import is_public_ipv4, resolve_ip +from app.utils.network_utils import is_enrichable_domain logger = logging.getLogger(__name__) @@ -432,7 +433,7 @@ def enrich( prev_doms = prev.get("domains") or {} all_ips = [ip for ip in artifacts.get("ips", []) if is_public_ipv4(ip)] - all_doms = artifacts.get("domains", []) + all_doms = [d for d in artifacts.get("domains", []) if is_enrichable_domain(d)] # Deduplicate: skip indicators already enriched in a previous batch run new_ips = [ip for ip in all_ips if ip not in prev_ips] diff --git a/app/utils/network_utils.py b/app/utils/network_utils.py index f29d10e..9ea84a6 100644 --- a/app/utils/network_utils.py +++ b/app/utils/network_utils.py @@ -117,6 +117,53 @@ def _validate_domain(domain: str) -> bool: return True +_PRIVATE_TLDS = frozenset( + { + "local", + "internal", + "lan", + "corp", + "home", + "intranet", + "test", + "example", + "invalid", + "localhost", + } +) + + +def is_enrichable_domain(domain: str) -> bool: + """True only for public, routable domains worth sending to OSINT providers. + + Prevents leaking internal infrastructure names (dc01.internal.corp, *.local) + to third-party SaaS (VirusTotal submissions are visible to paid subscribers). + + Rejects: invalid domains per ``_validate_domain`` (dot required, no + underscore), single-label names, reverse-DNS artifacts + (``*.in-addr.arpa``/``*.ip6.arpa``), IP-shaped strings, and private/internal + TLDs (local/internal/lan/corp/home/intranet/test/example/invalid/localhost). + """ + if not domain or not _validate_domain(domain): # dot required, no underscore + return False + lower = domain.lower().rstrip(".") + + # _validate_domain's regex allows all-numeric labels, so IP literals like + # "1.2.3.4" pass it — reject them explicitly since they aren't domains. + try: + ipaddress.ip_address(lower) + return False + except ValueError: + pass + + if lower.endswith((".in-addr.arpa", ".ip6.arpa")): + return False + tld = lower.rsplit(".", 1)[-1] + if tld in _PRIVATE_TLDS: + return False + return True + + def get_whois_info(target: str) -> dict | str: """ Retrieve WHOIS information for a domain or IP. diff --git a/tests/test_network_utils.py b/tests/test_network_utils.py index 2084124..9360400 100644 --- a/tests/test_network_utils.py +++ b/tests/test_network_utils.py @@ -4,9 +4,12 @@ from unittest.mock import patch +import pytest + from app.utils.network_utils import ( _validate_domain, bulk_resolve_ips, + is_enrichable_domain, is_public_ipv4, pick_top_public_ips, resolve_ip, @@ -74,6 +77,46 @@ def test_hyphen_ok(self): assert _validate_domain("my-host.example.com") is True +class TestIsEnrichableDomain: + """is_enrichable_domain must keep internal/private hostnames out of third-party + OSINT submissions (VirusTotal, OTX) while still allowing public domains through.""" + + @pytest.mark.parametrize("d", ["evil.com", "sub.example.org", "cdn.cloudflare.net"]) + def test_accepts_public_domains(self, d): + assert is_enrichable_domain(d) is True + + @pytest.mark.parametrize( + "d", + [ + "dc01.internal.corp", # private TLD + "printer.local", # private TLD + "host.lan", # private TLD + "server.home", # private TLD + "box.intranet", # private TLD + "vm.test", # private TLD + "site.invalid", # private TLD + "1.2.3.4", # IP-shaped, not a domain + "10.0.0.1.in-addr.arpa", # reverse-lookup junk + "1.0.0.0.ip6.arpa", # reverse-lookup junk (IPv6) + "localhost", # no dot, private TLD + "workstation", # single-label name, no dot + "_dmarc", # underscore + no dot + ], + ) + def test_rejects_internal_and_malformed(self, d): + assert is_enrichable_domain(d) is False + + def test_rejects_underscore_domain(self): + # Underscored labels (e.g. DKIM/DMARC records) are not enrichable hostnames. + assert is_enrichable_domain("under_score.example.com") is False + + def test_rejects_empty(self): + assert is_enrichable_domain("") is False + + def test_rejects_none(self): + assert is_enrichable_domain(None) is False + + class TestResolveIP: @patch("app.utils.network_utils.socket.gethostbyaddr") def test_success(self, mock_gethostbyaddr): diff --git a/tests/test_osint.py b/tests/test_osint.py index 0ec1dbe..4711cc4 100644 --- a/tests/test_osint.py +++ b/tests/test_osint.py @@ -20,6 +20,7 @@ PROBE_RESULT_UNREACHABLE, _cached_query, _j, + enrich, probe_providers, provider_status, ) @@ -303,3 +304,35 @@ def test_probe_passes_timeout(self): with _patched_session(get): probe_providers(ALL_KEYS, timeout=5.0) assert all(call.kwargs.get("timeout") == 5.0 for call in get.call_args_list) + + +class TestEnrichDomainFiltering: + """enrich() must not leak internal/private domain names to third-party OSINT + providers (VirusTotal, OTX) — only public, routable domains get queried.""" + + def _run(self, artifacts: dict) -> tuple[dict, list[str]]: + queried: list[str] = [] + + def fake_query_providers(indicator, providers, keys): + queried.append(indicator) + # cache_hits=1 so enrich() doesn't sleep(throttle) between calls. + return {"_raw": "ok"}, 1 + + cache = MagicMock() + with ( + patch("app.pipeline.osint._get_cache", return_value=cache), + patch("app.pipeline.osint._query_providers", side_effect=fake_query_providers), + ): + result = enrich(artifacts, keys={}) + return result, queried + + def test_internal_domain_is_never_queried(self): + result, queried = self._run({"ips": [], "domains": ["evil.com", "dc01.internal.corp"]}) + assert queried == ["evil.com"] + assert "evil.com" in result["domains"] + assert "dc01.internal.corp" not in result["domains"] + + def test_all_internal_domains_means_no_queries(self): + result, queried = self._run({"ips": [], "domains": ["printer.local", "host.lan"]}) + assert queried == [] + assert result["domains"] == {} From 3d656f2b374d24986b433822140935fa98978d65 Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Mon, 13 Jul 2026 20:43:27 +0700 Subject: [PATCH 06/35] fix: route ioc_export stix through shared helper (sha-512, ipv6, ja3) IOCExporter._ioc_to_stix_pattern was a third, stale copy of STIX-pattern logic that dropped SHA-512 hashes, mislabeled IPv6 as ipv4-addr, and always returned None for JA3 IOCs. Delegate to the shared app.utils.stix_export.ioc_to_stix_pattern helper instead. --- app/utils/ioc_export.py | 27 ++++----------------------- tests/test_phase4.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/app/utils/ioc_export.py b/app/utils/ioc_export.py index 2f66477..f6e6a39 100644 --- a/app/utils/ioc_export.py +++ b/app/utils/ioc_export.py @@ -367,29 +367,10 @@ def _export_stix_basic(self, ioc_types: list[str] | None = None, min_score: floa return json.dumps(bundle, indent=2).encode("utf-8") def _ioc_to_stix_pattern(self, ioc: IOCRecord) -> str | None: - """Convert IOC to STIX pattern.""" - from app.utils.stix_export import _escape_stix_value - - escaped = _escape_stix_value(ioc.value) - if ioc.ioc_type == "ip": - return f"[ipv4-addr:value = '{escaped}']" - elif ioc.ioc_type == "domain": - return f"[domain-name:value = '{escaped}']" - elif ioc.ioc_type == "hash": - # Determine hash type by length - if len(ioc.value) == 32: - return f"[file:hashes.MD5 = '{escaped}']" - elif len(ioc.value) == 40: - return f"[file:hashes.'SHA-1' = '{escaped}']" - elif len(ioc.value) == 64: - return f"[file:hashes.'SHA-256' = '{escaped}']" - elif ioc.ioc_type == "url": - return f"[url:value = '{escaped}']" - elif ioc.ioc_type == "ja3": - # JA3 doesn't have a standard STIX pattern - return None - - return None + """Convert IOC to STIX pattern via the shared serializer.""" + from app.utils.stix_export import ioc_to_stix_pattern + + return ioc_to_stix_pattern(ioc.ioc_type, ioc.value) def generate_ioc_filename(format_type: str) -> str: diff --git a/tests/test_phase4.py b/tests/test_phase4.py index b36fa69..93fd05f 100644 --- a/tests/test_phase4.py +++ b/tests/test_phase4.py @@ -453,6 +453,34 @@ def test_export_txt(self, sample_features): assert "1.2.3.4" in lines assert "5.6.7.8" in lines + def test_ioc_to_stix_pattern_sha512_hash(self): + exporter = IOCExporter() + record = IOCRecord(ioc_type="hash", value="e" * 128) + pattern = exporter._ioc_to_stix_pattern(record) + assert pattern == "[file:hashes.'SHA-512' = '" + "e" * 128 + "']" + + def test_ioc_to_stix_pattern_ipv6(self): + exporter = IOCExporter() + record = IOCRecord(ioc_type="ip", value="2001:db8::1") + pattern = exporter._ioc_to_stix_pattern(record) + assert pattern.startswith("[ipv6-addr") + + def test_ioc_to_stix_pattern_ja3_is_emitted(self): + exporter = IOCExporter() + record = IOCRecord(ioc_type="ja3", value="769,47-53-5-10,0-23-35,23-24,0") + pattern = exporter._ioc_to_stix_pattern(record) + assert pattern is not None + assert "x509-certificate:hashes.'JA3'" in pattern + + def test_export_stix_basic_includes_ja3_indicator(self): + features = {"artifacts": {"ja3": ["abc123"]}} + exporter = IOCExporter(features) + stix_bytes = exporter._export_stix_basic() + data = json.loads(stix_bytes.decode("utf-8")) + indicators = [o for o in data["objects"] if o["type"] == "indicator"] + assert len(indicators) == 1 + assert "JA3" in indicators[0]["pattern"] + def test_generate_ioc_filename(): filename = generate_ioc_filename("csv") From ce8098406ebc689e3cef96784c4be5ae4c204059 Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Mon, 13 Jul 2026 20:59:15 +0700 Subject: [PATCH 07/35] fix: dedupe ja3 attribution in attack mapper via authoritative lookup_ja3 _check_ja3_from_features carried its own inline known_malware_ja3 dict that contradicted app.pipeline.ja3's KNOWN_JA3_FINGERPRINTS -- the same hash was attributed to two different malware families (e.g. TrickBot here vs. Cobalt Strike there), eroding analyst trust in attribution. Route through lookup_ja3() instead, deriving confidence from its authoritative severity field rather than a flat constant. --- app/threat_intel/attack_mapping.py | 30 +++++++++++++++++---------- tests/test_phase4.py | 33 ++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/app/threat_intel/attack_mapping.py b/app/threat_intel/attack_mapping.py index 953ec4a..6596d2c 100644 --- a/app/threat_intel/attack_mapping.py +++ b/app/threat_intel/attack_mapping.py @@ -6,6 +6,8 @@ import logging from dataclasses import dataclass, field +from app.pipeline.ja3 import lookup_ja3 + logger = logging.getLogger(__name__) # Valid IOC types for validation @@ -18,6 +20,17 @@ MAX_JA3_FINGERPRINTS = 50 MAX_FLOWS = 1000 +# Confidence assigned to a malicious JA3 match, keyed by the authoritative +# severity from app.pipeline.ja3.KNOWN_JA3_FINGERPRINTS. Unknown/missing +# severities fall back to JA3_DEFAULT_CONFIDENCE. +JA3_SEVERITY_CONFIDENCE = { + "critical": 0.9, + "high": 0.75, + "medium": 0.6, + "low": 0.4, +} +JA3_DEFAULT_CONFIDENCE = 0.75 + # Average packet size estimate (bytes) when only packet count is available AVG_PACKET_SIZE_ESTIMATE = 800 @@ -504,24 +517,19 @@ def _check_ja3_from_features(self, features: dict) -> list[TechniqueMatch]: if not ja3_list: return techniques - # Known malicious JA3 patterns (subset for demonstration) - # In production, this would query a threat intel database - known_malware_ja3 = { - "72a589da586844d7f0818ce684948eea": "Emotet", - "a0e9f5d64349fb13191bc781f81f42e1": "TrickBot", - } - for ja3 in ja3_list: ja3_hash = ja3 if isinstance(ja3, str) else ja3.get("hash", "") - if ja3_hash in known_malware_ja3: - malware_name = known_malware_ja3[ja3_hash] + match = lookup_ja3(ja3_hash) + if match and match.get("malware"): + client = match.get("client", "Unknown") + confidence = JA3_SEVERITY_CONFIDENCE.get(match.get("severity"), JA3_DEFAULT_CONFIDENCE) techniques.append( TechniqueMatch( technique_id="T1071.001", technique_name="Web Protocols", tactic="command-and-control", - confidence=0.85, - evidence=[f"Known malware JA3 fingerprint detected: {malware_name}"], + confidence=confidence, + evidence=[f"Known malware JA3 fingerprint detected: {client}"], ) ) diff --git a/tests/test_phase4.py b/tests/test_phase4.py index 93fd05f..9672d01 100644 --- a/tests/test_phase4.py +++ b/tests/test_phase4.py @@ -178,6 +178,39 @@ def test_kill_chain_phase(self): # DNS tunneling maps to command-and-control and exfiltration assert mapping.kill_chain_phase in ["command-and-control", "exfiltration"] + def test_ja3_uses_authoritative_db(self): + mapper = ATTACKMapper() + m = mapper.map_analysis( + osint={"ja3": {"a0e9f5d64349fb13191bc781f81f42e1": {"malware": True, "client": "Cobalt Strike"}}} + ) + # technique evidence should reference the ja3.py client, and the hash must resolve + from app.pipeline.ja3 import lookup_ja3 + + assert lookup_ja3("a0e9f5d64349fb13191bc781f81f42e1")["client"] == "Cobalt Strike" + technique_ids = [t.technique_id for t in m.techniques] + assert technique_ids # sanity: osint ja3 path still produces a technique + + def test_ja3_from_features_uses_authoritative_db(self): + """Regression test: attack_mapping._check_ja3_from_features previously carried its + own inline `known_malware_ja3` dict that contradicted app.pipeline.ja3's + KNOWN_JA3_FINGERPRINTS — the same hash was attributed to two different malware + families. This hash is authoritatively "Cobalt Strike" per ja3.py, NOT "TrickBot" + (the old inline dict's wrong label for this hash). + """ + from app.pipeline.ja3 import lookup_ja3 + + ja3_hash = "a0e9f5d64349fb13191bc781f81f42e1" + authoritative = lookup_ja3(ja3_hash) + assert authoritative["client"] == "Cobalt Strike" + + mapper = ATTACKMapper() + features = {"artifacts": {"ja3": [ja3_hash]}} + mapping = mapper.map_analysis(features=features) + + evidence_text = " ".join(e for t in mapping.techniques for e in t.evidence) + assert "Cobalt Strike" in evidence_text + assert "TrickBot" not in evidence_text + # ============================================================================= # IOC Scorer Tests From 9b82f0e4f07fd31988c73533d8abb9337b00797d Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Mon, 13 Jul 2026 21:18:24 +0700 Subject: [PATCH 08/35] feat: build att&ck mapping in runner and persist to analyses.attack_json MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instantiate ATTACKMapper post-fan-out in run_pipeline() and populate PipelineResult.mitre_techniques/attack_mapping; a mapper failure is caught and logged so it never breaks the pipeline. Persist attack_mapping to a new analyses.attack_json column (idempotent ALTER, same compress/decompress pattern as dns_json/tls_json). Fixes a latent _check_tls bug found while wiring this up for the first time: analyze_certificates() reports per-cert is_self_signed/is_expired flags in a certificates list plus aggregate counts in alerts, not a list of {type, cert} alert objects — the mapper was dead code until now so this never fired in production. Updated the test in test_phase4.py that had encoded the fictitious shape as expected behavior, and added tests/test_attack_mapping.py with production-shape regression coverage. --- app/database/models.py | 3 + app/database/repository.py | 18 ++++- app/pipeline/runner.py | 25 +++++++ app/threat_intel/attack_mapping.py | 65 ++++++++++-------- tests/test_attack_mapping.py | 105 +++++++++++++++++++++++++++++ tests/test_case_management.py | 42 ++++++++++++ tests/test_jobs_schema.py | 10 +++ tests/test_phase4.py | 7 +- tests/test_pipeline_runner.py | 14 ++++ 9 files changed, 258 insertions(+), 31 deletions(-) create mode 100644 tests/test_attack_mapping.py diff --git a/app/database/models.py b/app/database/models.py index 7d7e30b..e77f8a2 100644 --- a/app/database/models.py +++ b/app/database/models.py @@ -156,6 +156,7 @@ class Analysis: yara_results: dict | None = None dns_analysis: dict | None = None tls_analysis: dict | None = None + attack_mapping: dict = field(default_factory=dict) iocs: list[IOC] = field(default_factory=list) def to_dict(self) -> dict: @@ -172,6 +173,7 @@ def to_dict(self) -> dict: "yara_results": self.yara_results, "dns_analysis": self.dns_analysis, "tls_analysis": self.tls_analysis, + "attack_mapping": self.attack_mapping, "iocs": [ioc.to_dict() for ioc in self.iocs], } @@ -196,6 +198,7 @@ def from_dict(cls, data: dict) -> "Analysis": yara_results=data.get("yara_results"), dns_analysis=data.get("dns_analysis"), tls_analysis=data.get("tls_analysis"), + attack_mapping=data.get("attack_mapping", {}), iocs=iocs, ) diff --git a/app/database/repository.py b/app/database/repository.py index 82e6902..8b1e625 100644 --- a/app/database/repository.py +++ b/app/database/repository.py @@ -78,6 +78,8 @@ def _init_schema(self): dns_json TEXT, tls_json TEXT ); + -- attack_json (ATT&CK mapping) is added below via idempotent ALTER + -- since this table may already exist from an earlier schema version. -- IOCs extracted from analyses CREATE TABLE IF NOT EXISTS iocs ( @@ -148,6 +150,12 @@ def _init_schema(self): conn.commit() except sqlite3.OperationalError: pass # column already exists + + try: + conn.execute("ALTER TABLE analyses ADD COLUMN attack_json TEXT") + conn.commit() + except sqlite3.OperationalError: + pass # column already exists finally: conn.close() @@ -411,13 +419,14 @@ def save_analysis(self, analysis: Analysis) -> str: yara_json = self._compress_json(analysis.yara_results) if analysis.yara_results else None dns_json = self._compress_json(analysis.dns_analysis) if analysis.dns_analysis else None tls_json = self._compress_json(analysis.tls_analysis) if analysis.tls_analysis else None + attack_json = self._compress_json(analysis.attack_mapping) if analysis.attack_mapping else None conn.execute( """ INSERT OR REPLACE INTO analyses (id, case_id, pcap_path, pcap_hash, packet_count, analyzed_at, - features_json, osint_json, report_md, yara_json, dns_json, tls_json) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + features_json, osint_json, report_md, yara_json, dns_json, tls_json, attack_json) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( analysis.id, @@ -432,6 +441,7 @@ def save_analysis(self, analysis: Analysis) -> str: yara_json, dns_json, tls_json, + attack_json, ), ) @@ -707,6 +717,9 @@ def _row_to_analysis(self, row: dict, conn: sqlite3.Connection) -> Analysis: yara_results = self._decompress_json(row.get("yara_json")) dns_analysis = self._decompress_json(row.get("dns_json")) tls_analysis = self._decompress_json(row.get("tls_json")) + # attack_json may be absent (NULL, or column missing on rows saved before + # this column existed) — always default to {} rather than None. + attack_mapping = self._decompress_json(row.get("attack_json")) or {} # Load IOCs ioc_rows = conn.execute("SELECT * FROM iocs WHERE analysis_id = ?", (row["id"],)).fetchall() @@ -734,6 +747,7 @@ def _row_to_analysis(self, row: dict, conn: sqlite3.Connection) -> Analysis: yara_results=yara_results, dns_analysis=dns_analysis, tls_analysis=tls_analysis, + attack_mapping=attack_mapping, iocs=iocs, ) diff --git a/app/pipeline/runner.py b/app/pipeline/runner.py index 386d8b6..ba2cae5 100644 --- a/app/pipeline/runner.py +++ b/app/pipeline/runner.py @@ -120,6 +120,7 @@ class PipelineResult: warnings: list[str] = field(default_factory=list) summary_narrative: str | None = None mitre_techniques: list[str] = field(default_factory=list) + attack_mapping: dict = field(default_factory=dict) dns_analysis: dict = field(default_factory=dict) tls_analysis: dict = field(default_factory=dict) beacon_df_records: list[dict] = field(default_factory=list) @@ -144,6 +145,7 @@ def to_dict(self) -> dict: "warnings": list(self.warnings), "summary_narrative": self.summary_narrative, "mitre_techniques": list(self.mitre_techniques), + "attack_mapping": dict(self.attack_mapping), "dns_analysis": dict(self.dns_analysis), "tls_analysis": dict(self.tls_analysis), "beacon_df_records": list(self.beacon_df_records), @@ -402,6 +404,27 @@ def _run_carve(h) -> None: features["artifacts"]["hashes"].append(sha) features["artifacts"]["hashes"] = uniq_sorted(features["artifacts"]["hashes"]) + # --- ATT&CK mapping (post-fan-out, not a canonical pipeline stage) --- + # Runs against whatever features/dns/tls/beacon data are final at this point. + # The runner has no yara/osint results (those run in the callers' main.py/queue.py + # after this function returns), so map_analysis is called without them — expected. + # A mapper bug must never break the pipeline, hence the broad except. + attack_mapping_dict: dict = {} + mitre_techniques: list[str] = [] + try: + from app.threat_intel import ATTACKMapper + + _mapping = ATTACKMapper().map_analysis( + features=features, + dns_analysis=dns_result, + tls_analysis=tls_result, + beacon_results=beacon_records, + ) + attack_mapping_dict = _mapping.to_dict() + mitre_techniques = [t.technique_id for t in _mapping.techniques] + except Exception as exc: + logger.error("ATT&CK mapping failed: %s", exc) + return PipelineResult( case_id=case_id, analysis_id=None, # caller writes the Analysis row and fills this in @@ -409,6 +432,8 @@ def _run_carve(h) -> None: duration_seconds=time.time() - start, stages_run=stages_run, warnings=warnings, + mitre_techniques=mitre_techniques, + attack_mapping=attack_mapping_dict, dns_analysis=dns_result, tls_analysis=tls_result, beacon_df_records=beacon_records, diff --git a/app/threat_intel/attack_mapping.py b/app/threat_intel/attack_mapping.py index 6596d2c..268e914 100644 --- a/app/threat_intel/attack_mapping.py +++ b/app/threat_intel/attack_mapping.py @@ -411,40 +411,49 @@ def _check_dns(self, dns_analysis: dict) -> list[TechniqueMatch]: return techniques def _check_tls(self, tls_analysis: dict) -> list[TechniqueMatch]: - """Check TLS certificate analysis for anomalies.""" + """Check TLS certificate analysis for anomalies. + + ``analyze_certificates()`` (app/pipeline/tls_certs.py) reports per-cert + flags in a ``certificates`` list (``is_self_signed``/``is_expired`` booleans) + and aggregate counts in ``alerts`` (``self_signed_count``/``expired_count``/ + ``high_risk_count``) — there is no list of ``{"type": ..., "cert": ...}`` + alert objects. Read the actual shape rather than a hypothetical one. + """ techniques = [] - alerts = tls_analysis.get("alerts", []) + certificates = tls_analysis.get("certificates", []) - for alert in alerts: - alert_type = alert.get("type", "") + def _cert_label(cert: dict) -> str: + return cert.get("subject_cn") or cert.get("fingerprint_sha256") or "unknown" - if alert_type == "self_signed": - rule = self.detection_rules["self_signed_cert"] - cert = alert.get("cert", "unknown") - for tech in rule["techniques"]: - techniques.append( - TechniqueMatch( - technique_id=tech["id"], - technique_name=tech["name"], - tactic=tech["tactic"], - confidence=0.7, - evidence=[f"Self-signed certificate: {cert}"], - ) + self_signed = [c for c in certificates if c.get("is_self_signed")] + if self_signed: + rule = self.detection_rules["self_signed_cert"] + cert_label = _cert_label(self_signed[0]) + for tech in rule["techniques"]: + techniques.append( + TechniqueMatch( + technique_id=tech["id"], + technique_name=tech["name"], + tactic=tech["tactic"], + confidence=0.7, + evidence=[f"Self-signed certificate: {cert_label}"], ) + ) - elif alert_type == "expired": - rule = self.detection_rules["expired_cert"] - cert = alert.get("cert", "unknown") - for tech in rule["techniques"]: - techniques.append( - TechniqueMatch( - technique_id=tech["id"], - technique_name=tech["name"], - tactic=tech["tactic"], - confidence=0.5, - evidence=[f"Expired certificate: {cert}"], - ) + expired = [c for c in certificates if c.get("is_expired")] + if expired: + rule = self.detection_rules["expired_cert"] + cert_label = _cert_label(expired[0]) + for tech in rule["techniques"]: + techniques.append( + TechniqueMatch( + technique_id=tech["id"], + technique_name=tech["name"], + tactic=tech["tactic"], + confidence=0.5, + evidence=[f"Expired certificate: {cert_label}"], ) + ) return techniques diff --git a/tests/test_attack_mapping.py b/tests/test_attack_mapping.py new file mode 100644 index 0000000..3cd4024 --- /dev/null +++ b/tests/test_attack_mapping.py @@ -0,0 +1,105 @@ +"""Tests for the ATT&CK mapping engine (app/threat_intel/attack_mapping.py). + +The mapper was built but never instantiated at runtime until it was wired +into app/pipeline/runner.py — these tests use production-shape output from +the real analysis stages (app/pipeline/tls_certs.py, app/pipeline/dns_analysis.py, +app/pipeline/beacon.py) rather than "looks reasonable" dicts, since a shape +mismatch here previously escaped notice precisely because the mapper was dead +code (see the `_check_tls` fix in this same change). +""" + +from __future__ import annotations + +from app.threat_intel import ATTACKMapper + + +class TestCheckTLS: + """Regression coverage for the tls_analysis shape consumed by _check_tls.""" + + def test_self_signed_certificate_is_detected(self): + """analyze_certificates() reports per-cert flags in `certificates`, not a + list of {"type": ..., "cert": ...} alert objects — must not raise. + """ + tls_analysis = { + "total_certificates": 1, + "certificates": [ + { + "subject_cn": "evil.example", + "is_self_signed": True, + "is_expired": False, + } + ], + "alerts": {"self_signed_count": 1, "expired_count": 0, "high_risk_count": 0}, + } + mapping = ATTACKMapper().map_analysis(tls_analysis=tls_analysis) + ids = {t.technique_id for t in mapping.techniques} + assert "T1587.003" in ids + assert "T1573.002" in ids + evidence = [e for t in mapping.techniques for e in t.evidence] + assert any("evil.example" in e for e in evidence) + + def test_expired_certificate_is_detected(self): + tls_analysis = { + "total_certificates": 1, + "certificates": [ + { + "subject_cn": "stale.example", + "is_self_signed": False, + "is_expired": True, + } + ], + "alerts": {"self_signed_count": 0, "expired_count": 1, "high_risk_count": 0}, + } + mapping = ATTACKMapper().map_analysis(tls_analysis=tls_analysis) + ids = {t.technique_id for t in mapping.techniques} + assert "T1573.002" in ids + + def test_clean_certificates_produce_no_techniques(self): + tls_analysis = { + "total_certificates": 1, + "certificates": [ + { + "subject_cn": "clean.example", + "is_self_signed": False, + "is_expired": False, + } + ], + "alerts": {"self_signed_count": 0, "expired_count": 0, "high_risk_count": 0}, + } + mapping = ATTACKMapper().map_analysis(tls_analysis=tls_analysis) + assert mapping.techniques == [] + + def test_missing_certificates_key_does_not_raise(self): + """Older/partial tls_analysis dicts without a certificates key are tolerated.""" + mapping = ATTACKMapper().map_analysis(tls_analysis={"alerts": {}}) + assert mapping.techniques == [] + + +class TestMapAnalysisEndToEnd: + """map_analysis() called the way app/pipeline/runner.py calls it (no yara/osint).""" + + def test_combined_production_shape_inputs_produce_a_mapping(self): + features = { + "flows": [{"dst": "1.2.3.4", "count": 5, "bytes": 20_000_000}], + "artifacts": {"ips": [], "domains": [], "urls": [], "hashes": [], "ja3": []}, + } + dns_analysis = {"alerts": {"dga_count": 2, "tunneling_count": 0, "fast_flux_count": 0}} + tls_analysis = { + "certificates": [{"subject_cn": "c2.example", "is_self_signed": True, "is_expired": False}], + "alerts": {"self_signed_count": 1, "expired_count": 0, "high_risk_count": 0}, + } + beacon_results = [{"dst": "1.2.3.4", "score": 0.85}] + + mapping = ATTACKMapper().map_analysis( + features=features, + dns_analysis=dns_analysis, + tls_analysis=tls_analysis, + beacon_results=beacon_results, + ) + ids = {t.technique_id for t in mapping.techniques} + assert "T1071.001" in ids # beaconing + assert "T1587.003" in ids # self-signed cert + assert any(tid.startswith("T1568") for tid in ids) # DGA + assert mapping.overall_severity in {"low", "medium", "high", "critical"} + d = mapping.to_dict() + assert "techniques" in d and "tactics_summary" in d diff --git a/tests/test_case_management.py b/tests/test_case_management.py index d563f10..fbb0514 100644 --- a/tests/test_case_management.py +++ b/tests/test_case_management.py @@ -404,6 +404,48 @@ def test_get_analysis(self, repo): assert retrieved is not None assert retrieved.packet_count == 500 + def test_save_analysis_round_trips_attack_mapping(self, repo): + """attack_mapping (ATT&CK technique matches) must survive save -> get.""" + case_id = repo.create_case(Case(title="With ATT&CK Mapping")) + attack_mapping = { + "techniques": [ + { + "technique_id": "T1071.001", + "technique_name": "Application Layer Protocol: Web Protocols", + "tactic": "command-and-control", + "confidence": 0.8, + "evidence": ["beacon score 0.91"], + } + ], + "tactics_summary": {"command-and-control": 1}, + "kill_chain_phase": "command-and-control", + "overall_severity": "high", + } + analysis = Analysis( + case_id=case_id, + pcap_path="/test.pcap", + attack_mapping=attack_mapping, + ) + analysis_id = repo.save_analysis(analysis) + + retrieved = repo.get_analysis(analysis_id) + assert retrieved is not None + assert retrieved.attack_mapping == attack_mapping + + # Also verify it round-trips via the case-level fetch path. + restored_case = repo.get_case(case_id) + assert restored_case.analyses[0].attack_mapping == attack_mapping + + def test_get_analysis_defaults_attack_mapping_to_empty_dict(self, repo): + """Old rows / analyses saved without a mapping must not crash on read.""" + case_id = repo.create_case(Case(title="No ATT&CK Mapping")) + analysis = Analysis(case_id=case_id, pcap_path="/test.pcap") + analysis_id = repo.save_analysis(analysis) + + retrieved = repo.get_analysis(analysis_id) + assert retrieved is not None + assert retrieved.attack_mapping == {} + def test_save_analysis_with_iocs(self, repo): """Test saving analysis with IOCs.""" case_id = repo.create_case(Case(title="With IOCs")) diff --git a/tests/test_jobs_schema.py b/tests/test_jobs_schema.py index b9859e5..6c0780e 100644 --- a/tests/test_jobs_schema.py +++ b/tests/test_jobs_schema.py @@ -26,6 +26,16 @@ def test_cases_has_source_column(tmp_path): assert "source" in cols, "cases.source column should exist for ui/api distinction" +def test_analyses_has_attack_json_column(tmp_path): + repo = CaseRepository(db_path=str(tmp_path / "test.db")) + conn = repo._get_conn() + try: + cols = [r[1] for r in conn.execute("PRAGMA table_info(analyses)").fetchall()] + finally: + conn.close() + assert "attack_json" in cols, "analyses.attack_json column should exist for persisted ATT&CK mappings" + + def test_jobs_table_columns(tmp_path): """Verify the jobs table has all expected columns.""" repo = CaseRepository(db_path=str(tmp_path / "test.db")) diff --git a/tests/test_phase4.py b/tests/test_phase4.py index 9672d01..90e8763 100644 --- a/tests/test_phase4.py +++ b/tests/test_phase4.py @@ -146,7 +146,12 @@ def test_dns_tunneling_detection(self): def test_self_signed_cert_detection(self): mapper = ATTACKMapper() - tls_analysis = {"alerts": [{"type": "self_signed", "cert": "test.com"}]} + # Production shape from app.pipeline.tls_certs.analyze_certificates(): per-cert + # boolean flags in "certificates", not a list of {"type": ..., "cert": ...} objects. + tls_analysis = { + "certificates": [{"subject_cn": "test.com", "is_self_signed": True, "is_expired": False}], + "alerts": {"self_signed_count": 1, "expired_count": 0, "high_risk_count": 0}, + } mapping = mapper.map_analysis(tls_analysis=tls_analysis) technique_ids = [t.technique_id for t in mapping.techniques] assert "T1573.002" in technique_ids diff --git a/tests/test_pipeline_runner.py b/tests/test_pipeline_runner.py index dbc28a3..e98070f 100644 --- a/tests/test_pipeline_runner.py +++ b/tests/test_pipeline_runner.py @@ -48,6 +48,12 @@ def test_pipeline_result_to_dict_is_json_serializable(): dns_analysis={"dga_count": 3, "tunneling": []}, tls_analysis={"certs": [{"subject": "evil.example"}]}, beacon_df_records=[{"src": "10.0.0.1", "dst": "1.2.3.4", "score": 0.91}], + attack_mapping={ + "techniques": [{"technique_id": "T1071.001", "technique_name": "Web Protocols"}], + "tactics_summary": {"command-and-control": 1}, + "kill_chain_phase": "command-and-control", + "overall_severity": "high", + }, ) serialized = json.dumps(result.to_dict()) restored = json.loads(serialized) @@ -57,6 +63,8 @@ def test_pipeline_result_to_dict_is_json_serializable(): assert restored["dns_analysis"]["dga_count"] == 3 assert restored["tls_analysis"]["certs"][0]["subject"] == "evil.example" assert restored["beacon_df_records"][0]["src"] == "10.0.0.1" + assert restored["attack_mapping"]["techniques"][0]["technique_id"] == "T1071.001" + assert restored["attack_mapping"]["overall_severity"] == "high" def test_run_pipeline_executes_all_stages_against_fixture(): @@ -108,6 +116,12 @@ def test_run_pipeline_executes_all_stages_against_fixture(): # Some progress events should have fired assert any(e.kind == "phase_start" for e in events) assert any(e.kind == "phase_done" for e in events) + # ATT&CK mapping runs post-fan-out and must always populate a shape, even + # when tiny.pcap yields no technique matches. + assert isinstance(result.attack_mapping, dict) + assert "techniques" in result.attack_mapping + assert isinstance(result.mitre_techniques, list) + assert all(isinstance(t, str) for t in result.mitre_techniques) def test_run_pipeline_skips_disabled_stages(): From f88d2233e9e47ab49ff3f09515ca90dc612f3429 Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Mon, 13 Jul 2026 21:35:21 +0700 Subject: [PATCH 09/35] feat: thread att&ck mapping through ui and api caller paths Populate PCAPResult.attack_mapping from the runner result and store it in session state for single-file, batch, and case-restore/save UI flows; set Analysis.attack_mapping in the API worker's persist step so the queue path saves it too (DB column/round-trip already added in task 2.2). --- app/api/queue.py | 1 + app/main.py | 7 +++++++ app/pipeline/batch.py | 3 +++ app/ui/cases_tab.py | 3 +++ tests/api/test_queue.py | 46 +++++++++++++++++++++++++++++++++++++++++ tests/test_batch.py | 12 +++++++++++ 6 files changed, 72 insertions(+) diff --git a/app/api/queue.py b/app/api/queue.py index 9666a24..6a31346 100644 --- a/app/api/queue.py +++ b/app/api/queue.py @@ -182,6 +182,7 @@ def _persist_analysis( dns_analysis=result.dns_analysis or None, tls_analysis=result.tls_analysis or None, ) + analysis.attack_mapping = result.attack_mapping if result.beacon_df_records: analysis.features["beacon_records"] = result.beacon_df_records analysis.iocs = repo.extract_iocs(analysis) diff --git a/app/main.py b/app/main.py index 79e6900..0f3f0c1 100644 --- a/app/main.py +++ b/app/main.py @@ -226,6 +226,7 @@ def _run_single_pcap_pipeline( beacon_df=beacon_df if isinstance(beacon_df, pd.DataFrame) else None, dns_analysis=result.dns_analysis or {}, tls_analysis=result.tls_analysis or {}, + attack_mapping=result.attack_mapping or {}, packet_count=result.packet_count, ) @@ -468,6 +469,7 @@ def _run_single_pcap_pipeline( "flow_asymmetry": None, "port_anomalies": None, "attack_timeline": [], + "attack_mapping": {}, "__batch_result": None, } ) @@ -617,6 +619,10 @@ def _run_single_pcap_pipeline( st.session_state["beacon_df"] = batch_result.merged_beacons st.session_state["dns_analysis"] = batch_result.aggregated_dns st.session_state["tls_analysis"] = batch_result.aggregated_tls + # No cross-file ATT&CK aggregation yet (batch.py has no merge helper for + # it) — mirror the first successful file's mapping, same fallback used + # for merged_features above. + st.session_state["attack_mapping"] = (first_ok.attack_mapping if first_ok else None) or {} # Carved payloads concatenated across all successful files st.session_state["carved"] = [ item for r in batch_result.pcap_results if not r.error for item in r.carved_items @@ -722,6 +728,7 @@ def _run_single_pcap_pipeline( st.session_state["carved"] = result.carved_items st.session_state["dns_analysis"] = result.dns_analysis or None st.session_state["tls_analysis"] = result.tls_analysis or None + st.session_state["attack_mapping"] = result.attack_mapping or {} _precompute_dash_aggregates(features.get("flows")) # a fresh run supersedes any restored case diff --git a/app/pipeline/batch.py b/app/pipeline/batch.py index d1d2d1c..7cfd98c 100644 --- a/app/pipeline/batch.py +++ b/app/pipeline/batch.py @@ -85,6 +85,9 @@ class PCAPResult: beacon_df: pd.DataFrame | None = None dns_analysis: dict[str, Any] = field(default_factory=dict) tls_analysis: dict[str, Any] = field(default_factory=dict) + # MITRE ATT&CK mapping (AttackMapping.to_dict() shape) from the runner — + # may be {} on mapper failure; treated shape-agnostically downstream. + attack_mapping: dict = field(default_factory=dict) packet_count: int = 0 error: str | None = None diff --git a/app/ui/cases_tab.py b/app/ui/cases_tab.py index c2d5c1f..aad406c 100644 --- a/app/ui/cases_tab.py +++ b/app/ui/cases_tab.py @@ -68,6 +68,7 @@ def _restore_analysis_to_session(analysis: Analysis) -> None: st.session_state["dns_analysis"] = analysis.dns_analysis st.session_state["tls_analysis"] = analysis.tls_analysis st.session_state["yara_results"] = analysis.yara_results + st.session_state["attack_mapping"] = analysis.attack_mapping or {} # Model default for report is "" but the app's no-report sentinel is None. st.session_state["report"] = analysis.report or None # Everything below isn't persisted on Analysis — reset it all, otherwise the @@ -604,6 +605,7 @@ def _quick_save_analysis(): yara_results=st.session_state.get("yara_results"), dns_analysis=st.session_state.get("dns_analysis"), tls_analysis=st.session_state.get("tls_analysis"), + attack_mapping=st.session_state.get("attack_mapping") or {}, ) # Extract IOCs @@ -637,6 +639,7 @@ def _add_current_analysis_to_case(case: Case): yara_results=st.session_state.get("yara_results"), dns_analysis=st.session_state.get("dns_analysis"), tls_analysis=st.session_state.get("tls_analysis"), + attack_mapping=st.session_state.get("attack_mapping") or {}, ) analysis.iocs = repo.extract_iocs(analysis) diff --git a/tests/api/test_queue.py b/tests/api/test_queue.py index cc9985b..cdcbe46 100644 --- a/tests/api/test_queue.py +++ b/tests/api/test_queue.py @@ -303,6 +303,52 @@ def fake_run_pipeline(pcap_path, case_id, options, progress, heartbeat=None): assert persisted.features["beacon_records"] == records +def test_worker_persists_attack_mapping(tmp_path, monkeypatch): + """attack_mapping must round-trip from the runner result into the persisted + Analysis, and mitre_techniques must flow through the job result blob.""" + import app.pipeline.runner as runner_mod + from app.pipeline.runner import PipelineResult + + mapping = { + "techniques": ["T1071.001"], + "tactics": {"command-and-control": ["T1071.001"]}, + } + techniques = ["T1071.001"] + + def fake_run_pipeline(pcap_path, case_id, options, progress, heartbeat=None): + return PipelineResult( + case_id=case_id, + packet_count=1, + features={ + "flows": [{"src": "10.0.0.1", "dst": "8.8.8.8", "proto": "TCP", "count": 9}], + "artifacts": {"ips": ["10.0.0.1", "8.8.8.8"], "domains": [], "urls": [], "hashes": [], "ja3": []}, + }, + attack_mapping=dict(mapping), + mitre_techniques=list(techniques), + ) + + # _worker_run imports run_pipeline from app.pipeline.runner at call time, + # so the patch must target the source module, not queue_mod. + monkeypatch.setattr(runner_mod, "run_pipeline", fake_run_pipeline) + + fake_pcap = tmp_path / "fake.pcap" + fake_pcap.write_bytes(b"\xd4\xc3\xb2\xa1" + b"\x00" * 20) + + db = str(tmp_path / "t.db") + repo = CaseRepository(db_path=db) + repo.create_case(Case(id="cafe0030", title="t", status=CaseStatus.IN_PROGRESS, severity=Severity.LOW)) + job_id = repo.create_job(Job(case_id="cafe0030", pcap_path=str(fake_pcap), options_json="{}")) + + _worker_run(job_id, db, str(fake_pcap), {"osint_enabled": False, "llm_enabled": False}) + + result = json.loads(repo.get_job(job_id).result_json) + assert result["mitre_techniques"] == techniques, "mitre_techniques must flow into the job result blob" + assert result["analysis_id"], "persistence must succeed with the faked pipeline result" + + persisted = repo.get_analysis(result["analysis_id"]) + assert persisted.attack_mapping == mapping, "attack_mapping must be persisted on the Analysis row" + + # --------------------------------------------------------------------------- # Task 3: progress reconciliation on completion # --------------------------------------------------------------------------- diff --git a/tests/test_batch.py b/tests/test_batch.py index 8d3fc3b..315a477 100644 --- a/tests/test_batch.py +++ b/tests/test_batch.py @@ -162,6 +162,18 @@ def test_carved_items_accepts_records(self): result = PCAPResult(path="/data/test.pcap", filename="test.pcap", carved_items=carved) assert result.carved_items == carved + def test_attack_mapping_defaults_empty(self): + result = PCAPResult(path="/data/test.pcap", filename="test.pcap") + assert result.attack_mapping == {} + + def test_attack_mapping_accepts_mapping(self): + mapping = { + "techniques": ["T1071.001"], + "tactics": {"command-and-control": ["T1071.001"]}, + } + result = PCAPResult(path="/data/test.pcap", filename="test.pcap", attack_mapping=mapping) + assert result.attack_mapping == mapping + class TestMergeZeekTables: """Test merging Zeek tables from multiple PCAPs.""" From 3918fa1e4a5a3eb806982aac4e51b0450bcb95b7 Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Mon, 13 Jul 2026 21:59:12 +0700 Subject: [PATCH 10/35] feat: render att&ck mapping in dashboard via from_dict reconstruction session_state["attack_mapping"] holds the dict form (AttackMapping.to_dict()) but render_attack_mapping expects the object form. Add from_dict classmethods on AttackMapping/TechniqueMatch (empty/partial dict safe) to reconstruct before rendering, wire the call into the Dashboard tab between the correlation and hunting-checklist sections, and seed attack_mapping in the boot-time session-state defaults for symmetry with the click-reset block. --- app/main.py | 12 +++++++ app/threat_intel/attack_mapping.py | 33 ++++++++++++++++++ tests/test_layout.py | 54 ++++++++++++++++++++++++++++++ tests/test_phase4.py | 45 +++++++++++++++++++++++++ 4 files changed, 144 insertions(+) diff --git a/app/main.py b/app/main.py index 0f3f0c1..744a338 100644 --- a/app/main.py +++ b/app/main.py @@ -46,6 +46,7 @@ make_results_panel, make_tabs, render_active_filters, + render_attack_mapping, render_batch_summary, render_carved, render_chart_hint, @@ -330,6 +331,7 @@ def _run_single_pcap_pipeline( ("correlations", None), ("flow_asymmetry", None), ("port_anomalies", None), + ("attack_mapping", {}), ("__pcap_paths", []), ("__batch_mode", False), ("__batch_result", None), @@ -1392,6 +1394,16 @@ def _run_single_pcap_pipeline( st.markdown("---") + # MITRE ATT&CK Mapping (session state holds the dict form; reconstruct + # the AttackMapping object the renderer expects) + _attack = st.session_state.get("attack_mapping") + if _attack: + from app.threat_intel import AttackMapping + + render_attack_mapping(st.container(), AttackMapping.from_dict(_attack)) + + st.markdown("---") + # Hunting checklist render_hunting_checklist( st.container(), diff --git a/app/threat_intel/attack_mapping.py b/app/threat_intel/attack_mapping.py index 268e914..437a1a7 100644 --- a/app/threat_intel/attack_mapping.py +++ b/app/threat_intel/attack_mapping.py @@ -55,6 +55,22 @@ def to_dict(self) -> dict: "evidence": self.evidence, } + @classmethod + def from_dict(cls, d: dict) -> TechniqueMatch: + """Reconstruct a ``TechniqueMatch`` from its ``to_dict()`` form. + + Missing keys fall back to safe defaults so a partial/malformed dict + (e.g. from a stale session-state entry) does not raise. + """ + d = d or {} + return cls( + technique_id=d.get("technique_id", ""), + technique_name=d.get("technique_name", ""), + tactic=d.get("tactic", ""), + confidence=d.get("confidence", 0.0), + evidence=list(d.get("evidence") or []), + ) + @dataclass class AttackMapping: @@ -74,6 +90,23 @@ def to_dict(self) -> dict: "overall_severity": self.overall_severity, } + @classmethod + def from_dict(cls, d: dict) -> AttackMapping: + """Reconstruct an ``AttackMapping`` from its ``to_dict()`` form. + + Handles an empty or partial dict gracefully — the mapper-failure + default stored in ``st.session_state["attack_mapping"]`` is ``{}``, + and ``from_dict({})`` must return a valid empty ``AttackMapping()`` + rather than raising. + """ + d = d or {} + return cls( + techniques=[TechniqueMatch.from_dict(t) for t in d.get("techniques") or []], + tactics_summary=dict(d.get("tactics_summary") or {}), + kill_chain_phase=d.get("kill_chain_phase", "unknown"), + overall_severity=d.get("overall_severity", "low"), + ) + # Kill chain phases in order of advancement KILL_CHAIN_ORDER = [ diff --git a/tests/test_layout.py b/tests/test_layout.py index decd501..962463b 100644 --- a/tests/test_layout.py +++ b/tests/test_layout.py @@ -2,11 +2,65 @@ from __future__ import annotations +from streamlit.testing.v1 import AppTest + from app.ui.layout import resolve_logo_path LIGHT = "logo-256.png" DARK = "logo-dark-256.png" +# Production-shape dict, as stored in st.session_state["attack_mapping"] +# (AttackMapping.to_dict() output from the ATT&CK mapping pipeline stage). +PROD_ATTACK_MAPPING_DICT = { + "techniques": [ + { + "technique_id": "T1071.001", + "technique_name": "Application Layer Protocol: Web Protocols", + "tactic": "command-and-control", + "confidence": 0.85, + "evidence": ["Beaconing detected with score 0.85 to 203.0.113.5"], + }, + { + "technique_id": "T1568.002", + "technique_name": "Dynamic Resolution: Domain Generation Algorithms", + "tactic": "command-and-control", + "confidence": 0.6, + "evidence": ["DGA domains detected: xk3jd9a.example.com"], + }, + ], + "tactics_summary": {"command-and-control": 2}, + "kill_chain_phase": "command-and-control", + "overall_severity": "high", +} + + +def _render_attack_mapping_app(): + import streamlit as st + + from app.threat_intel import AttackMapping + from app.ui.layout import render_attack_mapping + + attack_mapping = st.session_state.get("attack_mapping", {}) + render_attack_mapping(st.container(), AttackMapping.from_dict(attack_mapping)) + + +class TestRenderAttackMappingFromDict: + def test_renders_without_exception_and_shows_mitre_attck(self): + at = AppTest.from_function(_render_attack_mapping_app, default_timeout=30) + at.session_state["attack_mapping"] = PROD_ATTACK_MAPPING_DICT + at.run() + + assert not at.exception + expanders = [e for e in at.expander if "MITRE ATT&CK" in (e.label or "")] + assert expanders, "Dashboard should render a MITRE ATT&CK Mapping expander" + + def test_empty_dict_renders_without_exception(self): + at = AppTest.from_function(_render_attack_mapping_app, default_timeout=30) + at.session_state["attack_mapping"] = {} + at.run() + + assert not at.exception + def _make_assets(tmp_path, *names): for name in names: diff --git a/tests/test_phase4.py b/tests/test_phase4.py index 90e8763..7b2a460 100644 --- a/tests/test_phase4.py +++ b/tests/test_phase4.py @@ -74,6 +74,17 @@ def test_to_dict(self): assert d["confidence"] == 0.7 assert len(d["evidence"]) == 2 + def test_from_dict_round_trip(self): + tech = TechniqueMatch( + technique_id="T1071.001", + technique_name="Application Layer Protocol: Web Protocols", + tactic="command-and-control", + confidence=0.85, + evidence=["HTTP beaconing detected", "second signal"], + ) + rebuilt = TechniqueMatch.from_dict(tech.to_dict()) + assert rebuilt == tech + class TestAttackMapping: """Test AttackMapping dataclass.""" @@ -114,6 +125,40 @@ def test_to_dict(self): assert d["kill_chain_phase"] == "command-and-control" assert d["overall_severity"] == "high" + def test_from_dict_round_trip(self): + mapping = AttackMapping( + techniques=[ + TechniqueMatch("T1071", "Test1", "command-and-control", 0.8, ["e1"]), + TechniqueMatch("T1095", "Test2", "command-and-control", 0.7, []), + TechniqueMatch("T1041", "Test3", "exfiltration", 0.6, ["e2", "e3"]), + ], + tactics_summary={"command-and-control": 2, "exfiltration": 1}, + kill_chain_phase="exfiltration", + overall_severity="critical", + ) + rebuilt = AttackMapping.from_dict(mapping.to_dict()) + assert rebuilt == mapping + assert [t.technique_id for t in rebuilt.techniques] == [t.technique_id for t in mapping.techniques] + assert rebuilt.tactics_summary == mapping.tactics_summary + assert rebuilt.kill_chain_phase == mapping.kill_chain_phase + assert rebuilt.overall_severity == mapping.overall_severity + + def test_from_dict_empty_dict_returns_valid_empty_mapping(self): + mapping = AttackMapping.from_dict({}) + assert mapping == AttackMapping() + assert mapping.techniques == [] + assert mapping.tactics_summary == {} + assert mapping.kill_chain_phase == "unknown" + assert mapping.overall_severity == "low" + + def test_from_dict_none_like_partial_dict_does_not_crash(self): + # Mapper-failure default is {}; also guard a partial dict missing keys. + mapping = AttackMapping.from_dict({"kill_chain_phase": "execution"}) + assert mapping.techniques == [] + assert mapping.tactics_summary == {} + assert mapping.kill_chain_phase == "execution" + assert mapping.overall_severity == "low" + class TestATTACKMapper: """Test ATTACKMapper class.""" From 38da5b398970e494233e8c335ff845f4358fb2ed Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Mon, 13 Jul 2026 22:16:45 +0700 Subject: [PATCH 11/35] feat: add mitre att&ck section to pdf report --- app/main.py | 1 + app/reports/pdf_generator.py | 76 +++++++++++++++++++++++++++++++++++ tests/test_pdf_integration.py | 41 +++++++++++++++++++ 3 files changed, 118 insertions(+) diff --git a/app/main.py b/app/main.py index 744a338..905640a 100644 --- a/app/main.py +++ b/app/main.py @@ -1487,6 +1487,7 @@ def _run_single_pcap_pipeline( correlations=st.session_state.get("correlations"), geoip_data=_geoip_data or None, attack_timeline=st.session_state.get("attack_timeline"), + attack_mapping=st.session_state.get("attack_mapping"), ) if pdf_report: diff --git a/app/reports/pdf_generator.py b/app/reports/pdf_generator.py index 5dab5d2..0f594d1 100644 --- a/app/reports/pdf_generator.py +++ b/app/reports/pdf_generator.py @@ -124,6 +124,7 @@ class PDFReportGenerator: ("summary", "Executive Summary"), ("charts", "Visual Overview"), ("correlations", "Threat Correlation Summary"), + ("attack", "MITRE ATT&CK Mapping"), ("iocs", "Indicators of Compromise"), ("osint", "OSINT Analysis"), ("dns", "DNS Analysis"), @@ -182,6 +183,7 @@ def generate( correlations: list | None = None, geoip_data: list | None = None, attack_timeline: list | None = None, + attack_mapping: dict | None = None, ) -> PDFReport | None: """ Generate a complete PDF report. @@ -196,6 +198,8 @@ def generate( case_info: Optional case information. beacon_df: Optional beacon scoring DataFrame. correlations: Optional list of correlation results. + attack_mapping: Optional ``AttackMapping.to_dict()`` shape (or an + ``AttackMapping`` instance) from ``st.session_state["attack_mapping"]``. Returns: PDFReport object or None if generation fails. @@ -218,6 +222,7 @@ def generate( correlations=correlations, geoip_data=geoip_data, attack_timeline=attack_timeline, + attack_mapping=attack_mapping, ) # Generate PDF with accurate page count @@ -261,6 +266,7 @@ def _build_html( correlations: list | None = None, geoip_data: list | None = None, attack_timeline: list | None = None, + attack_mapping: dict | None = None, ) -> str: """Build the complete HTML document.""" # --- Section assembly: render first, number after (see _h2). --- @@ -287,6 +293,10 @@ def _build_html( if correlations: candidates.append(("correlations", self._render_correlation_section(correlations))) + # MITRE ATT&CK Mapping + if attack_mapping: + candidates.append(("attack", self._render_attack_section(attack_mapping))) + # Key Findings / IOC Summary candidates.append(("iocs", self._render_ioc_table(features, osint))) @@ -943,6 +953,68 @@ def _render_correlation_section(self, correlations: list) -> str: {detail_section}
+""" + + def _render_attack_section(self, attack_mapping) -> str: + """Render the MITRE ATT&CK technique mapping section. + + Accepts either an ``AttackMapping`` dataclass instance or its + ``to_dict()`` form — the shape persisted in + ``st.session_state["attack_mapping"]`` — and normalizes dicts via + ``AttackMapping.from_dict()`` so the rest of this method always works + with dataclass attributes. Returns "" (dropping the section) when + there are no detected techniques. + """ + from app.threat_intel.attack_mapping import AttackMapping + + if not attack_mapping: + return "" + + mapping = AttackMapping.from_dict(attack_mapping) if isinstance(attack_mapping, dict) else attack_mapping + + if not mapping.techniques: + return "" + + tactics_summary = mapping.tactics_summary or {} + tactics_text = ( + ", ".join( + f"{self._escape(tactic)} ({count})" + for tactic, count in sorted(tactics_summary.items(), key=lambda kv: -kv[1]) + ) + if tactics_summary + else "N/A" + ) + + technique_rows = [] + for tech in mapping.techniques: + evidence = "; ".join(tech.evidence[:3]) if tech.evidence else "" + technique_rows.append( + f"{self._escape(tech.technique_id)}" + f"{self._escape(tech.technique_name)}" + f"{self._escape(tech.tactic)}" + f"{tech.confidence:.0%}" + f"{self._escape(evidence)}" + ) + technique_table = "\n".join(technique_rows) + + return f""" +
+ {self._h2("attack")} + + + + + + + +
MetricValue
Kill Chain Phase{self._escape(mapping.kill_chain_phase)}
Overall Severity{self._escape(str(mapping.overall_severity).upper())}
Tactics Detected{tactics_text}
+

Detected Techniques

+ + + {technique_table} +
IDNameTacticConfidenceEvidence
+
+
""" def _render_beacon_section(self, beacon_df: "pd.DataFrame") -> str: @@ -1396,6 +1468,7 @@ def generate_pdf_report( correlations: list | None = None, geoip_data: list | None = None, attack_timeline: list | None = None, + attack_mapping: dict | None = None, ) -> PDFReport | None: """ Convenience function to generate a PDF report. @@ -1410,6 +1483,8 @@ def generate_pdf_report( config: Optional report configuration. beacon_df: Optional beacon scoring DataFrame. correlations: Optional list of correlation results. + attack_mapping: Optional ``AttackMapping.to_dict()`` shape (or an + ``AttackMapping`` instance) from ``st.session_state["attack_mapping"]``. Returns: PDFReport object or None. @@ -1426,4 +1501,5 @@ def generate_pdf_report( correlations=correlations, geoip_data=geoip_data, attack_timeline=attack_timeline, + attack_mapping=attack_mapping, ) diff --git a/tests/test_pdf_integration.py b/tests/test_pdf_integration.py index fdfb472..185ea3e 100644 --- a/tests/test_pdf_integration.py +++ b/tests/test_pdf_integration.py @@ -21,6 +21,7 @@ from app.analysis.correlation import CorrelationResult, CorrelationSignal from app.reports.pdf_generator import WEASYPRINT_AVAILABLE, PDFReportGenerator, ReportConfig +from app.threat_intel.attack_mapping import AttackMapping, TechniqueMatch # --------------------------------------------------------------------------- # Realistic fixtures — shaped exactly like live-pipeline output @@ -215,6 +216,35 @@ def realistic_geoip_data() -> list[dict]: ] +@pytest.fixture +def realistic_attack_mapping() -> dict: + """attack_mapping dict shape matching AttackMapping.to_dict() — the form + persisted in st.session_state["attack_mapping"] and passed to the PDF + generator, not a simplified dict with similar-looking keys.""" + mapping = AttackMapping( + techniques=[ + TechniqueMatch( + technique_id="T1071.001", + technique_name="Application Layer Protocol: Web Protocols", + tactic="command-and-control", + confidence=0.85, + evidence=["Beacon to 34.12.37.224 over HTTPS at regular intervals"], + ), + TechniqueMatch( + technique_id="T1568.002", + technique_name="Dynamic Resolution: Domain Generation Algorithms", + tactic="command-and-control", + confidence=0.92, + evidence=["DGA domain malicious.dga.xyz detected"], + ), + ], + tactics_summary={"command-and-control": 2}, + kill_chain_phase="command-and-control", + overall_severity="high", + ) + return mapping.to_dict() + + @pytest.fixture def full_report_html( realistic_features, @@ -225,6 +255,7 @@ def full_report_html( realistic_tls_analysis, realistic_yara_results, realistic_geoip_data, + realistic_attack_mapping, ) -> str: """Complete report HTML built from the production-shape fixtures above. @@ -245,6 +276,7 @@ def full_report_html( beacon_df=realistic_beacon_df, correlations=realistic_correlations, geoip_data=realistic_geoip_data, + attack_mapping=realistic_attack_mapping, ) @@ -267,6 +299,7 @@ def test_full_pdf_with_all_sections( realistic_tls_analysis, realistic_yara_results, realistic_geoip_data, + realistic_attack_mapping, ): """Full report with every section populated and every data type realistic. @@ -297,6 +330,7 @@ def test_full_pdf_with_all_sections( beacon_df=realistic_beacon_df, correlations=realistic_correlations, geoip_data=realistic_geoip_data, + attack_mapping=realistic_attack_mapping, ) assert pdf is not None, "PDF generation returned None — see error logs" @@ -316,6 +350,7 @@ def test_html_contains_every_expected_section( realistic_tls_analysis, realistic_yara_results, realistic_geoip_data, + realistic_attack_mapping, ): """Verify section IDs and key tokens appear in the generated HTML.""" gen = PDFReportGenerator(ReportConfig(title="Section Coverage Test")) @@ -330,12 +365,14 @@ def test_html_contains_every_expected_section( beacon_df=realistic_beacon_df, correlations=realistic_correlations, geoip_data=realistic_geoip_data, + attack_mapping=realistic_attack_mapping, ) # Section anchors assert 'id="summary"' in html assert 'id="charts"' in html assert 'id="correlations"' in html + assert 'id="attack"' in html assert 'id="iocs"' in html assert 'id="osint"' in html assert 'id="dns"' in html @@ -359,6 +396,10 @@ def test_html_contains_every_expected_section( # YARA match rendering assert "CobaltStrike_Beacon" in html + # ATT&CK technique table rendering + assert "T1071.001" in html + assert "command-and-control" in html + def test_pdf_with_charts_embeds_png_data_uris( self, realistic_features, From c5dd19e7224e06319c49c73eb38d5040eec76ac2 Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Mon, 13 Jul 2026 22:33:50 +0700 Subject: [PATCH 12/35] feat: derive feed mitre_techniques from analysis technique ids New analysis_techniques table stores per-analysis MITRE ATT&CK technique IDs (full-replace on re-save, like attack_json). The IOC feed's query_iocs now LEFT JOINs it and aggregates via GROUP_CONCAT(DISTINCT ...), replacing the always-empty mitre_techniques field. delete_case and clear_all cascade the new table; queue.py's _persist_analysis sets Analysis.mitre_techniques from the pipeline result before saving. --- app/api/feed.py | 6 ++- app/api/queue.py | 1 + app/database/models.py | 3 ++ app/database/repository.py | 32 ++++++++++++++++ tests/api/test_feed_query.py | 70 +++++++++++++++++++++++++++++++++++ tests/api/test_iocs.py | 49 ++++++++++++++++++++++++ tests/test_case_management.py | 66 ++++++++++++++++++++++++++++++++- tests/test_jobs_schema.py | 23 ++++++++++++ 8 files changed, 247 insertions(+), 3 deletions(-) diff --git a/app/api/feed.py b/app/api/feed.py index 63bd4e8..3e17b06 100644 --- a/app/api/feed.py +++ b/app/api/feed.py @@ -48,11 +48,13 @@ def query_iocs(repo: CaseRepository, filt: IOCFilter) -> list[dict[str, Any]]: MIN(a.analyzed_at) AS first_seen, MAX(a.analyzed_at) AS last_seen, GROUP_CONCAT(DISTINCT a.case_id) AS case_ids, - GROUP_CONCAT(DISTINCT t.name) AS tag_names + GROUP_CONCAT(DISTINCT t.name) AS tag_names, + GROUP_CONCAT(DISTINCT at.technique_id) AS technique_ids FROM iocs i JOIN analyses a ON i.analysis_id = a.id LEFT JOIN case_tags ct ON ct.case_id = a.case_id LEFT JOIN tags t ON t.id = ct.tag_id + LEFT JOIN analysis_techniques at ON at.analysis_id = i.analysis_id """ where: list[str] = [] params: list[Any] = [] @@ -102,7 +104,7 @@ def query_iocs(repo: CaseRepository, filt: IOCFilter) -> list[dict[str, Any]]: "first_seen": d["first_seen"], "last_seen": d["last_seen"], "case_ids": [c for c in (d.get("case_ids") or "").split(",") if c], - "mitre_techniques": [], # Future: derive from analysis features + "mitre_techniques": sorted({t for t in (d.get("technique_ids") or "").split(",") if t}), } ) return out diff --git a/app/api/queue.py b/app/api/queue.py index 6a31346..bdfcaab 100644 --- a/app/api/queue.py +++ b/app/api/queue.py @@ -183,6 +183,7 @@ def _persist_analysis( tls_analysis=result.tls_analysis or None, ) analysis.attack_mapping = result.attack_mapping + analysis.mitre_techniques = result.mitre_techniques if result.beacon_df_records: analysis.features["beacon_records"] = result.beacon_df_records analysis.iocs = repo.extract_iocs(analysis) diff --git a/app/database/models.py b/app/database/models.py index e77f8a2..4445ec5 100644 --- a/app/database/models.py +++ b/app/database/models.py @@ -157,6 +157,7 @@ class Analysis: dns_analysis: dict | None = None tls_analysis: dict | None = None attack_mapping: dict = field(default_factory=dict) + mitre_techniques: list[str] = field(default_factory=list) iocs: list[IOC] = field(default_factory=list) def to_dict(self) -> dict: @@ -174,6 +175,7 @@ def to_dict(self) -> dict: "dns_analysis": self.dns_analysis, "tls_analysis": self.tls_analysis, "attack_mapping": self.attack_mapping, + "mitre_techniques": list(self.mitre_techniques), "iocs": [ioc.to_dict() for ioc in self.iocs], } @@ -199,6 +201,7 @@ def from_dict(cls, data: dict) -> "Analysis": dns_analysis=data.get("dns_analysis"), tls_analysis=data.get("tls_analysis"), attack_mapping=data.get("attack_mapping", {}), + mitre_techniques=data.get("mitre_techniques", []), iocs=iocs, ) diff --git a/app/database/repository.py b/app/database/repository.py index 8b1e625..7262050 100644 --- a/app/database/repository.py +++ b/app/database/repository.py @@ -102,6 +102,14 @@ def _init_schema(self): updated_at TIMESTAMP ); + -- MITRE ATT&CK technique IDs derived per analysis (feeds the IOC feed's + -- mitre_techniques aggregation without post-SQL Python filtering). + CREATE TABLE IF NOT EXISTS analysis_techniques ( + analysis_id TEXT REFERENCES analyses(id) ON DELETE CASCADE, + technique_id TEXT NOT NULL, + UNIQUE(analysis_id, technique_id) + ); + -- Tags for organization CREATE TABLE IF NOT EXISTS tags ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -137,6 +145,7 @@ def _init_schema(self): CREATE INDEX IF NOT EXISTS idx_analyses_case ON analyses(case_id); CREATE INDEX IF NOT EXISTS idx_iocs_analysis ON iocs(analysis_id); CREATE INDEX IF NOT EXISTS idx_iocs_type_value ON iocs(ioc_type, value); + CREATE INDEX IF NOT EXISTS idx_analysis_techniques ON analysis_techniques(analysis_id); CREATE INDEX IF NOT EXISTS idx_notes_case ON notes(case_id); CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status); CREATE INDEX IF NOT EXISTS idx_jobs_case ON jobs(case_id); @@ -354,6 +363,10 @@ def delete_case(self, case_id: str) -> bool: "DELETE FROM iocs WHERE analysis_id IN (SELECT id FROM analyses WHERE case_id = ?)", (case_id,), ) + conn.execute( + "DELETE FROM analysis_techniques WHERE analysis_id IN (SELECT id FROM analyses WHERE case_id = ?)", + (case_id,), + ) conn.execute( "DELETE FROM notes WHERE case_id = ? OR analysis_id IN (SELECT id FROM analyses WHERE case_id = ?)", (case_id, case_id), @@ -382,6 +395,7 @@ def clear_all(self) -> bool: try: conn.execute("DELETE FROM notes") conn.execute("DELETE FROM iocs") + conn.execute("DELETE FROM analysis_techniques") conn.execute("DELETE FROM analyses") conn.execute("DELETE FROM case_tags") conn.execute("DELETE FROM tags") @@ -449,6 +463,17 @@ def save_analysis(self, analysis: Analysis) -> str: for ioc in analysis.iocs: self._save_ioc(conn, analysis.id, ioc) + # Save MITRE ATT&CK technique associations. Full replace (delete then + # insert) rather than accumulate: unlike iocs, a re-analysis's technique + # set should reflect only the most recent save, mirroring the + # full-replace semantics of attack_json/features_json on this row. + conn.execute("DELETE FROM analysis_techniques WHERE analysis_id = ?", (analysis.id,)) + for technique_id in analysis.mitre_techniques: + conn.execute( + "INSERT OR IGNORE INTO analysis_techniques (analysis_id, technique_id) VALUES (?, ?)", + (analysis.id, technique_id), + ) + conn.commit() logger.info("Saved analysis: %s", analysis.id) return analysis.id @@ -734,6 +759,12 @@ def _row_to_analysis(self, row: dict, conn: sqlite3.Connection) -> Analysis: for r in ioc_rows ] + # Load MITRE ATT&CK technique IDs + technique_rows = conn.execute( + "SELECT technique_id FROM analysis_techniques WHERE analysis_id = ?", (row["id"],) + ).fetchall() + mitre_techniques = [r["technique_id"] for r in technique_rows] + return Analysis( id=row["id"], case_id=row.get("case_id") or "", @@ -748,6 +779,7 @@ def _row_to_analysis(self, row: dict, conn: sqlite3.Connection) -> Analysis: dns_analysis=dns_analysis, tls_analysis=tls_analysis, attack_mapping=attack_mapping, + mitre_techniques=mitre_techniques, iocs=iocs, ) diff --git a/tests/api/test_feed_query.py b/tests/api/test_feed_query.py index 3f0ed9b..f4a9cca 100644 --- a/tests/api/test_feed_query.py +++ b/tests/api/test_feed_query.py @@ -109,3 +109,73 @@ def test_duplicate_ioc_takes_max_severity_score(tmp_path): row = next(r for r in rows if r["value"] == "198.51.100.7") assert row["score"] == 100 assert row["severity"] == "critical" + + +def test_query_iocs_derives_mitre_techniques(tmp_path): + """mitre_techniques must be derived from the analysis's technique IDs, not hardcoded [].""" + repo = CaseRepository(db_path=str(tmp_path / "t.db")) + repo.create_case(Case(id="case9001", title="t")) + analysis = Analysis( + case_id="case9001", + pcap_path="/tmp/x.pcap", + mitre_techniques=["T1071.001"], + iocs=[IOC(ioc_type=IOCType.IP, value="9.9.9.9", severity=Severity.HIGH)], + ) + repo.save_analysis(analysis) + + rows = query_iocs(repo, IOCFilter()) + row = next(r for r in rows if r["value"] == "9.9.9.9") + assert row["mitre_techniques"] == ["T1071.001"] + + +def test_query_iocs_no_techniques_returns_empty_list(tmp_path): + """An analysis with no mapped techniques must yield [], not crash the LEFT JOIN.""" + repo = _seed(tmp_path) + rows = query_iocs(repo, IOCFilter()) + for r in rows: + assert r["mitre_techniques"] == [] + + +def test_query_iocs_multi_technique_multi_tag_no_duplication(tmp_path): + """Fan-out guard: the LEFT JOIN analysis_techniques multiplies result rows + (one per technique), which could silently corrupt any non-DISTINCT aggregate. + Seed multiple techniques AND multiple tags on the same IOC's analysis and + confirm both tags and technique IDs come back deduped, not repeated.""" + repo = CaseRepository(db_path=str(tmp_path / "t.db")) + repo.create_case(Case(id="case9002", title="t", tags=["tag-a", "tag-b"])) + analysis = Analysis( + case_id="case9002", + pcap_path="/tmp/y.pcap", + mitre_techniques=["T1071.001", "T1059"], + iocs=[IOC(ioc_type=IOCType.IP, value="8.8.8.8", severity=Severity.HIGH)], + ) + repo.save_analysis(analysis) + + rows = query_iocs(repo, IOCFilter()) + row = next(r for r in rows if r["value"] == "8.8.8.8") + assert row["mitre_techniques"] == ["T1059", "T1071.001"] # sorted + assert sorted(row["tags"]) == ["tag-a", "tag-b"] + + +def test_query_iocs_techniques_dont_affect_pagination(tmp_path): + """LIMIT/OFFSET must still page over distinct (ioc_type, value) groups, not + over the post-join fan-out rows, when techniques are present.""" + repo = CaseRepository(db_path=str(tmp_path / "t.db")) + repo.create_case(Case(id="case9003", title="t", status=CaseStatus.IN_PROGRESS, severity=Severity.LOW)) + analysis = Analysis( + case_id="case9003", + pcap_path="x.pcap", + features={"artifacts": {}}, + mitre_techniques=["T1071.001", "T1059", "T1105"], + ) + analysis.iocs = [ + IOC(ioc_type=IOCType.IP, value="203.0.113.10", severity=Severity.HIGH), + IOC(ioc_type=IOCType.IP, value="203.0.113.11", severity=Severity.HIGH), + ] + repo.save_analysis(analysis) + + page1 = query_iocs(repo, IOCFilter(limit=1, offset=0)) + page2 = query_iocs(repo, IOCFilter(limit=1, offset=1)) + assert len(page1) == 1 + assert len(page2) == 1 + assert page1[0]["value"] != page2[0]["value"] diff --git a/tests/api/test_iocs.py b/tests/api/test_iocs.py index 034ad4e..26b6247 100644 --- a/tests/api/test_iocs.py +++ b/tests/api/test_iocs.py @@ -68,6 +68,55 @@ def test_iocs_json_filter_by_type(client): assert all(i["type"] == "ip" for i in r.json()["iocs"]) +# ── mitre_techniques derivation ───────────────────────────────────────────── + + +def test_iocs_json_includes_mitre_techniques(client): + """mitre_techniques must be derived from the analysis, not the hardcoded [].""" + from app.api.deps import get_repo + + repo = get_repo() + case = Case(id="case0007", title="mitre") + repo.create_case(case) + repo.save_analysis( + Analysis( + case_id=case.id, + pcap_path="/tmp/m.pcap", + mitre_techniques=["T1071.001"], + iocs=[IOC(ioc_type=IOCType.IP, value="9.9.9.9", severity=Severity.HIGH)], + ) + ) + + r = client.get("/api/v1/iocs.json", headers={"Authorization": "Bearer FEED"}) + assert r.status_code == 200 + row = next(i for i in r.json()["iocs"] if i["value"] == "9.9.9.9") + assert row["mitre_techniques"] == ["T1071.001"] + + +def test_iocs_json_multi_technique_multi_tag_no_tag_duplication(client): + """Fan-out guard at the API layer: multiple techniques + multiple tags on the + same IOC's analysis must not duplicate tags or technique IDs in the response.""" + from app.api.deps import get_repo + + repo = get_repo() + case = Case(id="case0008", title="multi", tags=["tag-x", "tag-y"]) + repo.create_case(case) + repo.save_analysis( + Analysis( + case_id=case.id, + pcap_path="/tmp/m2.pcap", + mitre_techniques=["T1059", "T1071.001"], + iocs=[IOC(ioc_type=IOCType.IP, value="7.7.7.7", severity=Severity.HIGH)], + ) + ) + + r = client.get("/api/v1/iocs.json", headers={"Authorization": "Bearer FEED"}) + assert r.status_code == 200 + row = next(i for i in r.json()["iocs"] if i["value"] == "7.7.7.7") + assert sorted(row["mitre_techniques"]) == ["T1059", "T1071.001"] + assert sorted(row["tags"]) == ["tag-x", "tag-y"] + + # ── ETag / 304 ────────────────────────────────────────────────────────────── diff --git a/tests/test_case_management.py b/tests/test_case_management.py index fbb0514..da768e3 100644 --- a/tests/test_case_management.py +++ b/tests/test_case_management.py @@ -148,6 +148,7 @@ def test_default_values(self): assert analysis.packet_count == 0 assert analysis.features == {} assert analysis.iocs == [] + assert analysis.mitre_techniques == [] def test_to_dict(self): analysis = Analysis( @@ -373,6 +374,19 @@ def test_clear_all_reaps_jobs(self, repo): finally: conn.close() + def test_clear_all_removes_analysis_techniques(self, repo): + """clear_all must delete analysis_techniques rows along with the case data.""" + case_id = repo.create_case(Case(title="With Techniques")) + repo.save_analysis(Analysis(case_id=case_id, pcap_path="/p.pcap", mitre_techniques=["T1071.001"])) + + assert repo.clear_all() is True + + conn = repo._get_conn() + try: + assert conn.execute("SELECT COUNT(*) FROM analysis_techniques").fetchone()[0] == 0 + finally: + conn.close() + def test_save_analysis_to_case(self, repo): """Test saving analysis to case.""" case_id = repo.create_case(Case(title="With Analysis")) @@ -446,6 +460,49 @@ def test_get_analysis_defaults_attack_mapping_to_empty_dict(self, repo): assert retrieved is not None assert retrieved.attack_mapping == {} + def test_save_analysis_round_trips_mitre_techniques(self, repo): + """mitre_techniques (technique IDs) must survive save -> get, dedup by table UNIQUE.""" + case_id = repo.create_case(Case(title="With MITRE Techniques")) + analysis = Analysis( + case_id=case_id, + pcap_path="/test.pcap", + mitre_techniques=["T1071.001", "T1059"], + ) + analysis_id = repo.save_analysis(analysis) + + retrieved = repo.get_analysis(analysis_id) + assert retrieved is not None + assert sorted(retrieved.mitre_techniques) == ["T1059", "T1071.001"] + + # Also verify it round-trips via the case-level fetch path. + restored_case = repo.get_case(case_id) + assert sorted(restored_case.analyses[0].mitre_techniques) == ["T1059", "T1071.001"] + + def test_get_analysis_defaults_mitre_techniques_to_empty_list(self, repo): + """Analyses saved without techniques must not crash on read.""" + case_id = repo.create_case(Case(title="No MITRE Techniques")) + analysis = Analysis(case_id=case_id, pcap_path="/test.pcap") + analysis_id = repo.save_analysis(analysis) + + retrieved = repo.get_analysis(analysis_id) + assert retrieved is not None + assert retrieved.mitre_techniques == [] + + def test_save_analysis_resaved_mitre_techniques_replaces_not_accumulates(self, repo): + """Re-saving the same analysis_id with a different technique list must fully + replace the prior set, not accumulate across saves (mirrors full-replace + semantics of attack_json/features_json, unlike the accumulating iocs table).""" + case_id = repo.create_case(Case(title="Resaved Techniques")) + analysis = Analysis(case_id=case_id, pcap_path="/test.pcap", mitre_techniques=["T1071.001"]) + analysis_id = repo.save_analysis(analysis) + + analysis.id = analysis_id + analysis.mitre_techniques = ["T1059"] + repo.save_analysis(analysis) + + retrieved = repo.get_analysis(analysis_id) + assert retrieved.mitre_techniques == ["T1059"] + def test_save_analysis_with_iocs(self, repo): """Test saving analysis with IOCs.""" case_id = repo.create_case(Case(title="With IOCs")) @@ -542,7 +599,13 @@ def test_delete_case_cascades_all_related_rows(self, tmp_path): repo.create_case( Case(id="del00001", title="t", status=CaseStatus.IN_PROGRESS, severity=Severity.LOW, tags=["x"]) ) - analysis = Analysis(case_id="del00001", pcap_path="x.pcap", packet_count=1, features={"artifacts": {}}) + analysis = Analysis( + case_id="del00001", + pcap_path="x.pcap", + packet_count=1, + features={"artifacts": {}}, + mitre_techniques=["T1071.001"], + ) analysis.iocs = [IOC(ioc_type=IOCType.IP, value="203.0.113.9", context="t", severity=Severity.HIGH)] aid = repo.save_analysis(analysis) repo.add_note("del00001", "investigation note", analysis_id=aid) @@ -555,6 +618,7 @@ def test_delete_case_cascades_all_related_rows(self, tmp_path): for table, where, arg in [ ("analyses", "id = ?", aid), ("iocs", "analysis_id = ?", aid), + ("analysis_techniques", "analysis_id = ?", aid), ("jobs", "id = ?", job_id), ("case_tags", "case_id = ?", "del00001"), ("notes", "case_id = ?", "del00001"), diff --git a/tests/test_jobs_schema.py b/tests/test_jobs_schema.py index 6c0780e..15ed8d1 100644 --- a/tests/test_jobs_schema.py +++ b/tests/test_jobs_schema.py @@ -36,6 +36,29 @@ def test_analyses_has_attack_json_column(tmp_path): assert "attack_json" in cols, "analyses.attack_json column should exist for persisted ATT&CK mappings" +def test_analysis_techniques_table_exists(tmp_path): + repo = CaseRepository(db_path=str(tmp_path / "test.db")) + conn = repo._get_conn() + try: + row = conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name='analysis_techniques'" + ).fetchone() + cols = [r[1] for r in conn.execute("PRAGMA table_info(analysis_techniques)").fetchall()] + finally: + conn.close() + assert row is not None, "analysis_techniques table should exist for feed mitre_techniques derivation" + assert set(cols) == {"analysis_id", "technique_id"} + + +def test_analysis_techniques_table_idempotent_recreate(tmp_path): + """_init_schema runs on every CaseRepository construction; the CREATE TABLE + IF NOT EXISTS / CREATE INDEX IF NOT EXISTS statements must not raise on a + second open against the same database file.""" + db_path = str(tmp_path / "test.db") + CaseRepository(db_path=db_path) + CaseRepository(db_path=db_path) # must not raise + + def test_jobs_table_columns(tmp_path): """Verify the jobs table has all expected columns.""" repo = CaseRepository(db_path=str(tmp_path / "test.db")) From 230e3298f4d162aa13d2f5e714dc574492b36571 Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Mon, 13 Jul 2026 22:43:47 +0700 Subject: [PATCH 13/35] fix: populate mitre_techniques on ui-saved analyses for feed consistency Both manual-save paths in cases_tab.py (_quick_save_analysis and _add_current_analysis_to_case) built Analysis objects without mitre_techniques, so cases saved via the UI shipped a permanently-empty IOC feed mitre_techniques field even though the API/queue path already derives it from result.mitre_techniques. Add a shared _session_mitre_techniques() helper that derives technique IDs from the session's attack_mapping dict and wire it into both save paths. --- app/ui/cases_tab.py | 21 +++++++++++++ tests/test_cases_tab.py | 67 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 tests/test_cases_tab.py diff --git a/app/ui/cases_tab.py b/app/ui/cases_tab.py index aad406c..6760e03 100644 --- a/app/ui/cases_tab.py +++ b/app/ui/cases_tab.py @@ -46,6 +46,25 @@ def _get_repo() -> CaseRepository: return st.session_state["case_repo"] +def _session_mitre_techniques() -> list[str]: + """Derive MITRE ATT&CK technique IDs from the session's attack mapping. + + Mirrors how the API/queue path derives ``Analysis.mitre_techniques`` from + ``result.mitre_techniques`` (see ``app/api/queue.py``) so that analyses + saved through the UI populate the IOC feed's ``mitre_techniques`` field + the same way as analyses saved via the API — both share ``data/cases.db``. + + Returns: + List of technique IDs (e.g. ``["T1071.001"]``), or an empty list if + no attack mapping is present in session state. + """ + return [ + t.get("technique_id") + for t in (st.session_state.get("attack_mapping") or {}).get("techniques", []) + if t.get("technique_id") + ] + + def _restore_analysis_to_session(analysis: Analysis) -> None: """Load a saved analysis back into session state for the Dashboard/Results tabs. @@ -606,6 +625,7 @@ def _quick_save_analysis(): dns_analysis=st.session_state.get("dns_analysis"), tls_analysis=st.session_state.get("tls_analysis"), attack_mapping=st.session_state.get("attack_mapping") or {}, + mitre_techniques=_session_mitre_techniques(), ) # Extract IOCs @@ -640,6 +660,7 @@ def _add_current_analysis_to_case(case: Case): dns_analysis=st.session_state.get("dns_analysis"), tls_analysis=st.session_state.get("tls_analysis"), attack_mapping=st.session_state.get("attack_mapping") or {}, + mitre_techniques=_session_mitre_techniques(), ) analysis.iocs = repo.extract_iocs(analysis) diff --git a/tests/test_cases_tab.py b/tests/test_cases_tab.py new file mode 100644 index 0000000..82563c3 --- /dev/null +++ b/tests/test_cases_tab.py @@ -0,0 +1,67 @@ +"""Tests for app/ui/cases_tab.py. + +Focused on ``_session_mitre_techniques``, the helper that derives the +``Analysis.mitre_techniques`` list from ``st.session_state["attack_mapping"]`` +for the two UI manual-save paths (``_quick_save_analysis`` and +``_add_current_analysis_to_case``). Without this, analyses saved through the +UI ship a permanently-empty ``mitre_techniques`` field even though the API +path (``app/api/queue.py``) populates it from ``result.mitre_techniques`` -- +both share the same ``data/cases.db``, so the IOC feed's ``mitre_techniques`` +column would be inconsistent depending on which path saved the case. + +Uses Streamlit's AppTest harness (see ``tests/test_config_ui.py`` for the +established pattern) since the helper reads ``st.session_state`` directly. +""" + +from streamlit.testing.v1 import AppTest + + +def _mitre_helper_app(): + import streamlit as st + + from app.ui.cases_tab import _session_mitre_techniques + + st.session_state["__result"] = _session_mitre_techniques() + + +def _make_app() -> AppTest: + return AppTest.from_function(_mitre_helper_app, default_timeout=30) + + +class TestSessionMitreTechniques: + def test_derives_technique_ids_from_attack_mapping(self): + at = _make_app() + at.session_state["attack_mapping"] = { + "techniques": [ + {"technique_id": "T1071.001", "tactic": "command-and-control"}, + {"technique_id": "T1059", "tactic": "execution"}, + ] + } + at.run() + + assert at.session_state["__result"] == ["T1071.001", "T1059"] + + def test_empty_attack_mapping_yields_empty_list(self): + at = _make_app() + at.session_state["attack_mapping"] = {} + at.run() + + assert at.session_state["__result"] == [] + + def test_missing_attack_mapping_yields_empty_list(self): + at = _make_app() + at.run() + + assert at.session_state["__result"] == [] + + def test_skips_techniques_missing_technique_id(self): + at = _make_app() + at.session_state["attack_mapping"] = { + "techniques": [ + {"tactic": "execution"}, + {"technique_id": "T1105"}, + ] + } + at.run() + + assert at.session_state["__result"] == ["T1105"] From b452fd0e704604bc633704c384efa662a8e8e77e Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Mon, 13 Jul 2026 22:54:09 +0700 Subject: [PATCH 14/35] fix: soften https beacon penalty for highly-regular high-confidence flows The blanket 0.15 multiplier on port 443 (BENIGN_SERVICE_PORTS) crushed a genuinely periodic HTTPS beacon scoring 0.9-1.0 raw down to ~0.135-0.15, below both BEACON_SCORE_THRESHOLD (0.6) and the correlation ingest gate (0.5), making real HTTPS C2 -- the dominant C2 transport -- invisible. Make the penalty conditional: only when a 443 flow is both very high-confidence (raw score >= 0.85) and essentially jitter-free (jitter_pct <= 15) does it get a softened 0.7 multiplier instead of 0.15, tuned so a perfectly periodic flow (raw ~1.0) clears threshold with margin (1.0 * 0.7 = 0.7 > 0.6). A naive 4x multiplier (0.15 -> 0.6) was considered and rejected -- it still leaves a 0.9-raw flow at 0.54, under threshold. Ordinary jittery HTTPS keep-alives fail the >=0.85 gate and keep the full penalty, so they stay suppressed. Restricted to port 443 only (SOFTENABLE_BENIGN_PORTS) -- DNS/NTP/IMAPS/etc. in BENIGN_SERVICE_PORTS are genuinely periodic infrastructure by design, not a C2 cover transport, so they keep the full penalty regardless of regularity. Considered relaxing the correlation.py:129 beacon_lookup > 0.5 gate as an alternative/additional fix, but left it untouched: a softened-but-real 443 beacon now scores above 0.5 on its own and passes that gate without further changes. --- app/pipeline/beacon.py | 56 +++++++++++++++++++++++-- tests/test_beacon.py | 93 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+), 3 deletions(-) diff --git a/app/pipeline/beacon.py b/app/pipeline/beacon.py index 1aa2228..c187175 100644 --- a/app/pipeline/beacon.py +++ b/app/pipeline/beacon.py @@ -53,6 +53,39 @@ "5353": 0.15, # mDNS } +# Ports where a very regular, high-confidence signal is allowed to soften +# (not erase) the benign-service penalty above. Restricted to HTTPS/HTTP — +# the transports real C2 frameworks actually abuse to blend in. DNS/NTP and +# the other BENIGN_SERVICE_PORTS entries are deliberately excluded: they are +# genuinely periodic infrastructure traffic by design, not something C2 uses +# as cover, so no amount of "regularity" should un-suppress them. +SOFTENABLE_BENIGN_PORTS: frozenset[str] = frozenset({"443"}) + +# Average packet size (bytes) below which a flow's payloads look C2-like. +# Real C2 beacons are SMALL packets; CDN/streaming keep-alives carry large +# payloads. Reused by both the softening gate below and the large-payload +# penalty in rank_beaconing. +C2_PAYLOAD_MAX_BYTES = 500 + +# Softened multiplier applied instead of BENIGN_SERVICE_PORTS[dport] when a +# flow on a softenable port meets ALL THREE C2-like conditions: +# 1. very high-confidence — raw score >= 0.85 +# 2. essentially jitter-free — jitter_pct <= 15 +# 3. small average payload — pkt_lens present and mean < C2_PAYLOAD_MAX_BYTES +# That combination is exactly what genuine small-packet HTTPS C2 looks like. +# Tuned against tests/test_beacon.py so a perfectly periodic small-payload +# 443 flow (raw score ~1.0) clears BEACON_SCORE_THRESHOLD (0.6) with margin: +# 1.0 * 0.7 = 0.7 > 0.6. The naive "penalty * 4" (0.15 -> 0.6) is NOT enough: +# a 0.9-raw flow would land at 0.9 * 0.6 = 0.54, still below threshold. +# +# The small-payload condition (3) is load-bearing: a machine-regular CDN +# heartbeat (zero jitter, LARGE 1200-byte payloads) is indistinguishable +# from C2 by timing+jitter alone, so requiring small payloads keeps those +# fully penalised. When pkt_lens is absent/empty we CANNOT confirm the flow +# is C2-like, so we conservatively do NOT soften. Ordinary jittery HTTPS +# keep-alives also fail the >=0.85 gate and keep the full 0.15 penalty. +SOFTENED_PENALTY = 0.7 + def periodicity_score(ts: list[float]) -> dict[str, object]: """Score timestamp periodicity for beaconing detection. @@ -205,10 +238,28 @@ def rank_beaconing(flows: list[dict[str, object]], top_n: int = 20) -> pd.DataFr # Applied BEFORE any threshold checks so that benign traffic # is scored down before it can appear as a candidate. + pkt_lens = f.get("pkt_lens", []) + # 1. Benign service ports FIRST — this is the most common FP source # (e.g., HTTPS keep-alives to CDNs on port 443) if dport in BENIGN_SERVICE_PORTS: - final_score *= BENIGN_SERVICE_PORTS[dport] + penalty = BENIGN_SERVICE_PORTS[dport] + # A very regular, high-confidence, SMALL-PAYLOAD signal on a + # softenable port (443) is exactly what real HTTPS C2 looks like + # — soften the penalty so it can still surface above threshold, + # instead of guaranteeing a sub-threshold score. Ordinary HTTPS + # keep-alives score moderately/show jitter, and CDN heartbeats + # carry large payloads — both keep the full penalty. + jitter_pct = jitter.get("jitter_pct") + if jitter_pct is None: + jitter_pct = 100.0 + # Small average payload is required and conservative: when pkt_lens + # is absent/empty we cannot confirm the flow is C2-like, so we do + # NOT soften. + small_payload = bool(pkt_lens) and (sum(pkt_lens) / len(pkt_lens) < C2_PAYLOAD_MAX_BYTES) + if dport in SOFTENABLE_BENIGN_PORTS and final_score >= 0.85 and jitter_pct <= 15 and small_payload: + penalty = SOFTENED_PENALTY + final_score *= penalty # 2. Well-known infrastructure IPs (DNS resolvers, NTP servers) if dst in INFRA_ALLOWLIST or src in INFRA_ALLOWLIST: @@ -220,10 +271,9 @@ def rank_beaconing(flows: list[dict[str, object]], top_n: int = 20) -> pd.DataFr # 4. High-volume large-payload flows (streaming/downloads, not C2) # Real C2 beacons are small, infrequent packets. - pkt_lens = f.get("pkt_lens", []) if pkt_lens and len(ts) > 200: avg_pkt_size = sum(pkt_lens) / len(pkt_lens) - if avg_pkt_size > 500: + if avg_pkt_size > C2_PAYLOAD_MAX_BYTES: final_score *= 0.25 rows.append( diff --git a/tests/test_beacon.py b/tests/test_beacon.py index 74f931b..7352f75 100644 --- a/tests/test_beacon.py +++ b/tests/test_beacon.py @@ -1,3 +1,4 @@ +from app.config import BEACON_SCORE_THRESHOLD from app.pipeline.beacon import jitter_score, periodicity_score, rank_beaconing @@ -100,3 +101,95 @@ def test_jitter_score_random(): # Random traffic should score lower than periodic periodic = jitter_score([float(i) for i in range(1, 21)]) assert res["jitter_score"] < periodic["jitter_score"] + + +def test_rank_beaconing_regular_443_clears_threshold(): + # A genuinely periodic, SMALL-PAYLOAD HTTPS beacon (raw score ~1.0) must + # survive the port-443 benign-service penalty and still clear + # BEACON_SCORE_THRESHOLD. Before the conditional softening, the blanket + # 0.15 multiplier crushed this down to ~0.15 — real HTTPS C2 was invisible + # to the pipeline. Small payloads (80 bytes) are what real C2 beacons look + # like, and are required for softening. + flows = [ + { + "src": "10.0.0.3", + "dst": "203.0.113.5", + "sport": "51000", + "dport": "443", + "proto": "tcp", + "pkt_times": [float(i) for i in range(1, 60)], # perfectly periodic + "pkt_lens": [80] * 59, # small, C2-like payloads + } + ] + df = rank_beaconing(flows, top_n=10) + assert len(df) == 1 + assert df.iloc[0]["score"] > BEACON_SCORE_THRESHOLD + + +def test_rank_beaconing_regular_443_large_payload_not_softened(): + # A machine-regular flow on 443 with LARGE payloads (like a CDN heartbeat) + # is indistinguishable from C2 by timing+jitter alone — it must NOT be + # softened. Only small-payload flows qualify; this keeps the full 0.15 + # penalty and stays well below threshold. Locks in the small-payload + # distinction that guards tests/test_integration.py::test_https_cdn_not_flagged. + flows = [ + { + "src": "10.0.0.5", + "dst": "13.224.0.9", # CDN-like IP + "sport": "51002", + "dport": "443", + "proto": "tcp", + "pkt_times": [float(i) for i in range(1, 60)], # perfectly periodic + "pkt_lens": [1200] * 59, # large payloads → NOT C2-like + } + ] + df = rank_beaconing(flows, top_n=10) + assert len(df) == 1 + # Full 0.15 penalty applied (raw ~1.0 * 0.15 = 0.15), no softening. + assert df.iloc[0]["score"] <= 0.15 + assert df.iloc[0]["score"] < BEACON_SCORE_THRESHOLD + + +def test_rank_beaconing_regular_443_no_pkt_lens_not_softened(): + # Conservative default: without pkt_lens we cannot confirm the flow is + # C2-like (small payloads), so we do NOT soften. Keeps the full penalty. + flows = [ + { + "src": "10.0.0.6", + "dst": "203.0.113.7", + "sport": "51003", + "dport": "443", + "proto": "tcp", + "pkt_times": [float(i) for i in range(1, 60)], # perfectly periodic + # no pkt_lens + } + ] + df = rank_beaconing(flows, top_n=10) + assert len(df) == 1 + assert df.iloc[0]["score"] <= 0.15 + assert df.iloc[0]["score"] < BEACON_SCORE_THRESHOLD + + +def test_rank_beaconing_jittery_443_stays_suppressed(): + # An ordinary HTTPS keep-alive with real-world jitter must NOT be + # promoted by the softening — only extremely regular, high-confidence + # signals qualify. This flow's raw score drops well below the 0.85 + # softening gate once jitter is introduced, so it keeps the full 0.15 + # penalty and stays suppressed below threshold. + import random + + random.seed(42) + ts = sorted(10.0 * i + random.uniform(-2, 2) for i in range(30)) + flows = [ + { + "src": "10.0.0.4", + "dst": "203.0.113.6", + "sport": "51001", + "dport": "443", + "proto": "tcp", + "pkt_times": ts, + } + ] + df = rank_beaconing(flows, top_n=10) + assert len(df) == 1 + assert df.iloc[0]["score"] < BEACON_SCORE_THRESHOLD From 58f484ba02f4ca5ae58795d6548416d2678cf2eb Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Mon, 13 Jul 2026 23:22:39 +0700 Subject: [PATCH 15/35] feat: add http analysis module (ua/creds/uri heuristics) --- app/config.py | 15 ++ app/pipeline/http_analysis.py | 308 ++++++++++++++++++++++++++ tests/test_http_analysis.py | 403 ++++++++++++++++++++++++++++++++++ 3 files changed, 726 insertions(+) create mode 100644 app/pipeline/http_analysis.py create mode 100644 tests/test_http_analysis.py diff --git a/app/config.py b/app/config.py index a709b7d..715b82e 100644 --- a/app/config.py +++ b/app/config.py @@ -93,6 +93,21 @@ BEACON_MIN_PACKETS = 4 # Minimum packets to evaluate for beaconing BEACON_TOP_N = 20 # Number of top beacon candidates to report +# HTTP Analysis +HTTP_SUSPICIOUS_URI_LEN = 512 # URI length above this → suspicious (possible exfil/exploit) +HTTP_SUSPICIOUS_UA_TOKENS = frozenset( + { + "python-requests", + "curl", + "wget", + "powershell", + "go-http-client", + "nmap", + "sqlmap", + "masscan", + } +) + # Flow Analysis FLOW_ASYMMETRY_RATIO = 10 # Outbound/inbound ratio above this → suspicious FLOW_ASYMMETRY_MIN_BYTES = 1_000_000 # 1 MB minimum to flag asymmetry diff --git a/app/pipeline/http_analysis.py b/app/pipeline/http_analysis.py new file mode 100644 index 0000000..ae053b7 --- /dev/null +++ b/app/pipeline/http_analysis.py @@ -0,0 +1,308 @@ +"""HTTP request analysis for threat detection. + +Provides detection for: +- Suspicious User-Agent strings (missing UA, known scripting/scanning tools) +- Cleartext credentials (Basic-auth username observed in Zeek http.log) +- Suspicious URIs (risky file extensions, oversized URIs, raw-IP file downloads) +""" + +from __future__ import annotations + +import logging +import re +from collections import Counter +from dataclasses import dataclass +from typing import Any + +import pandas as pd + +from app.config import HTTP_SUSPICIOUS_UA_TOKENS, HTTP_SUSPICIOUS_URI_LEN +from app.pipeline.state import PhaseHandle +from app.pipeline.zeek import load_zeek_any + +logger = logging.getLogger(__name__) + +# --- Result Limits --- +MAX_HTTP_RESULTS = 100 # Cap each result list to bound output size + +# Raw dotted-quad IPv4 pattern, used by the raw-IP-host heuristic. +IPV4_PATTERN = re.compile(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$") + +# File extensions associated with executables/scripts — risky when served over plain HTTP. +RISKY_EXTENSIONS = (".exe", ".dll", ".ps1", ".scr", ".bin") + +# Values Zeek/ASCII logs use to mean "missing" for a field. +_MISSING_VALUES = {"-", "nan", "none", ""} + + +@dataclass +class HTTPRequest: + """Parsed Zeek http.log record.""" + + ts: float + src: str + dst: str + host: str + uri: str + method: str + user_agent: str + username: str + status_code: int + + +def _clean(value: Any) -> str: + """Normalize a raw Zeek field to a stripped string; '-'/NaN/None/empty become ''.""" + if value is None: + return "" + text = str(value).strip() + if text.lower() in _MISSING_VALUES: + return "" + return text + + +def _clean_int(value: Any) -> int: + """Best-effort int parse; missing/unparseable values become 0.""" + text = _clean(value) + if not text: + return 0 + try: + return int(float(text)) + except (ValueError, TypeError): + return 0 + + +def parse_http_log(df: pd.DataFrame) -> list[HTTPRequest]: + """ + Parse Zeek http.log DataFrame into HTTPRequest objects. + + Uses dual-name column access (dotted Zeek names with underscore fallback) + and is string-tolerant: ASCII-mode logs deliver every field as text, with + "-" as the literal missing-value marker. + + Args: + df: DataFrame from Zeek http.log + + Returns: + List of HTTPRequest objects + """ + records = [] + + for row in df.to_dict(orient="records"): + try: + try: + ts = float(row.get("ts", 0) or 0) + except (ValueError, TypeError): + ts = 0.0 + + records.append( + HTTPRequest( + ts=ts, + src=_clean(row.get("id.orig_h", row.get("id_orig_h", ""))), + dst=_clean(row.get("id.resp_h", row.get("id_resp_h", ""))), + host=_clean(row.get("host", "")), + uri=_clean(row.get("uri", "")), + method=_clean(row.get("method", "")), + user_agent=_clean(row.get("user_agent", "")), + username=_clean(row.get("username", "")), + status_code=_clean_int(row.get("status_code", "")), + ) + ) + except (ValueError, TypeError) as e: + logger.debug("Failed to parse HTTP record: %s", e) + continue + + return records + + +def detect_suspicious_ua(req: HTTPRequest) -> str | None: + """ + Detect a suspicious User-Agent on an HTTP request. + + Flags: + - Empty/missing User-Agent on a request that has a Host (incomplete/scripted client) + - User-Agent naming a known scripting/scanning tool (case-insensitive substring) + + Args: + req: Parsed HTTP request + + Returns: + Reason string if suspicious, else None + """ + if not req.user_agent: + if req.host: + return "missing user-agent" + return None + + ua_lower = req.user_agent.lower() + for token in HTTP_SUSPICIOUS_UA_TOKENS: + if token in ua_lower: + return f"known tool user-agent ({token})" + + return None + + +def detect_cleartext_credentials(req: HTTPRequest) -> dict[str, str] | None: + """ + Detect cleartext (Basic-auth) credentials on an HTTP request. + + Only the username is emitted — never the password, even if the log carried one. + + Args: + req: Parsed HTTP request + + Returns: + {"host", "uri", "username"} dict if credentials were seen, else None + """ + if not req.username: + return None + return {"host": req.host, "uri": req.uri, "username": req.username} + + +def detect_suspicious_uri(req: HTTPRequest) -> str | None: + """ + Detect a suspicious URI on an HTTP request. + + Flags: + - Risky file extension (.exe/.dll/.ps1/.scr/.bin) + - Very long URI (> HTTP_SUSPICIOUS_URI_LEN chars) + - Raw IPv4 host serving what looks like a file download + + Args: + req: Parsed HTTP request + + Returns: + Reason string if suspicious, else None + """ + if not req.uri: + return None + + uri_lower = req.uri.lower() + reasons = [] + + matched_ext = next((ext for ext in RISKY_EXTENSIONS if ext in uri_lower), None) + if matched_ext: + reasons.append(f"risky file extension ({matched_ext})") + + if len(req.uri) > HTTP_SUSPICIOUS_URI_LEN: + reasons.append(f"long URI ({len(req.uri)} chars)") + + if matched_ext and IPV4_PATTERN.match(req.host): + reasons.append("raw-IP host serving file download") + + return "; ".join(reasons) if reasons else None + + +def analyze_http( + zeek_tables: dict[str, pd.DataFrame], + http_log_path: str | None = None, + phase: PhaseHandle | None = None, +) -> dict[str, Any]: + """ + Comprehensive HTTP request analysis from Zeek http.log. + + Args: + zeek_tables: Dictionary of Zeek log DataFrames (row-capped, see ZEEK_TABLE_MAX_ROWS) + http_log_path: Optional path to the on-disk http.log. When provided, the full + uncapped log is read via load_zeek_any instead of the capped in-memory table. + phase: PhaseHandle for progress updates + + Returns: + Dictionary with HTTP analysis results + """ + if phase and phase.should_skip(): + phase.done("HTTP analysis skipped.") + return {"skipped": True} + + if phase: + phase.set(5, "Parsing HTTP logs...") + + http_df: pd.DataFrame | None = None + if http_log_path: + try: + http_df = load_zeek_any(http_log_path) + except Exception as e: + logger.debug("Failed to load full http.log from %s: %s", http_log_path, e) + http_df = None + + if http_df is None: + http_df = zeek_tables.get("http.log") + + if http_df is None: + if phase: + phase.done("No HTTP data available.") + return {"skipped": True} + + if http_df.empty: + if phase: + phase.done("No HTTP data available.") + return {"error": "No HTTP log data", "records": 0} + + records = parse_http_log(http_df) + if not records: + if phase: + phase.done("No valid HTTP records found.") + return {"error": "No HTTP log data", "records": 0} + + if phase: + phase.set(30, f"Analyzing {len(records)} HTTP requests...") + + methods = Counter(r.method for r in records if r.method) + status_codes = Counter(str(r.status_code) for r in records if r.status_code) + unique_hosts = len({r.host for r in records if r.host}) + + suspicious_uas = [] + cleartext_creds = [] + suspicious_uris = [] + + total = len(records) + for i, r in enumerate(records): + if phase and i % 500 == 0: + pct = 30 + int((i / total) * 50) + phase.set(pct, f"Scanning request {i + 1}/{total}...") + + ua_reason = detect_suspicious_ua(r) + if ua_reason: + suspicious_uas.append({"host": r.host, "user_agent": r.user_agent, "uri": r.uri, "reason": ua_reason}) + + cred = detect_cleartext_credentials(r) + if cred: + cleartext_creds.append(cred) + + uri_reason = detect_suspicious_uri(r) + if uri_reason: + suspicious_uris.append({"host": r.host, "uri": r.uri, "reason": uri_reason}) + + if phase: + phase.set(85, "Finalizing HTTP analysis...") + + result = { + "total_requests": len(records), + "unique_hosts": unique_hosts, + "methods": dict(methods), + "status_codes": dict(status_codes), + "suspicious_user_agents": suspicious_uas[:MAX_HTTP_RESULTS], + "cleartext_credentials": cleartext_creds[:MAX_HTTP_RESULTS], + "suspicious_uris": suspicious_uris[:MAX_HTTP_RESULTS], + "alerts": { + "suspicious_ua_count": len(suspicious_uas), + "cleartext_cred_count": len(cleartext_creds), + "suspicious_uri_count": len(suspicious_uris), + }, + } + + if phase: + alerts = result["alerts"] + alert_msg = [] + if alerts["suspicious_ua_count"]: + alert_msg.append(f"{alerts['suspicious_ua_count']} suspicious UA") + if alerts["cleartext_cred_count"]: + alert_msg.append(f"{alerts['cleartext_cred_count']} cleartext creds") + if alerts["suspicious_uri_count"]: + alert_msg.append(f"{alerts['suspicious_uri_count']} suspicious URIs") + + summary = f"Analyzed {len(records)} HTTP requests, {unique_hosts} hosts." + if alert_msg: + summary += f" Alerts: {', '.join(alert_msg)}." + phase.done(summary) + + return result diff --git a/tests/test_http_analysis.py b/tests/test_http_analysis.py new file mode 100644 index 0000000..171a3e9 --- /dev/null +++ b/tests/test_http_analysis.py @@ -0,0 +1,403 @@ +"""Tests for HTTP analysis module.""" + +from unittest.mock import patch + +import pandas as pd +import pytest + +from app.pipeline.http_analysis import ( + HTTPRequest, + analyze_http, + detect_cleartext_credentials, + detect_suspicious_ua, + detect_suspicious_uri, + parse_http_log, +) + + +def _req(**overrides) -> HTTPRequest: + """Build an HTTPRequest with sane defaults, overridden per test.""" + defaults = dict( + ts=1234567890.0, + src="192.168.1.10", + dst="93.184.216.34", + host="www.example.com", + uri="/index.html", + method="GET", + user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64)", + username="", + status_code=200, + ) + defaults.update(overrides) + return HTTPRequest(**defaults) + + +class TestParseHTTPLog: + """Test parsing Zeek http.log DataFrame.""" + + def test_empty_dataframe(self): + df = pd.DataFrame() + records = parse_http_log(df) + assert records == [] + + def test_basic_parsing_dotted_columns(self): + df = pd.DataFrame( + [ + { + "ts": "1234567890.0", + "id.orig_h": "192.168.1.10", + "id.resp_h": "93.184.216.34", + "host": "www.example.com", + "uri": "/index.html", + "method": "GET", + "user_agent": "Mozilla/5.0", + "username": "-", + "password": "-", + "status_code": "200", + } + ] + ) + records = parse_http_log(df) + assert len(records) == 1 + r = records[0] + assert r.src == "192.168.1.10" + assert r.dst == "93.184.216.34" + assert r.host == "www.example.com" + assert r.uri == "/index.html" + assert r.method == "GET" + assert r.user_agent == "Mozilla/5.0" + assert r.username == "" # "-" treated as missing + assert r.status_code == 200 + + def test_underscore_column_fallback(self): + df = pd.DataFrame( + [ + { + "ts": "1234567890.0", + "id_orig_h": "10.0.0.5", + "id_resp_h": "10.0.0.1", + "host": "internal.local", + "uri": "/", + "method": "GET", + "user_agent": "-", + "status_code": "-", + } + ] + ) + records = parse_http_log(df) + assert len(records) == 1 + assert records[0].src == "10.0.0.5" + assert records[0].dst == "10.0.0.1" + assert records[0].user_agent == "" + assert records[0].status_code == 0 + + def test_missing_optional_fields_default_empty(self): + df = pd.DataFrame([{"ts": "1.0", "id.orig_h": "1.2.3.4"}]) + records = parse_http_log(df) + assert len(records) == 1 + assert records[0].host == "" + assert records[0].uri == "" + assert records[0].username == "" + + +class TestSuspiciousUA: + """Test suspicious User-Agent detection.""" + + def test_missing_ua_with_host_is_suspicious(self): + req = _req(user_agent="", host="www.example.com") + reason = detect_suspicious_ua(req) + assert reason is not None + assert "user-agent" in reason.lower() + + def test_missing_ua_without_host_is_not_flagged(self): + req = _req(user_agent="", host="") + assert detect_suspicious_ua(req) is None + + @pytest.mark.parametrize( + "ua", + [ + "python-requests/2.28.0", + "curl/7.79.1", + "Wget/1.21", + "PowerShell/7.2", + "Go-http-client/1.1", + "Nmap Scripting Engine", + "sqlmap/1.6", + ], + ) + def test_known_tool_ua_is_suspicious(self, ua): + req = _req(user_agent=ua) + reason = detect_suspicious_ua(req) + assert reason is not None + + def test_normal_browser_ua_is_not_flagged(self): + req = _req(user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15") + assert detect_suspicious_ua(req) is None + + +class TestCleartextCredentials: + """Test cleartext credential detection.""" + + def test_username_present_is_flagged(self): + req = _req(username="admin", uri="/login") + cred = detect_cleartext_credentials(req) + assert cred == {"host": req.host, "uri": "/login", "username": "admin"} + + def test_username_missing_is_not_flagged(self): + req = _req(username="") + assert detect_cleartext_credentials(req) is None + + def test_password_value_never_emitted(self): + req = _req(username="admin") + cred = detect_cleartext_credentials(req) + assert "password" not in cred + + +class TestSuspiciousURI: + """Test suspicious URI detection.""" + + @pytest.mark.parametrize("ext", [".exe", ".dll", ".ps1", ".scr", ".bin"]) + def test_risky_extension_is_flagged(self, ext): + req = _req(uri=f"/downloads/payload{ext}") + reason = detect_suspicious_uri(req) + assert reason is not None + + def test_long_uri_is_flagged(self): + req = _req(uri="/path?" + "a" * 600) + reason = detect_suspicious_uri(req) + assert reason is not None + assert "long" in reason.lower() + + def test_raw_ip_host_file_download_is_flagged(self): + req = _req(host="203.0.113.5", uri="/payload.dll") + reason = detect_suspicious_uri(req) + assert reason is not None + + def test_clean_uri_is_not_flagged(self): + req = _req(uri="/index.html") + assert detect_suspicious_uri(req) is None + + def test_empty_uri_is_not_flagged(self): + req = _req(uri="") + assert detect_suspicious_uri(req) is None + + +class TestAnalyzeHTTP: + """Test comprehensive HTTP analysis.""" + + def test_empty_zeek_tables_is_skipped(self): + result = analyze_http({}) + assert result == {"skipped": True} + + def test_empty_http_log_is_error(self): + result = analyze_http({"http.log": pd.DataFrame()}) + assert result.get("error") == "No HTTP log data" + assert result.get("records") == 0 + + def test_basic_analysis_all_heuristics(self): + df = pd.DataFrame( + [ + { + "ts": "1.0", + "id.orig_h": "192.168.1.10", + "id.resp_h": "93.184.216.34", + "host": "www.example.com", + "uri": "/index.html", + "method": "GET", + "user_agent": "Mozilla/5.0", + "username": "-", + "status_code": "200", + }, + { + "ts": "2.0", + "id.orig_h": "192.168.1.11", + "id.resp_h": "93.184.216.34", + "host": "www.example.com", + "uri": "/api/data", + "method": "GET", + "user_agent": "-", + "username": "-", + "status_code": "200", + }, + { + "ts": "3.0", + "id.orig_h": "192.168.1.12", + "id.resp_h": "198.51.100.9", + "host": "cdn.example.net", + "uri": "/pkg/tool.tar.gz", + "method": "GET", + "user_agent": "python-requests/2.28.0", + "username": "-", + "status_code": "200", + }, + { + "ts": "4.0", + "id.orig_h": "192.168.1.13", + "id.resp_h": "10.0.0.5", + "host": "10.0.0.5", + "uri": "/admin", + "method": "POST", + "user_agent": "Mozilla/5.0", + "username": "admin", + "password": "hunter2", + "status_code": "401", + }, + { + "ts": "5.0", + "id.orig_h": "192.168.1.14", + "id.resp_h": "198.51.100.10", + "host": "files.example.org", + "uri": "/downloads/update.exe", + "method": "GET", + "user_agent": "Mozilla/5.0", + "username": "-", + "status_code": "200", + }, + { + "ts": "6.0", + "id.orig_h": "192.168.1.15", + "id.resp_h": "203.0.113.5", + "host": "203.0.113.5", + "uri": "/malware.dll", + "method": "GET", + "user_agent": "Mozilla/5.0", + "username": "-", + "status_code": "200", + }, + { + "ts": "7.0", + "id.orig_h": "192.168.1.16", + "id.resp_h": "198.51.100.11", + "host": "files.example.org", + "uri": "/q?" + "x" * 600, + "method": "GET", + "user_agent": "Mozilla/5.0", + "username": "-", + "status_code": "200", + }, + ] + ) + result = analyze_http({"http.log": df}) + + assert result["total_requests"] == 7 + assert result["unique_hosts"] == 5 + assert result["methods"] == {"GET": 6, "POST": 1} + assert result["status_codes"]["200"] == 6 + assert result["status_codes"]["401"] == 1 + + assert result["alerts"]["suspicious_ua_count"] == 2 # missing UA + python-requests + assert result["alerts"]["cleartext_cred_count"] == 1 + assert result["alerts"]["suspicious_uri_count"] == 3 # .exe, .dll(+raw-ip), long query + + assert len(result["suspicious_user_agents"]) == 2 + assert len(result["cleartext_credentials"]) == 1 + assert result["cleartext_credentials"][0] == {"host": "10.0.0.5", "uri": "/admin", "username": "admin"} + # Password must never be emitted even though the log carried one. + assert all("password" not in c for c in result["cleartext_credentials"]) + assert len(result["suspicious_uris"]) == 3 + + def test_clean_traffic_yields_empty_alerts(self): + df = pd.DataFrame( + [ + { + "ts": "1.0", + "id.orig_h": "192.168.1.10", + "id.resp_h": "93.184.216.34", + "host": "www.example.com", + "uri": "/index.html", + "method": "GET", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)", + "username": "-", + "status_code": "200", + }, + { + "ts": "2.0", + "id.orig_h": "192.168.1.11", + "id.resp_h": "93.184.216.34", + "host": "www.example.com", + "uri": "/style.css", + "method": "GET", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)", + "username": "-", + "status_code": "200", + }, + ] + ) + result = analyze_http({"http.log": df}) + + assert result["total_requests"] == 2 + assert result["suspicious_user_agents"] == [] + assert result["cleartext_credentials"] == [] + assert result["suspicious_uris"] == [] + assert result["alerts"] == { + "suspicious_ua_count": 0, + "cleartext_cred_count": 0, + "suspicious_uri_count": 0, + } + + def test_http_log_path_used_over_capped_table(self): + """When http_log_path is given, the full uncapped log wins over zeek_tables.""" + capped_df = pd.DataFrame( + [ + { + "ts": "1.0", + "id.orig_h": "192.168.1.10", + "id.resp_h": "93.184.216.34", + "host": "capped.example.com", + "uri": "/", + "method": "GET", + "user_agent": "Mozilla/5.0", + "status_code": "200", + } + ] + ) + full_df = pd.DataFrame( + [ + { + "ts": "1.0", + "id.orig_h": "192.168.1.10", + "id.resp_h": "93.184.216.34", + "host": "full.example.com", + "uri": "/", + "method": "GET", + "user_agent": "Mozilla/5.0", + "status_code": "200", + }, + { + "ts": "2.0", + "id.orig_h": "192.168.1.11", + "id.resp_h": "93.184.216.34", + "host": "full.example.com", + "uri": "/extra", + "method": "GET", + "user_agent": "Mozilla/5.0", + "status_code": "200", + }, + ] + ) + with patch("app.pipeline.http_analysis.load_zeek_any", return_value=full_df) as mock_load: + result = analyze_http({"http.log": capped_df}, http_log_path="/fake/path/http.log") + + mock_load.assert_called_once_with("/fake/path/http.log") + assert result["total_requests"] == 2 + assert result["unique_hosts"] == 1 + + def test_http_log_path_load_failure_falls_back_to_zeek_tables(self): + df = pd.DataFrame( + [ + { + "ts": "1.0", + "id.orig_h": "192.168.1.10", + "id.resp_h": "93.184.216.34", + "host": "www.example.com", + "uri": "/", + "method": "GET", + "user_agent": "Mozilla/5.0", + "status_code": "200", + } + ] + ) + with patch("app.pipeline.http_analysis.load_zeek_any", side_effect=OSError("boom")): + result = analyze_http({"http.log": df}, http_log_path="/does/not/exist/http.log") + + assert result["total_requests"] == 1 From 94d01c0f36e64b69a8a23dbb42c31b0441895770 Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Mon, 13 Jul 2026 23:40:12 +0700 Subject: [PATCH 16/35] feat: wire http analysis stage into pipeline runner Adds the HTTP analysis stage (app/pipeline/http_analysis.py, from a prior commit) to the concurrent post-parse fan-out in app/pipeline/runner.py, gated on http.log being present in zeek_tables. Registers the "HTTP Analysis" phase in main.py's two phase lists and session-state defaults/reset blocks for consistency with the other zeek-log stages. --- app/main.py | 4 + app/pipeline/runner.py | 47 ++++++-- tests/test_pipeline_runner.py | 198 ++++++++++++++++++++++++++++++++-- 3 files changed, 232 insertions(+), 17 deletions(-) diff --git a/app/main.py b/app/main.py index 905640a..881185b 100644 --- a/app/main.py +++ b/app/main.py @@ -326,6 +326,7 @@ def _run_single_pcap_pipeline( ("runtime_logs", []), ("map_reset_counter", 0), ("dns_analysis", None), + ("http_analysis", None), ("tls_analysis", None), ("yara_results", None), ("correlations", None), @@ -440,6 +441,7 @@ def _run_single_pcap_pipeline( ("Zeek processing", do_zeek), ("DNS Analysis", do_zeek), # Requires Zeek dns.log ("TLS Certificate Analysis", do_zeek), # Requires Zeek ssl.log + ("HTTP Analysis", do_zeek), # Requires Zeek http.log ("Beaconing ranking", True), ("HTTP carving (tshark)", do_carve), ("YARA Scanning", do_carve and do_yara), # Requires carved files @@ -465,6 +467,7 @@ def _run_single_pcap_pipeline( "__pcap_path": pcap_path, "__pcap_paths": pcap_paths or [pcap_path], "dns_analysis": None, + "http_analysis": None, "tls_analysis": None, "yara_results": None, "correlations": None, @@ -534,6 +537,7 @@ def _run_single_pcap_pipeline( ("Zeek processing", do_zeek), ("DNS Analysis", do_zeek), ("TLS Certificate Analysis", do_zeek), + ("HTTP Analysis", do_zeek), ("Beaconing ranking", True), ("HTTP carving (tshark)", do_carve), ("YARA Scanning", do_carve and do_yara), diff --git a/app/pipeline/runner.py b/app/pipeline/runner.py index ba2cae5..4ef132b 100644 --- a/app/pipeline/runner.py +++ b/app/pipeline/runner.py @@ -38,6 +38,7 @@ from app.pipeline.beacon import rank_beaconing from app.pipeline.carve import CarveError, carve_http_payloads from app.pipeline.dns_analysis import analyze_dns +from app.pipeline.http_analysis import analyze_http from app.pipeline.pcap_count import count_packets_fast from app.pipeline.progress import Progress from app.pipeline.pyshark_pass import parse_pcap_pyshark @@ -56,6 +57,7 @@ WARNING_ZEEK_NO_LOGS = "zeek_no_logs" WARNING_DNS_ANALYSIS_FAILED = "dns_analysis_failed" WARNING_TLS_CERTS_FAILED = "tls_certs_failed" +WARNING_HTTP_ANALYSIS_FAILED = "http_analysis_failed" WARNING_BEACON_FAILED = "beacon_failed" WARNING_CARVE_FAILED = "carve_failed" WARNING_ZEEK_TRUNCATED = "zeek_tables_truncated" @@ -123,6 +125,7 @@ class PipelineResult: attack_mapping: dict = field(default_factory=dict) dns_analysis: dict = field(default_factory=dict) tls_analysis: dict = field(default_factory=dict) + http_analysis: dict = field(default_factory=dict) beacon_df_records: list[dict] = field(default_factory=list) # Intermediate state — available to callers (e.g. Streamlit) that need to run @@ -148,6 +151,7 @@ def to_dict(self) -> dict: "attack_mapping": dict(self.attack_mapping), "dns_analysis": dict(self.dns_analysis), "tls_analysis": dict(self.tls_analysis), + "http_analysis": dict(self.http_analysis), "beacon_df_records": list(self.beacon_df_records), } @@ -165,10 +169,11 @@ def run_pipeline( Stages 2 (PyShark) and 3 (Zeek) run concurrently via ThreadPoolExecutor when both are enabled — they're I/O-bound subprocesses against the same pcap and independent until the merge step. After that join, stages 4–7 (DNS, TLS, beaconing, carving) - fan out into a second ThreadPoolExecutor — they are mutually independent, and the - main thread assembles ``stages_run``/``warnings`` in canonical order after the join - so the output stays deterministic. Each stage is gated by its corresponding - ``PipelineOptions`` flag. Failures in individual stages are recorded in + plus HTTP analysis (gated on http.log being present) fan out into a second + ThreadPoolExecutor — they are mutually independent, and the main thread assembles + ``stages_run``/``warnings`` in canonical order after the join so the output stays + deterministic. Each stage is gated by its corresponding ``PipelineOptions`` flag. + Failures in individual stages are recorded in ``result.warnings`` rather than aborting the run. """ start = time.time() @@ -197,6 +202,7 @@ def run_pipeline( total_pkts: int | None = None dns_result: dict = {} tls_result: dict = {} + http_result: dict = {} beacon_records: list[dict] = [] carved: list[dict] = [] @@ -301,13 +307,14 @@ def _run_zeek(h) -> None: except Exception as exc: logger.warning("merge_zeek_dns failed: %s", exc) - # --- Stages 4-7: post-parse analysis fan-out --- - # DNS + TLS read zeek_tables; beacon reads features["flows"]; carve reads only the - # pcap. They are mutually independent, so run them concurrently. Phase handles are - # created on the main thread (Streamlit ScriptRunContext requirement) — workers only - # call set()/done(). Workers never mutate shared state: each writes its own key in + # --- Stages 4-7 (+ HTTP analysis): post-parse analysis fan-out --- + # DNS + TLS + HTTP read zeek_tables (HTTP only runs when http.log is present); + # beacon reads features["flows"]; carve reads only the pcap. They are mutually + # independent, so run them concurrently. Phase handles are created on the main + # thread (Streamlit ScriptRunContext requirement) — workers only call + # set()/done(). Workers never mutate shared state: each writes its own key in # `outcomes`, and the carve hash-backfill happens after the join. - canonical = ("dns_analysis", "tls_certs", "beacon", "carve") + canonical = ("dns_analysis", "tls_certs", "http_analysis", "beacon", "carve") outcomes: dict[str, dict] = {} def _run_dns(h) -> None: @@ -330,6 +337,17 @@ def _run_tls(h) -> None: outcomes.setdefault("tls_certs", {"warning": WARNING_TLS_CERTS_FAILED}) h.done("TLS analysis failed.") + def _run_http(h) -> None: + try: + outcomes["http_analysis"] = { + "result": analyze_http(zeek_tables, http_log_path=zeek_log_paths.get("http.log"), phase=h) or {} + } + h.done("HTTP analysis complete.") + except Exception as exc: + logger.error("HTTP analysis failed: %s", exc) + outcomes.setdefault("http_analysis", {"warning": WARNING_HTTP_ANALYSIS_FAILED}) + h.done("HTTP analysis failed.") + def _run_beacon(h) -> None: try: h.set(30, "Scoring flows…") @@ -366,6 +384,12 @@ def _run_carve(h) -> None: if options.do_zeek and zeek_tables: jobs.append((_run_dns, progress.start_phase("DNS Analysis"))) jobs.append((_run_tls, progress.start_phase("TLS Certificate Analysis"))) + # Same gate as DNS/TLS (not "http.log" in zeek_tables): the phase must always + # run when Zeek ran so the UI's total_phases denominator matches the phases + # actually started. analyze_http reads the uncapped http.log when present and + # returns {"skipped": True} on all-HTTPS / DNS-only captures — mirroring how + # DNS/TLS run and return empty/skipped when their log is absent. + jobs.append((_run_http, progress.start_phase("HTTP Analysis"))) if features.get("flows"): jobs.append((_run_beacon, progress.start_phase("Beaconing ranking"))) if options.do_carve: @@ -391,6 +415,8 @@ def _run_carve(h) -> None: dns_result = out["result"] elif name == "tls_certs": tls_result = out["result"] + elif name == "http_analysis": + http_result = out["result"] elif name == "beacon": beacon_records = out["result"] elif name == "carve": @@ -436,6 +462,7 @@ def _run_carve(h) -> None: attack_mapping=attack_mapping_dict, dns_analysis=dns_result, tls_analysis=tls_result, + http_analysis=http_result, beacon_df_records=beacon_records, features=features, zeek_tables=zeek_tables, diff --git a/tests/test_pipeline_runner.py b/tests/test_pipeline_runner.py index e98070f..2c90b08 100644 --- a/tests/test_pipeline_runner.py +++ b/tests/test_pipeline_runner.py @@ -181,11 +181,11 @@ def test_streamlit_to_options_mapping(): def test_stages_4_to_7_run_concurrently(monkeypatch, tmp_path): - """Stages 4 (DNS), 5 (TLS), 6 (beacon), 7 (carve) must overlap in time. + """Stages 4 (DNS), 5 (TLS), HTTP, 6 (beacon), 7 (carve) must overlap in time. All upstream stages are stubbed so only the post-parse fan-out is exercised. - Each of the four stage functions increments a lock-protected counter, sleeps, - then decrements — if they run sequentially the observed max concurrency is 1. + Each stage function increments a lock-protected counter, sleeps, then + decrements — if they run sequentially the observed max concurrency is 1. """ import threading import time @@ -238,6 +238,7 @@ def _fn(*args, **kwargs): monkeypatch.setattr(R, "analyze_dns", _tracked({})) monkeypatch.setattr(R, "analyze_certificates", _tracked({})) + monkeypatch.setattr(R, "analyze_http", _tracked({})) monkeypatch.setattr(R, "rank_beaconing", _tracked(pd.DataFrame())) monkeypatch.setattr(R, "carve_http_payloads", _tracked([])) @@ -248,17 +249,19 @@ def _fn(*args, **kwargs): progress=CallbackProgress(callback=lambda _e: None, total_phases=0), ) - for stage in ("dns_analysis", "tls_certs", "beacon", "carve"): + for stage in ("dns_analysis", "tls_certs", "http_analysis", "beacon", "carve"): assert stage in result.stages_run, f"{stage} did not run" assert state["max"] >= 2, f"stages 4-7 ran sequentially (max concurrency observed: {state['max']})" def test_stage_order_deterministic_and_carve_hashes_backfilled(monkeypatch, tmp_path): - """stages_run keeps the canonical dns→tls→beacon→carve order and carve hashes land in features. + """stages_run keeps the canonical dns→tls→http→beacon→carve order and carve hashes land in features. Regression guard for the fan-out: workers record outcomes per stage, the main thread assembles stages_run in canonical order and backfills carved sha256 - hashes into features["artifacts"]["hashes"] after the join. + hashes into features["artifacts"]["hashes"] after the join. HTTP analysis runs + on the same gate as DNS/TLS (Zeek ran + non-empty zeek_tables), so it appears + in the canonical order even though this fixture's zeek_tables carries no http.log. """ import pandas as pd @@ -293,6 +296,7 @@ def test_stage_order_deterministic_and_carve_hashes_backfilled(monkeypatch, tmp_ monkeypatch.setattr(R, "merge_zeek_dns", lambda zt, f: f) monkeypatch.setattr(R, "analyze_dns", lambda *a, **kw: {}) monkeypatch.setattr(R, "analyze_certificates", lambda *a, **kw: {}) + monkeypatch.setattr(R, "analyze_http", lambda *a, **kw: {}) monkeypatch.setattr(R, "rank_beaconing", lambda *a, **kw: pd.DataFrame()) monkeypatch.setattr(R, "carve_http_payloads", lambda *a, **kw: [{"sha256": carved_sha, "path": "x"}]) @@ -303,12 +307,191 @@ def test_stage_order_deterministic_and_carve_hashes_backfilled(monkeypatch, tmp_ progress=CallbackProgress(callback=lambda _e: None, total_phases=0), ) - canonical = ["dns_analysis", "tls_certs", "beacon", "carve"] + canonical = ["dns_analysis", "tls_certs", "http_analysis", "beacon", "carve"] observed = [s for s in result.stages_run if s in set(canonical)] assert observed == canonical, f"stage order not deterministic: {result.stages_run}" assert carved_sha in result.features["artifacts"]["hashes"], "carve sha256 backfill missing post-join" +def test_http_analysis_stage_wired_into_runner(monkeypatch, tmp_path): + """The HTTP analysis stage runs when Zeek ran and its result lands on + PipelineResult.http_analysis, stages_run, and to_dict(). It reads the uncapped + on-disk http.log path (zeek_log_paths["http.log"]), not the capped table.""" + import pandas as pd + + import app.pipeline.runner as R + from app.pipeline.progress import CallbackProgress + from app.pipeline.runner import PipelineOptions, run_pipeline + + pcap = tmp_path / "fake.pcap" + pcap.write_bytes(b"") + + http_log_path = str(tmp_path / "http.log") + seen_paths: list[str | None] = [] + + def _fake_analyze_http(zeek_tables, http_log_path=None, phase=None): + seen_paths.append(http_log_path) + return {"total_requests": 3} + + monkeypatch.setattr(R, "run_zeek", lambda p, d, phase=None: {"http.log": http_log_path}) + monkeypatch.setattr(R, "load_zeek_any", lambda p: pd.DataFrame({"host": ["evil.example"]})) + monkeypatch.setattr(R, "merge_zeek_dns", lambda zt, f: f) + monkeypatch.setattr(R, "analyze_http", _fake_analyze_http) + + result = run_pipeline( + pcap_path=str(pcap), + case_id="http_test", + options=PipelineOptions( + osint_enabled=False, + llm_enabled=False, + do_pyshark=False, + do_zeek=True, + do_carve=False, + do_yara=False, + pre_count=False, + ), + progress=CallbackProgress(callback=lambda _e: None, total_phases=0), + ) + + assert "http_analysis" in result.stages_run + assert result.http_analysis == {"total_requests": 3} + assert seen_paths == [http_log_path], "analyze_http must receive the uncapped on-disk http.log path" + + serialized = result.to_dict() + assert serialized["http_analysis"] == {"total_requests": 3} + + +def test_http_analysis_stage_runs_and_skips_when_no_http_log(monkeypatch, tmp_path): + """On an all-HTTPS / DNS-only capture (Zeek ran, no http.log), the HTTP stage + still runs — mirroring DNS/TLS — so the UI phase always starts and completes. + + The gate is `do_zeek and zeek_tables` (identical to DNS/TLS), NOT + `"http.log" in zeek_tables`; a tighter gate would leave the registered + "HTTP Analysis" phase uncreated and the Overall progress bar stuck below 100%. + analyze_http runs here with http_log_path=None (no http.log key) and returns + {"skipped": True}, so http_analysis appears in stages_run with a skipped result. + """ + import pandas as pd + + import app.pipeline.runner as R + from app.pipeline.progress import CallbackProgress + from app.pipeline.runner import PipelineOptions, run_pipeline + + pcap = tmp_path / "fake.pcap" + pcap.write_bytes(b"") + + seen_paths: list[str | None] = [] + + def _fake_analyze_http(zeek_tables, http_log_path=None, phase=None): + seen_paths.append(http_log_path) + return {"skipped": True} + + monkeypatch.setattr(R, "run_zeek", lambda p, d, phase=None: {"dns.log": str(tmp_path / "d.log")}) + monkeypatch.setattr(R, "load_zeek_any", lambda p: pd.DataFrame({"query": ["a.com"]})) + monkeypatch.setattr(R, "merge_zeek_dns", lambda zt, f: f) + monkeypatch.setattr(R, "analyze_dns", lambda *a, **kw: {}) + monkeypatch.setattr(R, "analyze_http", _fake_analyze_http) + + result = run_pipeline( + pcap_path=str(pcap), + case_id="http_skip_test", + options=PipelineOptions( + osint_enabled=False, + llm_enabled=False, + do_pyshark=False, + do_zeek=True, + do_carve=False, + do_yara=False, + pre_count=False, + ), + progress=CallbackProgress(callback=lambda _e: None, total_phases=0), + ) + + assert "http_analysis" in result.stages_run, "HTTP phase must run whenever Zeek ran (DNS/TLS parity)" + assert result.http_analysis == {"skipped": True} + assert seen_paths == [None], "analyze_http should receive http_log_path=None when no http.log was produced" + + +def test_http_analysis_stage_not_scheduled_without_zeek_tables(monkeypatch, tmp_path): + """The HTTP stage must NOT run when the gate is false — i.e. Zeek produced no + logs at all (empty zeek_tables). It shares DNS/TLS's gate, so it is dormant in + exactly the same conditions they are.""" + import app.pipeline.runner as R + from app.pipeline.progress import CallbackProgress + from app.pipeline.runner import PipelineOptions, run_pipeline + + pcap = tmp_path / "fake.pcap" + pcap.write_bytes(b"") + + def _boom(*args, **kwargs): + raise AssertionError("analyze_http must not run when zeek_tables is empty") + + # Zeek runs but yields no logs → empty zeek_tables → dns/tls/http all dormant. + monkeypatch.setattr(R, "run_zeek", lambda p, d, phase=None: {}) + monkeypatch.setattr(R, "merge_zeek_dns", lambda zt, f: f) + monkeypatch.setattr(R, "analyze_http", _boom) + + result = run_pipeline( + pcap_path=str(pcap), + case_id="http_gate_test", + options=PipelineOptions( + osint_enabled=False, + llm_enabled=False, + do_pyshark=False, + do_zeek=True, + do_carve=False, + do_yara=False, + pre_count=False, + ), + progress=CallbackProgress(callback=lambda _e: None, total_phases=0), + ) + + assert "http_analysis" not in result.stages_run + assert "dns_analysis" not in result.stages_run # same gate — both dormant + assert result.http_analysis == {} + assert result.to_dict()["http_analysis"] == {} + + +def test_http_analysis_failure_recorded_as_warning(monkeypatch, tmp_path): + """An exception inside the HTTP stage must be recorded as a warning, not raised.""" + import pandas as pd + + import app.pipeline.runner as R + from app.pipeline.progress import CallbackProgress + from app.pipeline.runner import PipelineOptions, run_pipeline + + pcap = tmp_path / "fake.pcap" + pcap.write_bytes(b"") + + monkeypatch.setattr(R, "run_zeek", lambda p, d, phase=None: {"http.log": str(tmp_path / "http.log")}) + monkeypatch.setattr(R, "load_zeek_any", lambda p: pd.DataFrame({"host": ["evil.example"]})) + monkeypatch.setattr(R, "merge_zeek_dns", lambda zt, f: f) + + def _raise(*args, **kwargs): + raise RuntimeError("boom") + + monkeypatch.setattr(R, "analyze_http", _raise) + + result = run_pipeline( + pcap_path=str(pcap), + case_id="http_fail_test", + options=PipelineOptions( + osint_enabled=False, + llm_enabled=False, + do_pyshark=False, + do_zeek=True, + do_carve=False, + do_yara=False, + pre_count=False, + ), + progress=CallbackProgress(callback=lambda _e: None, total_phases=0), + ) + + assert "http_analysis" not in result.stages_run + assert R.WARNING_HTTP_ANALYSIS_FAILED in result.warnings + assert result.http_analysis == {} + + def _stub_stage_dirs(monkeypatch, tmp_path, zeek_logs=None): """Redirect config dirs to tmp_path and stub zeek/carve to capture their out-dir args. @@ -341,6 +524,7 @@ def _fake_carve(p, d, phase=None): monkeypatch.setattr(R, "merge_zeek_dns", lambda zt, f: f) monkeypatch.setattr(R, "analyze_dns", lambda *a, **kw: {}) monkeypatch.setattr(R, "analyze_certificates", lambda *a, **kw: {}) + monkeypatch.setattr(R, "analyze_http", lambda *a, **kw: {}) monkeypatch.setattr(R, "carve_http_payloads", _fake_carve) return captured From e54f369eb05d06a15b845bae7bc9d370b1d9a18d Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Tue, 14 Jul 2026 00:14:47 +0700 Subject: [PATCH 17/35] feat: add http cleartext-cred and suspicious-ua signals to correlation --- app/analysis/correlation.py | 55 ++++++++++++++++++++++-- app/main.py | 2 + tests/test_correlation.py | 86 +++++++++++++++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 3 deletions(-) diff --git a/app/analysis/correlation.py b/app/analysis/correlation.py index 98ad404..7221c77 100644 --- a/app/analysis/correlation.py +++ b/app/analysis/correlation.py @@ -28,6 +28,8 @@ "flow_asymmetry": 0.04, "shodan_vulns": 0.10, "shodan_exposure": 0.04, + "http_cleartext_cred": 0.10, + "http_suspicious_ua": 0.06, } VERDICT_THRESHOLDS = { @@ -78,6 +80,30 @@ def _get_verdict(score: float) -> str: return "low" +def _build_http_lookup(http_analysis: dict | None) -> dict[str, list[tuple[str, Any, float]]]: + """Build a Host-header -> [(signal_name, value, score), ...] lookup from HTTP analysis. + + The Zeek ``http.log`` Host header may be a raw IP or a domain, so this + lookup is matched against both IP and domain indicators by the callers + (``_collect_ip_signals`` / ``_collect_domain_signals``). + """ + lookup: dict[str, list[tuple[str, Any, float]]] = {} + if not http_analysis: + return lookup + + for cred in http_analysis.get("cleartext_credentials", []) or []: + host = cred.get("host") if isinstance(cred, dict) else None + if host: + lookup.setdefault(host, []).append(("http_cleartext_cred", cred.get("username", ""), 0.9)) + + for ua in http_analysis.get("suspicious_user_agents", []) or []: + host = ua.get("host") if isinstance(ua, dict) else None + if host: + lookup.setdefault(host, []).append(("http_suspicious_ua", ua.get("reason", ""), 0.6)) + + return lookup + + def _collect_ip_signals( ip: str, osint: dict, @@ -85,6 +111,7 @@ def _collect_ip_signals( tls_lookup: dict[str, list[str]], yara_ips: set[str], asymmetry_lookup: dict[str, float], + http_lookup: dict[str, list[tuple[str, Any, float]]], ) -> list[CorrelationSignal]: """Collect all signals for an IP indicator.""" signals: list[CorrelationSignal] = [] @@ -147,6 +174,10 @@ def _collect_ip_signals( CorrelationSignal("flow_asymmetry", round(asymmetry_lookup[ip], 2), asymmetry_lookup[ip], "flow_analysis") ) + # HTTP findings (Host header matched a raw IP) + for name, value, score in http_lookup.get(ip, []): + signals.append(CorrelationSignal(name, value, score, "http")) + return signals @@ -154,6 +185,7 @@ def _collect_domain_signals( domain: str, osint: dict, dns_analysis: dict, + http_lookup: dict[str, list[tuple[str, Any, float]]], ) -> list[CorrelationSignal]: """Collect all signals for a domain indicator.""" signals: list[CorrelationSignal] = [] @@ -182,13 +214,24 @@ def _collect_domain_signals( signals.append(CorrelationSignal("dns_tunneling", tunnel.get("score", 0), tunnel["score"], "dns")) break + # HTTP findings (Host header matched a domain) + for name, value, score in http_lookup.get(domain, []): + signals.append(CorrelationSignal(name, value, score, "http")) + return signals # Tier definitions — a strong OSINT hit alone can push to "high"; behavioural # signals need corroboration; contextual signals alone cap at "medium". _TIER1_DEFINITIVE = {"vt_detections", "greynoise_malicious"} -_TIER2_BEHAVIOURAL = {"beacon_score", "flow_asymmetry", "dns_tunneling", "dga_domain"} +_TIER2_BEHAVIOURAL = { + "beacon_score", + "flow_asymmetry", + "dns_tunneling", + "dga_domain", + "http_cleartext_cred", + "http_suspicious_ua", +} _TIER3_CONTEXTUAL = {"abuseipdb", "self_signed_cert", "expired_cert", "yara_match", "shodan_vulns", "shodan_exposure"} # Strong-signal floors — a single definitive signal sets a minimum score @@ -243,6 +286,7 @@ def correlate_indicators( tls_analysis: dict | None = None, yara_results: dict | None = None, asymmetry_results: list | None = None, + http_analysis: dict | None = None, ) -> list[CorrelationResult]: """ Correlate indicators across all analysis modules. @@ -255,6 +299,9 @@ def correlate_indicators( tls_analysis: TLS certificate analysis yara_results: YARA scan results asymmetry_results: Flow asymmetry results + http_analysis: HTTP analysis results (cleartext credentials, suspicious + user agents). The Host header may be an IP or a domain, so matches + are attributed to whichever indicator type it corresponds to. Returns: List of CorrelationResult sorted by composite score (highest first) @@ -301,12 +348,14 @@ def correlate_indicators( if dst: asymmetry_lookup[dst] = max(asymmetry_lookup.get(dst, 0), score) + http_lookup = _build_http_lookup(http_analysis) + results: list[CorrelationResult] = [] # Correlate IPs ips = [ip for ip in features.get("artifacts", {}).get("ips", []) if is_public_ipv4(ip)] for ip in ips: - signals = _collect_ip_signals(ip, osint, beacon_lookup, tls_lookup, yara_ips, asymmetry_lookup) + signals = _collect_ip_signals(ip, osint, beacon_lookup, tls_lookup, yara_ips, asymmetry_lookup, http_lookup) if not signals: continue composite = _compute_composite(signals) @@ -323,7 +372,7 @@ def correlate_indicators( # Correlate domains domains = features.get("artifacts", {}).get("domains", []) for domain in domains: - signals = _collect_domain_signals(domain, osint, dns_analysis) + signals = _collect_domain_signals(domain, osint, dns_analysis, http_lookup) if not signals: continue composite = _compute_composite(signals) diff --git a/app/main.py b/app/main.py index 881185b..3dce229 100644 --- a/app/main.py +++ b/app/main.py @@ -671,6 +671,7 @@ def _run_single_pcap_pipeline( dns_analysis=st.session_state.get("dns_analysis"), tls_analysis=st.session_state.get("tls_analysis"), yara_results=st.session_state.get("yara_results"), + http_analysis=st.session_state.get("http_analysis"), ) st.session_state["correlations"] = correlations @@ -763,6 +764,7 @@ def _run_single_pcap_pipeline( dns_analysis=st.session_state.get("dns_analysis"), tls_analysis=st.session_state.get("tls_analysis"), yara_results=st.session_state.get("yara_results"), + http_analysis=st.session_state.get("http_analysis"), ) st.session_state["correlations"] = correlations diff --git a/tests/test_correlation.py b/tests/test_correlation.py index f9d757a..2db4032 100644 --- a/tests/test_correlation.py +++ b/tests/test_correlation.py @@ -203,3 +203,89 @@ def test_correlate_shodan_transport_error_dict_ignored(): } results = correlate_indicators(features=features, osint=osint) assert results == [] + + +def test_correlate_http_cleartext_cred_lights_up_ip_indicator(): + """A cleartext-credential finding whose Host header is a raw IP should + attribute an http_cleartext_cred signal to that IP indicator.""" + features = { + "artifacts": {"ips": ["1.2.3.4"], "domains": []}, + "flows": [], + } + http_analysis = { + "cleartext_credentials": [{"host": "1.2.3.4", "uri": "/login", "username": "admin"}], + "suspicious_user_agents": [], + } + results = correlate_indicators(features=features, http_analysis=http_analysis) + assert len(results) == 1 + assert results[0].indicator == "1.2.3.4" + assert results[0].indicator_type == "ip" + sig = next(s for s in results[0].signals if s.name == "http_cleartext_cred") + assert sig.source == "http" + assert 0.0 < sig.score <= 1.0 + + +def test_correlate_http_cleartext_cred_lights_up_domain_indicator(): + """The same finding, but with a domain Host header, should attribute the + signal to the domain indicator instead (host may be IP OR domain).""" + features = { + "artifacts": {"ips": [], "domains": ["portal.evil-corp.test"]}, + "flows": [], + } + http_analysis = { + "cleartext_credentials": [{"host": "portal.evil-corp.test", "uri": "/login", "username": "admin"}], + "suspicious_user_agents": [], + } + results = correlate_indicators(features=features, http_analysis=http_analysis) + assert len(results) == 1 + assert results[0].indicator == "portal.evil-corp.test" + assert results[0].indicator_type == "domain" + assert any(s.name == "http_cleartext_cred" and s.source == "http" for s in results[0].signals) + + +def test_correlate_http_suspicious_ua_signal(): + features = { + "artifacts": {"ips": ["5.6.7.8"], "domains": []}, + "flows": [], + } + http_analysis = { + "cleartext_credentials": [], + "suspicious_user_agents": [ + {"host": "5.6.7.8", "user_agent": "sqlmap/1.6", "uri": "/", "reason": "known tool user-agent (sqlmap)"} + ], + } + results = correlate_indicators(features=features, http_analysis=http_analysis) + assert len(results) == 1 + sig = next(s for s in results[0].signals if s.name == "http_suspicious_ua") + assert sig.source == "http" + assert 0.0 < sig.score <= 1.0 + + +def test_correlate_http_cleartext_cred_scores_higher_than_suspicious_ua(): + """Cleartext creds are a stronger indicator than a suspicious UA alone.""" + features = {"artifacts": {"ips": ["9.9.9.9"], "domains": []}, "flows": []} + http_analysis = { + "cleartext_credentials": [{"host": "9.9.9.9", "uri": "/login", "username": "admin"}], + "suspicious_user_agents": [{"host": "9.9.9.9", "user_agent": "sqlmap/1.6", "uri": "/", "reason": "known tool"}], + } + results = correlate_indicators(features=features, http_analysis=http_analysis) + cred_sig = next(s for s in results[0].signals if s.name == "http_cleartext_cred") + ua_sig = next(s for s in results[0].signals if s.name == "http_suspicious_ua") + assert cred_sig.score > ua_sig.score + + +def test_correlate_without_http_analysis_has_no_http_signals(): + """Omitting http_analysis (the pre-3.5 default) must not change behaviour.""" + features = {"artifacts": {"ips": ["8.8.8.8"], "domains": []}, "flows": []} + osint = {"ips": {"8.8.8.8": {"greynoise": {"classification": "malicious"}}}, "domains": {}} + results = correlate_indicators(features=features, osint=osint) + assert len(results) == 1 + assert not any(s.name.startswith("http_") for s in results[0].signals) + + +def test_correlate_http_analysis_none_explicit_is_noop(): + features = {"artifacts": {"ips": ["8.8.8.8"], "domains": []}, "flows": []} + osint = {"ips": {"8.8.8.8": {"greynoise": {"classification": "malicious"}}}, "domains": {}} + results = correlate_indicators(features=features, osint=osint, http_analysis=None) + assert len(results) == 1 + assert not any(s.name.startswith("http_") for s in results[0].signals) From 92e88e329587e1416e77021b4d9609634810531b Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Tue, 14 Jul 2026 00:47:06 +0700 Subject: [PATCH 18/35] feat: render http analysis in raw data tab and persist via ui/api paths --- app/api/queue.py | 5 ++ app/main.py | 7 +++ app/pipeline/batch.py | 3 ++ app/ui/cases_tab.py | 5 ++ app/ui/layout.py | 104 +++++++++++++++++++++++++++++++++++++ tests/api/test_queue.py | 45 ++++++++++++++++ tests/test_batch.py | 13 +++++ tests/test_case_restore.py | 15 ++++++ tests/test_layout.py | 103 ++++++++++++++++++++++++++++++++++++ 9 files changed, 300 insertions(+) diff --git a/app/api/queue.py b/app/api/queue.py index bdfcaab..4ef9a55 100644 --- a/app/api/queue.py +++ b/app/api/queue.py @@ -186,6 +186,11 @@ def _persist_analysis( analysis.mitre_techniques = result.mitre_techniques if result.beacon_df_records: analysis.features["beacon_records"] = result.beacon_df_records + if result.http_analysis: + # Analysis has no dedicated http_analysis column — stash it in + # features (mirroring beacon_records above); features_json is + # compressed+persisted, and the UI restore path reads it back out. + analysis.features["http_analysis"] = result.http_analysis analysis.iocs = repo.extract_iocs(analysis) result.analysis_id = repo.save_analysis(analysis) except Exception: diff --git a/app/main.py b/app/main.py index 3dce229..3c4f8cd 100644 --- a/app/main.py +++ b/app/main.py @@ -55,6 +55,7 @@ render_dns_analysis, render_flow_asymmetry, render_flows, + render_http_analysis, render_hunting_checklist, render_ioc_search, render_ja3, @@ -227,6 +228,7 @@ def _run_single_pcap_pipeline( beacon_df=beacon_df if isinstance(beacon_df, pd.DataFrame) else None, dns_analysis=result.dns_analysis or {}, tls_analysis=result.tls_analysis or {}, + http_analysis=result.http_analysis or {}, attack_mapping=result.attack_mapping or {}, packet_count=result.packet_count, ) @@ -629,6 +631,9 @@ def _run_single_pcap_pipeline( # it) — mirror the first successful file's mapping, same fallback used # for merged_features above. st.session_state["attack_mapping"] = (first_ok.attack_mapping if first_ok else None) or {} + # No cross-file HTTP aggregation helper either (see attack_mapping + # above) — mirror the first successful file's HTTP findings. + st.session_state["http_analysis"] = (first_ok.http_analysis if first_ok else None) or {} # Carved payloads concatenated across all successful files st.session_state["carved"] = [ item for r in batch_result.pcap_results if not r.error for item in r.carved_items @@ -735,6 +740,7 @@ def _run_single_pcap_pipeline( st.session_state["carved"] = result.carved_items st.session_state["dns_analysis"] = result.dns_analysis or None st.session_state["tls_analysis"] = result.tls_analysis or None + st.session_state["http_analysis"] = result.http_analysis or None st.session_state["attack_mapping"] = result.attack_mapping or {} _precompute_dash_aggregates(features.get("flows")) @@ -1534,6 +1540,7 @@ def _run_single_pcap_pipeline( feats = st.session_state.get("features") or {} render_flows(results_panel, feats.get("flows")) render_dns_analysis(results_panel, st.session_state.get("dns_analysis")) + render_http_analysis(results_panel, st.session_state.get("http_analysis")) render_tls_certificates(results_panel, st.session_state.get("tls_analysis")) render_ja3( results_panel, diff --git a/app/pipeline/batch.py b/app/pipeline/batch.py index 7cfd98c..6d8cf60 100644 --- a/app/pipeline/batch.py +++ b/app/pipeline/batch.py @@ -85,6 +85,9 @@ class PCAPResult: beacon_df: pd.DataFrame | None = None dns_analysis: dict[str, Any] = field(default_factory=dict) tls_analysis: dict[str, Any] = field(default_factory=dict) + # HTTP request analysis (suspicious UA/cleartext creds/suspicious URI + # heuristics) from app.pipeline.http_analysis.analyze_http. + http_analysis: dict[str, Any] = field(default_factory=dict) # MITRE ATT&CK mapping (AttackMapping.to_dict() shape) from the runner — # may be {} on mapper failure; treated shape-agnostically downstream. attack_mapping: dict = field(default_factory=dict) diff --git a/app/ui/cases_tab.py b/app/ui/cases_tab.py index 6760e03..25fb258 100644 --- a/app/ui/cases_tab.py +++ b/app/ui/cases_tab.py @@ -86,6 +86,11 @@ def _restore_analysis_to_session(analysis: Analysis) -> None: st.session_state["osint"] = analysis.osint or {} st.session_state["dns_analysis"] = analysis.dns_analysis st.session_state["tls_analysis"] = analysis.tls_analysis + # http_analysis has no dedicated Analysis column — the API path stashes it + # inside features (mirroring beacon_records; see app/api/queue.py). Falls + # back to None (not {}) when absent, matching dns/tls_analysis's "didn't + # run" semantics rather than a false "ran clean". + st.session_state["http_analysis"] = features.get("http_analysis") st.session_state["yara_results"] = analysis.yara_results st.session_state["attack_mapping"] = analysis.attack_mapping or {} # Model default for report is "" but the app's no-report sentinel is None. diff --git a/app/ui/layout.py b/app/ui/layout.py index dabfd3c..e3023b5 100644 --- a/app/ui/layout.py +++ b/app/ui/layout.py @@ -1766,6 +1766,110 @@ def render_dns_analysis(result_col, dns_analysis: dict | None): st.dataframe(df_top, width="stretch", hide_index=True) +def render_http_analysis(result_col, http_analysis: dict | None): + """Render HTTP request analysis: suspicious User-Agents, cleartext credentials, suspicious URIs.""" + with result_col: + expanded = bool( + http_analysis is not None + and ( + http_analysis.get("alerts", {}).get("cleartext_cred_count", 0) + or http_analysis.get("alerts", {}).get("suspicious_uri_count", 0) + or http_analysis.get("alerts", {}).get("suspicious_ua_count", 0) + ) + ) + with st.expander("HTTP Analysis", expanded=expanded): + # analyze_http always returns a truthy dict when it executes, so + # None means the stage was skipped or failed — never "ran clean". + if http_analysis is None: + st.info( + "📭 HTTP analysis didn't run in this session (stage skipped or failed) — re-run with Zeek enabled." + ) + return + if http_analysis.get("skipped"): + st.info("📭 HTTP analysis was skipped for this run.") + return + if http_analysis.get("error"): + st.info(f"📭 HTTP analysis: {http_analysis['error']}") + return + + alerts = http_analysis.get("alerts", {}) + ua_count = alerts.get("suspicious_ua_count", 0) + cred_count = alerts.get("cleartext_cred_count", 0) + uri_count = alerts.get("suspicious_uri_count", 0) + + # Summary metrics + col1, col2, col3, col4, col5 = st.columns(5) + with col1: + st.metric("HTTP Requests", http_analysis.get("total_requests", 0)) + with col2: + st.metric("Unique Hosts", http_analysis.get("unique_hosts", 0)) + with col3: + if ua_count: + st.metric("Suspicious UAs", ua_count, delta="Warning", delta_color="inverse") + else: + st.metric("Suspicious UAs", 0) + with col4: + if cred_count: + st.metric("Cleartext Creds", cred_count, delta="Warning", delta_color="inverse") + else: + st.metric("Cleartext Creds", 0) + with col5: + if uri_count: + st.metric("Suspicious URIs", uri_count, delta="Warning", delta_color="inverse") + else: + st.metric("Suspicious URIs", 0) + + # Alert banners + if cred_count: + st.error(f"**Cleartext Credentials:** {cred_count} Basic-auth username(s) seen in plaintext HTTP!") + if uri_count: + st.error(f"**Suspicious URIs:** {uri_count} risky/oversized/raw-IP-hosted URIs detected!") + if ua_count: + st.warning(f"**Suspicious User-Agents:** {ua_count} missing/known-tool user-agents detected!") + + # Tabs for detailed data + tab_creds, tab_ua, tab_uri = st.tabs(["Cleartext Credentials", "Suspicious User-Agents", "Suspicious URIs"]) + + with tab_creds: + creds_list = http_analysis.get("cleartext_credentials", []) + if creds_list: + df_creds = pd.DataFrame(creds_list) + display_cols = ["host", "uri", "username"] + display_cols = [c for c in display_cols if c in df_creds.columns] + render_export_buttons( + df_creds[display_cols], "http_cleartext_creds", key_suffix="http_creds", is_dataframe=True + ) + st.dataframe(df_creds[display_cols], width="stretch", hide_index=True) + else: + st.caption("No cleartext credentials detected.") + + with tab_ua: + ua_list = http_analysis.get("suspicious_user_agents", []) + if ua_list: + df_ua = pd.DataFrame(ua_list) + display_cols = ["host", "user_agent", "uri", "reason"] + display_cols = [c for c in display_cols if c in df_ua.columns] + render_export_buttons( + df_ua[display_cols], "http_suspicious_ua", key_suffix="http_ua", is_dataframe=True + ) + st.dataframe(df_ua[display_cols], width="stretch", hide_index=True) + else: + st.caption("No suspicious user-agents detected.") + + with tab_uri: + uri_list = http_analysis.get("suspicious_uris", []) + if uri_list: + df_uri = pd.DataFrame(uri_list) + display_cols = ["host", "uri", "reason"] + display_cols = [c for c in display_cols if c in df_uri.columns] + render_export_buttons( + df_uri[display_cols], "http_suspicious_uri", key_suffix="http_uri", is_dataframe=True + ) + st.dataframe(df_uri[display_cols], width="stretch", hide_index=True) + else: + st.caption("No suspicious URIs detected.") + + def render_tls_certificates(result_col, tls_analysis: dict | None): """Render TLS certificate analysis results.""" with result_col: diff --git a/tests/api/test_queue.py b/tests/api/test_queue.py index cdcbe46..883899f 100644 --- a/tests/api/test_queue.py +++ b/tests/api/test_queue.py @@ -349,6 +349,51 @@ def fake_run_pipeline(pcap_path, case_id, options, progress, heartbeat=None): assert persisted.attack_mapping == mapping, "attack_mapping must be persisted on the Analysis row" +def test_worker_http_analysis_round_trip(tmp_path, monkeypatch): + """http_analysis has no dedicated Analysis column — it must be stashed in + features['http_analysis'] (mirroring the beacon_records precedent) and + survive persistence.""" + import app.pipeline.runner as runner_mod + from app.pipeline.runner import PipelineResult + + http_analysis = { + "total_requests": 5, + "unique_hosts": 3, + "alerts": {"cleartext_cred_count": 1, "suspicious_ua_count": 0, "suspicious_uri_count": 0}, + "cleartext_credentials": [{"host": "10.0.0.5", "uri": "/admin", "username": "admin"}], + } + + def fake_run_pipeline(pcap_path, case_id, options, progress, heartbeat=None): + return PipelineResult( + case_id=case_id, + packet_count=1, + features={ + "flows": [{"src": "10.0.0.1", "dst": "8.8.8.8", "proto": "TCP", "count": 9}], + "artifacts": {"ips": ["10.0.0.1", "8.8.8.8"], "domains": [], "urls": [], "hashes": [], "ja3": []}, + }, + http_analysis=dict(http_analysis), + ) + + # _worker_run imports run_pipeline from app.pipeline.runner at call time, + # so the patch must target the source module, not queue_mod. + monkeypatch.setattr(runner_mod, "run_pipeline", fake_run_pipeline) + + fake_pcap = tmp_path / "fake.pcap" + fake_pcap.write_bytes(b"\xd4\xc3\xb2\xa1" + b"\x00" * 20) + + db = str(tmp_path / "t.db") + repo = CaseRepository(db_path=db) + repo.create_case(Case(id="cafe0031", title="t", status=CaseStatus.IN_PROGRESS, severity=Severity.LOW)) + job_id = repo.create_job(Job(case_id="cafe0031", pcap_path=str(fake_pcap), options_json="{}")) + + _worker_run(job_id, db, str(fake_pcap), {"osint_enabled": False, "llm_enabled": False}) + + result = json.loads(repo.get_job(job_id).result_json) + assert result["analysis_id"], "persistence must succeed with the faked pipeline result" + persisted = repo.get_analysis(result["analysis_id"]) + assert persisted.features["http_analysis"] == http_analysis + + # --------------------------------------------------------------------------- # Task 3: progress reconciliation on completion # --------------------------------------------------------------------------- diff --git a/tests/test_batch.py b/tests/test_batch.py index 315a477..f27d960 100644 --- a/tests/test_batch.py +++ b/tests/test_batch.py @@ -174,6 +174,19 @@ def test_attack_mapping_accepts_mapping(self): result = PCAPResult(path="/data/test.pcap", filename="test.pcap", attack_mapping=mapping) assert result.attack_mapping == mapping + def test_http_analysis_defaults_empty(self): + result = PCAPResult(path="/data/test.pcap", filename="test.pcap") + assert result.http_analysis == {} + + def test_http_analysis_accepts_dict(self): + http_analysis = { + "total_requests": 5, + "unique_hosts": 3, + "alerts": {"cleartext_cred_count": 1, "suspicious_ua_count": 0, "suspicious_uri_count": 0}, + } + result = PCAPResult(path="/data/test.pcap", filename="test.pcap", http_analysis=http_analysis) + assert result.http_analysis == http_analysis + class TestMergeZeekTables: """Test merging Zeek tables from multiple PCAPs.""" diff --git a/tests/test_case_restore.py b/tests/test_case_restore.py index 001d0fb..78d42c1 100644 --- a/tests/test_case_restore.py +++ b/tests/test_case_restore.py @@ -152,6 +152,21 @@ def test_none_dns_and_tls_passed_through(self): assert st.session_state["dns_analysis"] is None assert st.session_state["tls_analysis"] is None + def test_http_analysis_restored_from_features_stash(self): + """http_analysis has no dedicated Analysis column — it's stashed inside + features (mirroring how beacon_records is stashed, see app/api/queue.py) + and must be pulled back out on restore.""" + analysis = _make_analysis(features={"flows": [], "http_analysis": {"total_requests": 3}}) + _restore_analysis_to_session(analysis) + assert st.session_state["http_analysis"] == {"total_requests": 3} + + def test_http_analysis_none_when_absent_from_features(self): + """A case saved before http_analysis existed (or via the UI quick-save + path, which doesn't stash it) must not surface stale/wrong data.""" + analysis = _make_analysis() # default features has no http_analysis key + _restore_analysis_to_session(analysis) + assert st.session_state["http_analysis"] is None + class TestDetailViewAutoRestoreGuard: """The case-detail auto-restore must be a one-shot keyed on restored_analysis_id. diff --git a/tests/test_layout.py b/tests/test_layout.py index 962463b..07aa954 100644 --- a/tests/test_layout.py +++ b/tests/test_layout.py @@ -62,6 +62,109 @@ def test_empty_dict_renders_without_exception(self): assert not at.exception +# Production-shape dict, as stored in st.session_state["http_analysis"] +# (analyze_http() output from app/pipeline/http_analysis.py). +PROD_HTTP_ANALYSIS_DICT = { + "total_requests": 7, + "unique_hosts": 5, + "methods": {"GET": 6, "POST": 1}, + "status_codes": {"200": 6, "401": 1}, + "suspicious_user_agents": [ + {"host": "10.0.0.5", "user_agent": "", "uri": "/", "reason": "missing user-agent"}, + { + "host": "10.0.0.6", + "user_agent": "python-requests/2.31", + "uri": "/api", + "reason": "known tool user-agent (python-requests)", + }, + ], + "cleartext_credentials": [ + {"host": "10.0.0.5", "uri": "/admin", "username": "admin"}, + ], + "suspicious_uris": [ + {"host": "10.0.0.7", "uri": "/payload.exe", "reason": "risky file extension (.exe)"}, + { + "host": "203.0.113.9", + "uri": "/files/tool.dll", + "reason": "risky file extension (.dll); raw-IP host serving file download", + }, + {"host": "10.0.0.8", "uri": "/" + "a" * 600, "reason": "long URI (605 chars)"}, + ], + "alerts": { + "suspicious_ua_count": 2, + "cleartext_cred_count": 1, + "suspicious_uri_count": 3, + }, +} + + +def _render_http_analysis_app(): + import streamlit as st + + from app.ui.layout import render_http_analysis + + render_http_analysis(st.container(), st.session_state.get("http_analysis")) + + +class TestRenderHttpAnalysis: + def test_production_shape_dict_renders_without_exception(self): + at = AppTest.from_function(_render_http_analysis_app, default_timeout=30) + at.session_state["http_analysis"] = PROD_HTTP_ANALYSIS_DICT + at.run() + + assert not at.exception + expanders = [e for e in at.expander if "HTTP Analysis" in (e.label or "")] + assert expanders, "Raw Data tab should render an HTTP Analysis expander" + assert expanders[0].proto.expanded, "alerts present -> expander should default to expanded" + + def test_none_shows_info_and_returns(self): + at = AppTest.from_function(_render_http_analysis_app, default_timeout=30) + at.session_state["http_analysis"] = None + at.run() + + assert not at.exception + assert at.info, "None http_analysis should render an info message, not crash" + + def test_skipped_shows_info(self): + at = AppTest.from_function(_render_http_analysis_app, default_timeout=30) + at.session_state["http_analysis"] = {"skipped": True} + at.run() + + assert not at.exception + assert at.info + + def test_error_shows_info(self): + at = AppTest.from_function(_render_http_analysis_app, default_timeout=30) + at.session_state["http_analysis"] = {"error": "No HTTP log data", "records": 0} + at.run() + + assert not at.exception + assert at.info + + def test_clean_run_no_alerts_not_expanded_and_shows_captions(self): + at = AppTest.from_function(_render_http_analysis_app, default_timeout=30) + at.session_state["http_analysis"] = { + "total_requests": 3, + "unique_hosts": 2, + "methods": {"GET": 3}, + "status_codes": {"200": 3}, + "suspicious_user_agents": [], + "cleartext_credentials": [], + "suspicious_uris": [], + "alerts": {"suspicious_ua_count": 0, "cleartext_cred_count": 0, "suspicious_uri_count": 0}, + } + at.run() + + assert not at.exception + expanders = [e for e in at.expander if "HTTP Analysis" in (e.label or "")] + assert expanders + assert not expanders[0].proto.expanded, "no alerts -> expander should default to collapsed" + captions = [c.value for c in at.caption] + assert any("No cleartext credentials" in c for c in captions) + assert any("No suspicious user-agents" in c for c in captions) + assert any("No suspicious URIs" in c for c in captions) + + def _make_assets(tmp_path, *names): for name in names: (tmp_path / name).write_bytes(b"\x89PNG") From 069931a7d3715905c638583d90d6bfa9d05787b7 Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Tue, 14 Jul 2026 01:02:54 +0700 Subject: [PATCH 19/35] fix: stash http_analysis and beacon_records on ui-saved analyses for restore parity --- app/ui/cases_tab.py | 34 +++++++++++- tests/test_case_restore.py | 4 +- tests/test_cases_tab.py | 109 ++++++++++++++++++++++++++++++++++--- 3 files changed, 134 insertions(+), 13 deletions(-) diff --git a/app/ui/cases_tab.py b/app/ui/cases_tab.py index 25fb258..96c3b24 100644 --- a/app/ui/cases_tab.py +++ b/app/ui/cases_tab.py @@ -65,6 +65,36 @@ def _session_mitre_techniques() -> list[str]: ] +def _session_analysis_features() -> dict: + """Build the ``features`` dict for a UI-saved ``Analysis``, stashing http_analysis/beacon_records. + + Mirrors how ``app/api/queue.py:_persist_analysis`` stashes + ``result.http_analysis`` and ``result.beacon_df_records`` into + ``result.features`` before saving -- ``Analysis`` has no dedicated column + for either, and the case-restore path (``_restore_analysis_to_session``) + reads them back out of ``features``. Without this, a case saved through + the UI's manual-save paths (``_quick_save_analysis`` / + ``_add_current_analysis_to_case``) would lose its HTTP findings + (cleartext creds / suspicious UA / suspicious URIs) and beacon records on + restore, even though the API-saved path keeps them. + + Returns a shallow copy of ``st.session_state["features"]`` -- the live + session dict is never mutated in place, since it may still be read by + other code in the same rerun. + + Returns: + A features dict with ``http_analysis`` and ``beacon_records`` set + from session state (``None``/``[]`` respectively when absent). + """ + features = dict(st.session_state.get("features") or {}) + features["http_analysis"] = st.session_state.get("http_analysis") + beacon_df = st.session_state.get("beacon_df") + features["beacon_records"] = ( + beacon_df.to_dict("records") if isinstance(beacon_df, pd.DataFrame) and not beacon_df.empty else [] + ) + return features + + def _restore_analysis_to_session(analysis: Analysis) -> None: """Load a saved analysis back into session state for the Dashboard/Results tabs. @@ -623,7 +653,7 @@ def _quick_save_analysis(): case_id=case_id, pcap_path=st.session_state.get("__pcap_path", ""), packet_count=st.session_state.get("__total_pkts", 0), - features=features, + features=_session_analysis_features(), osint=st.session_state.get("osint") or {}, report=st.session_state.get("report") or "", yara_results=st.session_state.get("yara_results"), @@ -658,7 +688,7 @@ def _add_current_analysis_to_case(case: Case): case_id=case.id, pcap_path=st.session_state.get("__pcap_path", ""), packet_count=st.session_state.get("__total_pkts", 0), - features=features, + features=_session_analysis_features(), osint=st.session_state.get("osint") or {}, report=st.session_state.get("report") or "", yara_results=st.session_state.get("yara_results"), diff --git a/tests/test_case_restore.py b/tests/test_case_restore.py index 78d42c1..cac84f4 100644 --- a/tests/test_case_restore.py +++ b/tests/test_case_restore.py @@ -161,8 +161,8 @@ def test_http_analysis_restored_from_features_stash(self): assert st.session_state["http_analysis"] == {"total_requests": 3} def test_http_analysis_none_when_absent_from_features(self): - """A case saved before http_analysis existed (or via the UI quick-save - path, which doesn't stash it) must not surface stale/wrong data.""" + """A case saved before http_analysis existed (its features dict has no + such key) must not surface stale/wrong data.""" analysis = _make_analysis() # default features has no http_analysis key _restore_analysis_to_session(analysis) assert st.session_state["http_analysis"] is None diff --git a/tests/test_cases_tab.py b/tests/test_cases_tab.py index 82563c3..5f182dd 100644 --- a/tests/test_cases_tab.py +++ b/tests/test_cases_tab.py @@ -1,20 +1,35 @@ """Tests for app/ui/cases_tab.py. -Focused on ``_session_mitre_techniques``, the helper that derives the -``Analysis.mitre_techniques`` list from ``st.session_state["attack_mapping"]`` -for the two UI manual-save paths (``_quick_save_analysis`` and -``_add_current_analysis_to_case``). Without this, analyses saved through the -UI ship a permanently-empty ``mitre_techniques`` field even though the API -path (``app/api/queue.py``) populates it from ``result.mitre_techniques`` -- -both share the same ``data/cases.db``, so the IOC feed's ``mitre_techniques`` -column would be inconsistent depending on which path saved the case. +Focused on two session-state-derived helpers used by the UI manual-save paths +(``_quick_save_analysis`` and ``_add_current_analysis_to_case``), both of +which build an ``Analysis(...)`` from ``st.session_state`` rather than a +``PipelineResult`` (the API path's source, see ``app/api/queue.py``): + +- ``_session_mitre_techniques`` derives ``Analysis.mitre_techniques`` from + ``st.session_state["attack_mapping"]``. Without this, analyses saved + through the UI ship a permanently-empty ``mitre_techniques`` field even + though the API path populates it from ``result.mitre_techniques`` -- both + share the same ``data/cases.db``, so the IOC feed's ``mitre_techniques`` + column would be inconsistent depending on which path saved the case. + +- ``_session_analysis_features`` builds the ``features`` dict passed to + ``Analysis(...)``, stashing ``http_analysis`` and ``beacon_records`` into + it exactly as ``app/api/queue.py:_persist_analysis`` stashes them onto + ``result.features`` (neither has a dedicated ``Analysis`` column). Without + this, a case saved through the UI loses its HTTP findings (cleartext + creds / suspicious UA / suspicious URIs) and beacon records on restore, + while the same case saved via the API keeps them. Uses Streamlit's AppTest harness (see ``tests/test_config_ui.py`` for the -established pattern) since the helper reads ``st.session_state`` directly. +established pattern) since the helpers read ``st.session_state`` directly. """ +import pandas as pd +import streamlit as st from streamlit.testing.v1 import AppTest +from app.ui.cases_tab import _session_analysis_features + def _mitre_helper_app(): import streamlit as st @@ -65,3 +80,79 @@ def test_skips_techniques_missing_technique_id(self): at.run() assert at.session_state["__result"] == ["T1105"] + + +class TestSessionAnalysisFeatures: + """``_session_analysis_features`` builds the ``features`` dict for both UI + manual-save paths, stashing ``http_analysis``/``beacon_records`` exactly as + ``app/api/queue.py:_persist_analysis`` stashes them onto ``result.features`` + (see module docstring). Uses bare ``st.session_state`` directly (no AppTest + script-run context needed), matching ``tests/test_case_restore.py``. + """ + + def setup_method(self): + st.session_state.clear() + + def test_stashes_http_analysis_from_session(self): + st.session_state["features"] = {"flows": []} + st.session_state["http_analysis"] = {"cleartext_creds": ["user:pass@1.2.3.4"]} + + result = _session_analysis_features() + + assert result["http_analysis"] == {"cleartext_creds": ["user:pass@1.2.3.4"]} + + def test_http_analysis_none_when_absent_from_session(self): + st.session_state["features"] = {"flows": []} + + result = _session_analysis_features() + + assert result["http_analysis"] is None + + def test_stashes_beacon_records_from_beacon_df(self): + st.session_state["features"] = {"flows": []} + st.session_state["beacon_df"] = pd.DataFrame([{"src": "10.0.0.5", "dst": "203.0.113.9", "score": 0.87}]) + + result = _session_analysis_features() + + assert result["beacon_records"] == [{"src": "10.0.0.5", "dst": "203.0.113.9", "score": 0.87}] + + def test_beacon_records_empty_when_beacon_df_missing(self): + st.session_state["features"] = {"flows": []} + + result = _session_analysis_features() + + assert result["beacon_records"] == [] + + def test_beacon_records_empty_when_beacon_df_empty(self): + st.session_state["features"] = {"flows": []} + st.session_state["beacon_df"] = pd.DataFrame() + + result = _session_analysis_features() + + assert result["beacon_records"] == [] + + def test_preserves_other_feature_keys(self): + st.session_state["features"] = {"flows": [{"src": "1.1.1.1"}]} + + result = _session_analysis_features() + + assert result["flows"] == [{"src": "1.1.1.1"}] + + def test_does_not_mutate_live_session_features_dict(self): + """The live ``st.session_state["features"]`` dict must not gain the + stash keys in place -- it may be read elsewhere in the same rerun, and + an in-place mutation would leak the stash into unrelated readers.""" + original = {"flows": []} + st.session_state["features"] = original + + _session_analysis_features() + + assert "http_analysis" not in original + assert "beacon_records" not in original + assert st.session_state["features"] is original + + def test_missing_features_yields_stash_only(self): + result = _session_analysis_features() + + assert result["http_analysis"] is None + assert result["beacon_records"] == [] From 2ab7643df0c7ee1f510d95c39646954ddbf66ca8 Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Tue, 14 Jul 2026 01:21:57 +0700 Subject: [PATCH 20/35] feat: add case crud api (list, patch, notes) --- app/api/app.py | 2 +- app/api/models.py | 28 +++++ app/api/routers/cases.py | 99 +++++++++++++++- tests/api/test_cases.py | 243 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 369 insertions(+), 3 deletions(-) diff --git a/app/api/app.py b/app/api/app.py index 4a22bbf..dee1239 100644 --- a/app/api/app.py +++ b/app/api/app.py @@ -196,7 +196,7 @@ def create_app() -> FastAPI: CORSMiddleware, allow_origins=settings.cors_origins, allow_credentials=False, - allow_methods=["GET", "POST", "DELETE"], + allow_methods=["GET", "POST", "PATCH", "DELETE"], allow_headers=["Authorization", "Content-Type", "If-None-Match", "X-Request-ID"], ) diff --git a/app/api/models.py b/app/api/models.py index a6508a8..7e5869b 100644 --- a/app/api/models.py +++ b/app/api/models.py @@ -73,6 +73,34 @@ class IOCFeedResponse(BaseModel): next_cursor: str | None = None +class CaseListItem(BaseModel): + """Light case shape for GET /api/v1/cases — no embedded analyses/notes.""" + + id: str + title: str + description: str = "" + status: str + severity: str + created_at: str | None = None + updated_at: str | None = None + closed_at: str | None = None + tags: list[str] = Field(default_factory=list) + + +class CasePatchRequest(BaseModel): + """Partial update for PATCH /api/v1/cases/{id} — all fields optional.""" + + title: str | None = Field(default=None, min_length=1, max_length=200) + description: str | None = Field(default=None, max_length=5000) + status: str | None = Field(default=None, pattern=r"^(open|in_progress|closed)$") + severity: str | None = Field(default=None, pattern=r"^(low|medium|high|critical)$") + tags: list[str] | None = None + + +class NoteRequest(BaseModel): + content: str = Field(..., min_length=1) + + class PcapSubmissionForm(BaseModel): """Multipart form fields for POST /pcaps.""" diff --git a/app/api/routers/cases.py b/app/api/routers/cases.py index 3bbbc3c..a44689e 100644 --- a/app/api/routers/cases.py +++ b/app/api/routers/cases.py @@ -1,4 +1,4 @@ -"""GET /api/v1/cases/{id}, DELETE, /report.pdf""" +"""Case CRUD: GET (get/list), PATCH, DELETE, /report.pdf, and notes.""" from __future__ import annotations @@ -10,11 +10,13 @@ import shutil import tempfile -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Query from fastapi.responses import FileResponse, JSONResponse, Response from app.api.deps import get_repo, require_full_scope +from app.api.models import CaseListItem, CasePatchRequest, NoteRequest from app.api.queue import cancel_queued_job +from app.database.models import CaseStatus, Severity logger = logging.getLogger(__name__) @@ -27,6 +29,39 @@ def _reports_dir() -> pathlib.Path: ) +# Registered BEFORE the "/{case_id}" routes below — a literal "" path never +# actually collides with "/{case_id}" (different segment counts), but keeping +# the list route first avoids relying on that being true forever. +@router.get("") +def list_cases( + status: str | None = Query(default=None), + tag: str | None = Query(default=None), + search: str | None = Query(default=None), + limit: int = Query(default=100, ge=1, le=500), + offset: int = Query(default=0, ge=0), + _scope=Depends(require_full_scope), + repo=Depends(get_repo), +): + status_filter = CaseStatus.from_str(status) if status else None + tags_filter = [tag] if tag else None + cases = repo.list_cases(status=status_filter, tags=tags_filter, search=search, limit=limit, offset=offset) + items = [ + CaseListItem( + id=c.id, + title=c.title, + description=c.description, + status=c.status.value, + severity=c.severity.value, + created_at=c.created_at.isoformat() if c.created_at else None, + updated_at=c.updated_at.isoformat() if c.updated_at else None, + closed_at=c.closed_at.isoformat() if c.closed_at else None, + tags=c.tags, + ) + for c in cases + ] + return {"cases": [item.model_dump() for item in items], "count": len(items)} + + @router.get("/{case_id}") def get_case(case_id: str, _scope=Depends(require_full_scope), repo=Depends(get_repo)): case = repo.get_case(case_id) @@ -176,3 +211,63 @@ def delete_case(case_id: str, _scope=Depends(require_full_scope), repo=Depends(g logger.warning("Artifact cleanup after deleting case %s failed: %s", case_id, exc) return Response(status_code=204) + + +@router.patch("/{case_id}") +def patch_case( + case_id: str, + body: CasePatchRequest, + _scope=Depends(require_full_scope), + repo=Depends(get_repo), +): + case = repo.get_case(case_id) + if not case: + raise HTTPException(status_code=404, detail="case_not_found") + + if body.title is not None: + case.title = body.title + if body.description is not None: + case.description = body.description + if body.severity is not None: + case.severity = Severity.from_str(body.severity) + if body.tags is not None: + case.tags = body.tags + + if body.status is not None: + new_status = CaseStatus.from_str(body.status) + if new_status == CaseStatus.CLOSED: + case.close() # sets closed_at + updated_at + else: + case.status = new_status + + repo.update_case(case) + return JSONResponse(content=case.to_dict()) + + +@router.post("/{case_id}/notes", status_code=201) +def add_note( + case_id: str, + body: NoteRequest, + _scope=Depends(require_full_scope), + repo=Depends(get_repo), +): + # add_note() does not validate case_id (FK enforcement is off), so check first. + case = repo.get_case(case_id) + if not case: + raise HTTPException(status_code=404, detail="case_not_found") + + note_id = repo.add_note(case_id, body.content) + return {"id": note_id, "content": body.content} + + +@router.get("/{case_id}/notes") +def list_notes( + case_id: str, + _scope=Depends(require_full_scope), + repo=Depends(get_repo), +): + case = repo.get_case(case_id) + if not case: + raise HTTPException(status_code=404, detail="case_not_found") + + return {"notes": [n.to_dict() for n in case.notes]} diff --git a/tests/api/test_cases.py b/tests/api/test_cases.py index 00cfc3b..1df833d 100644 --- a/tests/api/test_cases.py +++ b/tests/api/test_cases.py @@ -416,3 +416,246 @@ def test_api_reports_dir_env_takes_precedence(client, tmp_path, monkeypatch): assert not (legacy_reports / f"{case_id}.pdf").exists(), ( "PCAP_HUNTER_REPORTS_DIR must be ignored when PCAP_HUNTER_API_REPORTS_DIR is set" ) + + +# ==================== GET /api/v1/cases (list) ==================== + + +def test_list_cases_returns_seeded_cases(client): + from app.api.deps import get_repo + from app.database.models import Case + + repo = get_repo() + id1 = repo.create_case(Case(title="alpha")) + id2 = repo.create_case(Case(title="beta")) + + r = client.get("/api/v1/cases", headers={"Authorization": "Bearer MAIN"}) + assert r.status_code == 200 + body = r.json() + assert body["count"] == 2 + ids = {c["id"] for c in body["cases"]} + assert ids == {id1, id2} + # Light shape: list_cases() never loads analyses/notes, so the response must not + # imply case content it never fetched. + assert "analyses" not in body["cases"][0] + assert "notes" not in body["cases"][0] + + +def test_list_cases_filters_by_status(client): + from app.api.deps import get_repo + from app.database.models import Case, CaseStatus + + repo = get_repo() + repo.create_case(Case(title="open-case", status=CaseStatus.OPEN)) + closed_id = repo.create_case(Case(title="closed-case", status=CaseStatus.CLOSED)) + + r = client.get("/api/v1/cases", params={"status": "closed"}, headers={"Authorization": "Bearer MAIN"}) + assert r.status_code == 200 + body = r.json() + assert body["count"] == 1 + assert body["cases"][0]["id"] == closed_id + + +def test_list_cases_filters_by_tag(client): + from app.api.deps import get_repo + from app.database.models import Case + + repo = get_repo() + tagged_id = repo.create_case(Case(title="tagged", tags=["malware"])) + repo.create_case(Case(title="untagged")) + + r = client.get("/api/v1/cases", params={"tag": "malware"}, headers={"Authorization": "Bearer MAIN"}) + assert r.status_code == 200 + body = r.json() + assert body["count"] == 1 + assert body["cases"][0]["id"] == tagged_id + + +def test_list_cases_search_filter(client): + from app.api.deps import get_repo + from app.database.models import Case + + repo = get_repo() + match_id = repo.create_case(Case(title="beacon investigation")) + repo.create_case(Case(title="unrelated")) + + r = client.get("/api/v1/cases", params={"search": "beacon"}, headers={"Authorization": "Bearer MAIN"}) + assert r.status_code == 200 + body = r.json() + assert body["count"] == 1 + assert body["cases"][0]["id"] == match_id + + +def test_list_cases_limit_offset(client): + from app.api.deps import get_repo + from app.database.models import Case + + repo = get_repo() + for i in range(3): + repo.create_case(Case(title=f"case-{i}")) + + r = client.get("/api/v1/cases", params={"limit": 1, "offset": 0}, headers={"Authorization": "Bearer MAIN"}) + assert r.status_code == 200 + assert len(r.json()["cases"]) == 1 + + +def test_list_cases_requires_auth(client): + r = client.get("/api/v1/cases") + assert r.status_code == 401 + + +# ==================== PATCH /api/v1/cases/{id} ==================== + + +def test_patch_case_updates_fields(client): + from app.api.deps import get_repo + from app.database.models import Case + + case_id = get_repo().create_case(Case(title="orig", description="orig-desc")) + + r = client.patch( + f"/api/v1/cases/{case_id}", + json={"title": "new-title", "severity": "high", "tags": ["a", "b"]}, + headers={"Authorization": "Bearer MAIN"}, + ) + assert r.status_code == 200 + body = r.json() + assert body["title"] == "new-title" + assert body["severity"] == "high" + assert sorted(body["tags"]) == ["a", "b"] + assert body["description"] == "orig-desc" # untouched + + +def test_patch_case_status_closed_sets_closed_at(client): + from app.api.deps import get_repo + from app.database.models import Case + + case_id = get_repo().create_case(Case(title="orig")) + + r = client.patch( + f"/api/v1/cases/{case_id}", + json={"status": "closed"}, + headers={"Authorization": "Bearer MAIN"}, + ) + assert r.status_code == 200 + body = r.json() + assert body["status"] == "closed" + assert body["closed_at"] is not None + + +def test_patch_case_404_on_missing(client): + r = client.patch( + "/api/v1/cases/zzzz9999", + json={"title": "x"}, + headers={"Authorization": "Bearer MAIN"}, + ) + assert r.status_code == 404 + assert r.json()["code"] == "case_not_found" + + +def test_patch_case_invalid_status_returns_422(client): + from app.api.deps import get_repo + from app.database.models import Case + + case_id = get_repo().create_case(Case(title="orig")) + + r = client.patch( + f"/api/v1/cases/{case_id}", + json={"status": "not-a-status"}, + headers={"Authorization": "Bearer MAIN"}, + ) + assert r.status_code == 422 + + +def test_patch_case_invalid_severity_returns_422(client): + from app.api.deps import get_repo + from app.database.models import Case + + case_id = get_repo().create_case(Case(title="orig")) + + r = client.patch( + f"/api/v1/cases/{case_id}", + json={"severity": "not-a-severity"}, + headers={"Authorization": "Bearer MAIN"}, + ) + assert r.status_code == 422 + + +def test_patch_case_empty_body_is_noop(client): + from app.api.deps import get_repo + from app.database.models import Case + + case_id = get_repo().create_case(Case(title="orig", description="d")) + + r = client.patch(f"/api/v1/cases/{case_id}", json={}, headers={"Authorization": "Bearer MAIN"}) + assert r.status_code == 200 + body = r.json() + assert body["title"] == "orig" + assert body["description"] == "d" + + +# ==================== POST/GET /api/v1/cases/{id}/notes ==================== + + +def test_post_note_adds_note(client): + from app.api.deps import get_repo + from app.database.models import Case + + case_id = get_repo().create_case(Case(title="x")) + + r = client.post( + f"/api/v1/cases/{case_id}/notes", + json={"content": "analyst note"}, + headers={"Authorization": "Bearer MAIN"}, + ) + assert r.status_code == 201 + body = r.json() + assert body["content"] == "analyst note" + assert isinstance(body["id"], int) + + +def test_post_note_404_on_missing_case(client): + r = client.post( + "/api/v1/cases/zzzz9999/notes", + json={"content": "x"}, + headers={"Authorization": "Bearer MAIN"}, + ) + assert r.status_code == 404 + assert r.json()["code"] == "case_not_found" + + +def test_post_note_empty_content_returns_422(client): + from app.api.deps import get_repo + from app.database.models import Case + + case_id = get_repo().create_case(Case(title="x")) + + r = client.post( + f"/api/v1/cases/{case_id}/notes", + json={"content": ""}, + headers={"Authorization": "Bearer MAIN"}, + ) + assert r.status_code == 422 + + +def test_get_notes_lists_notes(client): + from app.api.deps import get_repo + from app.database.models import Case + + repo = get_repo() + case_id = repo.create_case(Case(title="x")) + repo.add_note(case_id, "first note") + repo.add_note(case_id, "second note") + + r = client.get(f"/api/v1/cases/{case_id}/notes", headers={"Authorization": "Bearer MAIN"}) + assert r.status_code == 200 + body = r.json() + assert len(body["notes"]) == 2 + contents = {n["content"] for n in body["notes"]} + assert contents == {"first note", "second note"} + + +def test_get_notes_404_on_missing_case(client): + r = client.get("/api/v1/cases/zzzz9999/notes", headers={"Authorization": "Bearer MAIN"}) + assert r.status_code == 404 + assert r.json()["code"] == "case_not_found" From 1d4917f2ab3c34cb7120dc74a1e1c720f548db4c Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Tue, 14 Jul 2026 01:34:39 +0700 Subject: [PATCH 21/35] feat: add single-ioc exact-match lookup endpoint Adds GET /api/v1/iocs/lookup for the #1 SOAR enrichment pattern (is this IP/domain/hash known, and its score) so clients no longer need to pull the whole feed and filter locally. - IOCFilter gains an optional value field; query_iocs adds an exact 'AND i.value = ?' predicate (uses idx_iocs_type_value), composing with the existing type/tag/case_id filters and mitre_techniques join. - New route reuses IOCEntry/IOCFeedResponse as response_model (previously declared but unused); cursor pagination is skipped since a single value aggregates to at most one row. --- app/api/feed.py | 7 +++++ app/api/routers/iocs.py | 25 +++++++++++++++ tests/api/test_feed_query.py | 45 ++++++++++++++++++++++++++ tests/api/test_iocs.py | 61 ++++++++++++++++++++++++++++++++++++ 4 files changed, 138 insertions(+) diff --git a/app/api/feed.py b/app/api/feed.py index 3e17b06..77a578f 100644 --- a/app/api/feed.py +++ b/app/api/feed.py @@ -28,6 +28,7 @@ class IOCFilter: types: list[str] = field(default_factory=list) tag: str | None = None case_id: str | None = None + value: str | None = None # exact match on i.value, for single-IOC lookup limit: int = 1000 offset: int = 0 @@ -74,6 +75,12 @@ def query_iocs(repo: CaseRepository, filt: IOCFilter) -> list[dict[str, Any]]: "a.case_id IN (SELECT ct2.case_id FROM case_tags ct2 JOIN tags t2 ON t2.id = ct2.tag_id WHERE t2.name = ?)" ) params.append(filt.tag) + if filt.value: + # Exact match — uses idx_iocs_type_value. Composes with the other + # WHERE predicates (e.g. types) via AND; aggregation/pagination below + # are unaffected since this only narrows the row set pre-GROUP BY. + where.append("i.value = ?") + params.append(filt.value) if where: sql += " WHERE " + " AND ".join(where) diff --git a/app/api/routers/iocs.py b/app/api/routers/iocs.py index 6f7b545..14167fb 100644 --- a/app/api/routers/iocs.py +++ b/app/api/routers/iocs.py @@ -15,6 +15,7 @@ from app.api.deps import get_repo, require_feed_scope from app.api.feed import IOCFilter, query_iocs +from app.api.models import IOCFeedResponse router = APIRouter(prefix="/api/v1", tags=["egress"]) @@ -105,6 +106,30 @@ def iocs_json( return _conditional_response(request, body, "application/json", _last_modified(rows)) +# ── Single-IOC lookup ─────────────────────────────────────────────────────── + + +@router.get("/iocs/lookup", response_model=IOCFeedResponse) +def iocs_lookup( + value: str = Query(..., min_length=1), + min_score: int = Query(default=0, ge=0, le=100), + type: str | None = Query(default=None), + limit: int = Query(default=1000, ge=1, le=10000), + _scope=Depends(require_feed_scope), + repo=Depends(get_repo), +): + """Exact-match single-IOC lookup — the #1 SOAR enrichment pattern. + + A single value normally aggregates to 0 or 1 row, so cursor pagination + is unnecessary here; next_cursor is always null. An empty ``value`` is + rejected (422) rather than silently returning the whole feed. + """ + filt = _build_filter(since=None, min_score=min_score, types=type, tag=None, case_id=None, limit=limit, cursor=None) + filt.value = value + rows = query_iocs(repo, filt) + return {"iocs": rows, "count": len(rows), "next_cursor": None} + + # ── CSV feed ──────────────────────────────────────────────────────────────── CSV_HEADER = [ diff --git a/tests/api/test_feed_query.py b/tests/api/test_feed_query.py index f4a9cca..ea09679 100644 --- a/tests/api/test_feed_query.py +++ b/tests/api/test_feed_query.py @@ -157,6 +157,51 @@ def test_query_iocs_multi_technique_multi_tag_no_duplication(tmp_path): assert sorted(row["tags"]) == ["tag-a", "tag-b"] +def test_query_iocs_value_filter_exact_match(tmp_path): + """An exact value filter returns only the matching IOC row.""" + repo = _seed(tmp_path) + rows = query_iocs(repo, IOCFilter(value="1.2.3.4")) + assert len(rows) == 1 + assert rows[0]["value"] == "1.2.3.4" + assert rows[0]["score"] == 75 + assert rows[0]["severity"] == "high" + + +def test_query_iocs_value_filter_unknown_returns_empty(tmp_path): + repo = _seed(tmp_path) + rows = query_iocs(repo, IOCFilter(value="9.9.9.9")) + assert rows == [] + + +def test_query_iocs_value_filter_is_exact_not_substring(tmp_path): + """value='1.2.3.4' must not also match '1.2.3.40' (substring false-positive).""" + repo = CaseRepository(db_path=str(tmp_path / "t.db")) + repo.create_case(Case(id="case9004", title="t")) + analysis = Analysis( + case_id="case9004", + pcap_path="/tmp/z.pcap", + iocs=[ + IOC(ioc_type=IOCType.IP, value="1.2.3.4", severity=Severity.HIGH), + IOC(ioc_type=IOCType.IP, value="1.2.3.40", severity=Severity.LOW), + ], + ) + repo.save_analysis(analysis) + + rows = query_iocs(repo, IOCFilter(value="1.2.3.4")) + assert len(rows) == 1 + assert rows[0]["value"] == "1.2.3.4" + + +def test_query_iocs_value_filter_composes_with_type_filter(tmp_path): + """value filter must AND with the existing type filter, not override it.""" + repo = _seed(tmp_path) + rows = query_iocs(repo, IOCFilter(value="1.2.3.4", types=["domain"])) + assert rows == [] + rows = query_iocs(repo, IOCFilter(value="1.2.3.4", types=["ip"])) + assert len(rows) == 1 + assert rows[0]["value"] == "1.2.3.4" + + def test_query_iocs_techniques_dont_affect_pagination(tmp_path): """LIMIT/OFFSET must still page over distinct (ioc_type, value) groups, not over the post-join fan-out rows, when techniques are present.""" diff --git a/tests/api/test_iocs.py b/tests/api/test_iocs.py index 26b6247..06edca6 100644 --- a/tests/api/test_iocs.py +++ b/tests/api/test_iocs.py @@ -235,6 +235,67 @@ def test_iocs_stix_ja3_not_dropped(client): assert "x509-certificate" in matching[0] +# ── Single-IOC lookup ─────────────────────────────────────────────────────── + + +def test_iocs_lookup_exact_match(client): + r = client.get("/api/v1/iocs/lookup?value=1.2.3.4", headers={"Authorization": "Bearer FEED"}) + assert r.status_code == 200 + body = r.json() + assert body["count"] == 1 + assert body["next_cursor"] is None + assert len(body["iocs"]) == 1 + entry = body["iocs"][0] + assert entry["value"] == "1.2.3.4" + assert entry["type"] == "ip" + assert entry["score"] == 75 + assert entry["severity"] == "high" + + +def test_iocs_lookup_unknown_value_returns_empty(client): + r = client.get("/api/v1/iocs/lookup?value=nonexistent.example", headers={"Authorization": "Bearer FEED"}) + assert r.status_code == 200 + assert r.json() == {"iocs": [], "count": 0, "next_cursor": None} + + +def test_iocs_lookup_requires_feed_scope(client): + r = client.get("/api/v1/iocs/lookup?value=1.2.3.4") + assert r.status_code == 401 + + +def test_iocs_lookup_missing_value_param_422(client): + r = client.get("/api/v1/iocs/lookup", headers={"Authorization": "Bearer FEED"}) + assert r.status_code == 422 + + +def test_iocs_lookup_empty_value_422_not_whole_feed(client): + # An empty value must be rejected, never silently dump the entire feed. + r = client.get("/api/v1/iocs/lookup?value=", headers={"Authorization": "Bearer FEED"}) + assert r.status_code == 422 + + +def test_iocs_lookup_exact_not_substring(client): + """value=1.2.3.4 must not also return 1.2.3.40.""" + from app.api.deps import get_repo + + repo = get_repo() + case = Case(id="case0009", title="substring") + repo.create_case(case) + repo.save_analysis( + Analysis( + case_id=case.id, + pcap_path="/tmp/sub.pcap", + iocs=[IOC(ioc_type=IOCType.IP, value="1.2.3.40", severity=Severity.LOW)], + ) + ) + + r = client.get("/api/v1/iocs/lookup?value=1.2.3.4", headers={"Authorization": "Bearer FEED"}) + assert r.status_code == 200 + body = r.json() + assert body["count"] == 1 + assert body["iocs"][0]["value"] == "1.2.3.4" + + # ── Pagination ────────────────────────────────────────────────────────────── From dc6008468ceea8f17fad2dace13f64515a1fd745 Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Tue, 14 Jul 2026 01:43:45 +0700 Subject: [PATCH 22/35] feat: add cef ioc feed endpoint Add GET /api/v1/iocs.cef for ArcSight/QRadar/Sentinel-style SIEM ingestion, backed by a new feed_rows_to_cef() adapter in cef_export.py. The existing _events_from_iocs() expects ScoredIOC-shaped priority_score (0.0-1.0) and drops anything below 0.4; feed rows carry an integer score (25/50/75/100) instead, so a naive score/100 mapping would silently drop LOW-severity IOCs from a feed pull. feed_rows_to_cef emits one CEF event per row with no gating, reusing the existing _SEVERITY_MAP and CEFEvent/format_syslog escaping logic. The route mirrors iocs.csv: same _build_filter query params, same _conditional_response ETag/Last-Modified handling. --- app/api/routers/iocs.py | 32 ++++++++++++++ app/utils/cef_export.py | 44 +++++++++++++++++++ tests/api/test_iocs.py | 92 ++++++++++++++++++++++++++++++++++++++++ tests/test_cef_export.py | 57 +++++++++++++++++++++++++ 4 files changed, 225 insertions(+) diff --git a/app/api/routers/iocs.py b/app/api/routers/iocs.py index 14167fb..4fb5365 100644 --- a/app/api/routers/iocs.py +++ b/app/api/routers/iocs.py @@ -16,6 +16,7 @@ from app.api.deps import get_repo, require_feed_scope from app.api.feed import IOCFilter, query_iocs from app.api.models import IOCFeedResponse +from app.utils.cef_export import feed_rows_to_cef router = APIRouter(prefix="/api/v1", tags=["egress"]) @@ -189,6 +190,37 @@ def iocs_csv( return _conditional_response(request, body, "text/csv", _last_modified(rows)) +# ── CEF feed ──────────────────────────────────────────────────────────────── + + +@router.get("/iocs.cef") +def iocs_cef( + request: Request, + since: str | None = Query(default=None), + min_score: int = Query(default=0, ge=0, le=100), + type: str | None = Query(default=None), + tag: str | None = Query(default=None), + case_id: str | None = Query(default=None), + limit: int = Query(default=1000, ge=1, le=10000), + cursor: str | None = Query(default=None), + _scope=Depends(require_feed_scope), + repo=Depends(get_repo), +): + """CEF/syslog egress feed for ArcSight/QRadar/Sentinel-style SIEM ingestion. + + Uses the dedicated ``feed_rows_to_cef`` adapter (not the priority-score-gated + ``_events_from_iocs``), so the CEF output faithfully mirrors exactly the rows + the same filters (min_score/type/etc.) would return via iocs.json/csv — + including LOW-severity IOCs that a naive score/100 -> priority_score mapping + would otherwise silently drop. + """ + filt = _build_filter(since, min_score, type, tag, case_id, limit, cursor) + rows = query_iocs(repo, filt) + + body = feed_rows_to_cef(rows, hostname="pcap-hunter").encode("utf-8") + return _conditional_response(request, body, "text/plain", _last_modified(rows)) + + # ── STIX 2.1 feed ────────────────────────────────────────────────────────── diff --git a/app/utils/cef_export.py b/app/utils/cef_export.py index 0325aa1..32ecd4f 100644 --- a/app/utils/cef_export.py +++ b/app/utils/cef_export.py @@ -277,6 +277,50 @@ def generate_cef_events( return events +def feed_rows_to_cef(rows: list[dict], hostname: str = "pcap-hunter") -> str: + """Render feed-query IOC rows (type/value/score/severity/tags/...) as CEF syslog lines. + + One CEF event per row — no priority-score gating; the caller's feed filters + (min_score/type) already decide inclusion. This is distinct from + :func:`_events_from_iocs`, which expects ``ScoredIOC``-shaped objects with a + 0.0-1.0 ``priority_score`` and silently drops anything below 0.4 — feed rows + instead carry an integer ``score`` (25/50/75/100, derived from severity), and + a CEF feed pull must faithfully return exactly the rows the feed query + returned, including LOW-severity ones. + + Args: + rows: Feed rows as returned by :func:`app.api.feed.query_iocs` — each a + dict with at least ``type``, ``value``, ``score``, ``severity``, and + ``tags`` keys. + hostname: Hostname for the syslog header. + + Returns: + Newline-separated CEF/syslog lines, one per row. Empty string if + ``rows`` is empty. + """ + events: list[CEFEvent] = [] + for row in rows: + severity_label = str(row.get("severity") or "medium").lower() + severity = _SEVERITY_MAP.get(severity_label, 5) + tags = row.get("tags") or [] + tag_str = ";".join(str(t) for t in tags) + + events.append( + CEFEvent( + signature_id="IOC-FEED-001", + name=f"IOC Feed Entry ({severity_label})", + severity=severity, + extensions={ + "value": str(row.get("value", "")), + "type": str(row.get("type", "")), + "score": str(row.get("score", "")), + "tags": tag_str, + }, + ) + ) + return "\n".join(format_syslog(e, hostname=hostname) for e in events) + + def export_cef_text( correlations: list | None = None, beacon_df: Any = None, diff --git a/tests/api/test_iocs.py b/tests/api/test_iocs.py index 06edca6..9cff103 100644 --- a/tests/api/test_iocs.py +++ b/tests/api/test_iocs.py @@ -235,6 +235,98 @@ def test_iocs_stix_ja3_not_dropped(client): assert "x509-certificate" in matching[0] +# ── CEF feed ──────────────────────────────────────────────────────────────── + + +def test_iocs_cef_low_ioc_not_dropped(client): + """A LOW-severity IOC (score 25) must appear in CEF output. + + The generic priority-score gate in cef_export._events_from_iocs drops + anything below 0.4 — a naive `score/100` mapping would silently drop a + LOW (25 -> 0.25) IOC from a feed pull. The dedicated feed adapter must + not apply that gate at all. + """ + from app.api.deps import get_repo + + repo = get_repo() + case = Case(id="case0010", title="cef-low") + repo.create_case(case) + repo.save_analysis( + Analysis( + case_id=case.id, + pcap_path="/tmp/low.pcap", + iocs=[IOC(ioc_type=IOCType.IP, value="66.66.66.66", severity=Severity.LOW)], + ) + ) + + r = client.get("/api/v1/iocs.cef", headers={"Authorization": "Bearer FEED"}) + assert r.status_code == 200 + assert r.headers["content-type"].startswith("text/plain") + assert "CEF:0" in r.text + # The seeded HIGH IOC (fixture) and the new LOW IOC must BOTH appear. + assert "1.2.3.4" in r.text + assert "66.66.66.66" in r.text + + +def test_iocs_cef_respects_min_score(client): + from app.api.deps import get_repo + + repo = get_repo() + case = Case(id="case0011", title="cef-minscore") + repo.create_case(case) + repo.save_analysis( + Analysis( + case_id=case.id, + pcap_path="/tmp/lo2.pcap", + iocs=[IOC(ioc_type=IOCType.IP, value="77.77.77.77", severity=Severity.LOW)], + ) + ) + + r = client.get("/api/v1/iocs.cef?min_score=50", headers={"Authorization": "Bearer FEED"}) + assert r.status_code == 200 + assert "77.77.77.77" not in r.text # LOW (25) filtered out by min_score=50 + assert "1.2.3.4" in r.text # HIGH (75) survives the >=50 filter + + +def test_iocs_cef_requires_feed_scope(client): + r = client.get("/api/v1/iocs.cef") + assert r.status_code == 401 + + +def test_iocs_cef_escaping_safe(client): + """CEF reserved characters (=, \\) in a value must be escaped, not raw.""" + from app.api.deps import get_repo + + repo = get_repo() + case = Case(id="case0013", title="cef-escape") + repo.create_case(case) + repo.save_analysis( + Analysis( + case_id=case.id, + pcap_path="/tmp/esc.pcap", + iocs=[IOC(ioc_type=IOCType.URL, value="http://evil.example/a=b")], + ) + ) + + r = client.get("/api/v1/iocs.cef", headers={"Authorization": "Bearer FEED"}) + assert r.status_code == 200 + assert "a\\=b" in r.text + assert "a=b" not in r.text.replace("a\\=b", "") + + +def test_iocs_cef_etag_round_trip(client): + r1 = client.get("/api/v1/iocs.cef", headers={"Authorization": "Bearer FEED"}) + assert r1.status_code == 200 + etag = r1.headers["ETag"] + + r2 = client.get( + "/api/v1/iocs.cef", + headers={"Authorization": "Bearer FEED", "If-None-Match": etag}, + ) + assert r2.status_code == 304 + assert r2.content == b"" + + # ── Single-IOC lookup ─────────────────────────────────────────────────────── diff --git a/tests/test_cef_export.py b/tests/test_cef_export.py index d5b34d4..e253962 100644 --- a/tests/test_cef_export.py +++ b/tests/test_cef_export.py @@ -5,6 +5,7 @@ from app.utils.cef_export import ( CEFEvent, export_cef_text, + feed_rows_to_cef, format_syslog, generate_cef_events, ) @@ -148,3 +149,59 @@ def test_full_export(self): def test_empty_export(self): text = export_cef_text() assert text == "" + + +class TestFeedRowsToCEF: + """feed_rows_to_cef renders feed-query rows (score 25-100, no priority_score) + directly — it must NOT apply the 0.4 priority-score gate used by + _events_from_iocs, or a LOW-severity row (score 25) would be silently + dropped from a feed pull.""" + + def test_low_row_not_dropped(self): + rows = [{"type": "ip", "value": "1.2.3.4", "score": 25, "severity": "low", "tags": []}] + text = feed_rows_to_cef(rows) + assert "CEF:0" in text + assert "1.2.3.4" in text + + def test_one_event_per_row(self): + rows = [ + {"type": "ip", "value": "1.1.1.1", "score": 25, "severity": "low", "tags": []}, + {"type": "ip", "value": "2.2.2.2", "score": 100, "severity": "critical", "tags": []}, + ] + text = feed_rows_to_cef(rows) + lines = [ln for ln in text.strip().splitlines() if ln] + assert len(lines) == 2 + + def test_severity_mapping_reuses_severity_map(self): + rows = [{"type": "ip", "value": "9.9.9.9", "score": 100, "severity": "critical", "tags": []}] + text = feed_rows_to_cef(rows) + # _SEVERITY_MAP["critical"] == 10 + assert "|10|" in text + + def test_unknown_severity_defaults_sanely(self): + rows = [{"type": "ip", "value": "8.8.8.8", "score": 50, "severity": "bogus", "tags": []}] + text = feed_rows_to_cef(rows) + assert "CEF:0" in text # doesn't raise; renders with a fallback severity + + def test_escaping_of_special_chars_in_value_and_tags(self): + rows = [ + { + "type": "url", + "value": "http://evil.example/a=b", + "score": 50, + "severity": "medium", + "tags": ["tag=1", "back\\slash"], + } + ] + text = feed_rows_to_cef(rows) + assert "a\\=b" in text + assert "tag\\=1" in text + assert "back\\\\slash" in text + + def test_empty_rows(self): + assert feed_rows_to_cef([]) == "" + + def test_hostname_in_syslog_header(self): + rows = [{"type": "ip", "value": "1.2.3.4", "score": 75, "severity": "high", "tags": []}] + text = feed_rows_to_cef(rows, hostname="feedhost") + assert "feedhost" in text From c24aeb706da7ed7cd950fd13959da4df5c80182d Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Tue, 14 Jul 2026 01:54:53 +0700 Subject: [PATCH 23/35] fix: escape crlf in cef output to prevent syslog log injection --- app/utils/cef_export.py | 32 +++++++++++++++++++++++++++----- tests/test_cef_export.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/app/utils/cef_export.py b/app/utils/cef_export.py index 32ecd4f..8e1fb5c 100644 --- a/app/utils/cef_export.py +++ b/app/utils/cef_export.py @@ -40,6 +40,30 @@ } +def _escape_cef_extension(value: str) -> str: + """Escape a CEF extension value per the CEF spec. + + Backslash MUST be escaped first, so the literal ``\\n``/``\\r`` sequences + introduced below for real newline/CR characters are not themselves + re-escaped. Embedded CR/LF are rendered as the literal two-character + escape sequences (not stripped) so attacker-influenced IOC values (e.g. + derived from PCAP data) can't inject a second CEF/syslog line into the + ingesting SIEM. + """ + return str(value).replace("\\", "\\\\").replace("=", "\\=").replace("\r", "\\r").replace("\n", "\\n") + + +def _escape_cef_header_field(value: str) -> str: + """Escape a CEF header field (e.g. ``name``) per the CEF spec. + + Backslash and pipe are escaped per spec. Header fields have no + standard escape sequence for a literal newline, so embedded CR/LF are + collapsed to a space instead — this keeps the record on a single line + without silently deleting the word-break the character implied. + """ + return value.replace("\\", "\\\\").replace("|", "\\|").replace("\r", " ").replace("\n", " ") + + @dataclass class CEFEvent: """A single CEF event ready for syslog emission.""" @@ -54,13 +78,11 @@ def to_cef(self) -> str: """Render as a CEF-formatted string (without syslog header).""" ext_parts = [] for k, v in self.extensions.items(): - # CEF extension values: escape = and backslash - safe_v = str(v).replace("\\", "\\\\").replace("=", "\\=") - ext_parts.append(f"{k}={safe_v}") + ext_parts.append(f"{k}={_escape_cef_extension(v)}") ext_str = " ".join(ext_parts) - # Escape pipe characters in header fields - name_safe = self.name.replace("|", "\\|") + # Escape pipe/backslash and neutralize CR/LF in header fields + name_safe = _escape_cef_header_field(self.name) return ( f"CEF:{_CEF_VERSION}|{_VENDOR}|{_PRODUCT}|{_PRODUCT_VERSION}" diff --git a/tests/test_cef_export.py b/tests/test_cef_export.py index e253962..2dddb4a 100644 --- a/tests/test_cef_export.py +++ b/tests/test_cef_export.py @@ -43,6 +43,28 @@ def test_empty_extensions(self): cef = event.to_cef() assert cef.endswith("|1|") + def test_extension_value_newline_escaping(self): + """A newline/CR embedded in an attacker-influenced extension value must not + be able to forge a second CEF/syslog line.""" + event = CEFEvent( + signature_id="T-1", + name="Test", + severity=1, + extensions={"cs1": "evil\nCEF:0|Fake|Fake|1.0|999|Forged Event|10|src=9.9.9.9\r"}, + ) + cef = event.to_cef() + assert "\n" not in cef + assert "\r" not in cef + assert "cs1=evil\\nCEF:0|Fake|Fake|1.0|999|Forged Event|10|src\\=9.9.9.9\\r" in cef + + def test_name_newline_does_not_split_header(self): + """A newline embedded in the name field must not split the CEF header + into a second forged line.""" + event = CEFEvent(signature_id="T-1", name="Evil\nCEF:0|Fake|Fake|1.0|999|Forged|10|", severity=1) + cef = event.to_cef() + assert "\n" not in cef + assert "\r" not in cef + class TestFormatSyslog: def test_syslog_header(self): @@ -205,3 +227,21 @@ def test_hostname_in_syslog_header(self): rows = [{"type": "ip", "value": "1.2.3.4", "score": 75, "severity": "high", "tags": []}] text = feed_rows_to_cef(rows, hostname="feedhost") assert "feedhost" in text + + def test_hostile_value_with_embedded_newline_yields_one_line_per_event(self): + """IOC values are attacker-influenced (derived from PCAP data). A value + containing an embedded newline must not be able to inject a forged + second CEF/syslog line into the SIEM feed.""" + rows = [ + { + "type": "domain", + "value": "evil.example\nCEF:0|Fake|Fake|1.0|999|Forged Event|10|src=9.9.9.9", + "score": 100, + "severity": "critical", + "tags": [], + }, + {"type": "ip", "value": "2.2.2.2", "score": 50, "severity": "medium", "tags": []}, + ] + text = feed_rows_to_cef(rows) + lines = [ln for ln in text.strip().splitlines() if ln] + assert len(lines) == 2 From d3ddd83df4e221ec597a57173fc15ed3c6310a37 Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Tue, 14 Jul 2026 02:30:31 +0700 Subject: [PATCH 24/35] feat: add ssrf-safe completion webhook on job submit SOAR clients can now pass webhook_url on POST /api/v1/pcaps instead of busy-polling GET /jobs/{id}. The worker subprocess POSTs a small envelope (job_id, case_id, status, analysis_id) to the URL on both terminal states (done/failed), with retry and optional HMAC signing. is_safe_webhook_url (app/utils/network_utils.py) is the only SSRF guard, since hardened_session does not itself block private IPs: it requires http/https, resolves the host, and rejects if any resolved address is not public (loopback/private/link-local/reserved/multicast/unspecified), closing a DNS-rebinding-style bypass. The router validates at submit time (422 on an unsafe URL); the worker's _dispatch_webhook re-validates immediately before every POST as defense in depth, and never raises so a webhook failure can't affect the job's own terminal status. --- app/api/queue.py | 111 +++++++++++++ app/api/routers/pcaps.py | 10 ++ app/api/settings.py | 6 + app/utils/network_utils.py | 59 +++++++ tests/api/test_auth.py | 2 + tests/api/test_gc.py | 2 + tests/api/test_key_auth.py | 2 + tests/api/test_pcaps.py | 54 +++++++ tests/api/test_settings.py | 22 +++ tests/api/test_webhook.py | 301 ++++++++++++++++++++++++++++++++++++ tests/test_network_utils.py | 109 +++++++++++++ 11 files changed, 678 insertions(+) create mode 100644 tests/api/test_webhook.py diff --git a/app/api/queue.py b/app/api/queue.py index 4ef9a55..ebae631 100644 --- a/app/api/queue.py +++ b/app/api/queue.py @@ -5,6 +5,7 @@ import json import logging import os +import time from abc import ABC, abstractmethod from concurrent.futures import ProcessPoolExecutor from dataclasses import dataclass, field @@ -198,6 +199,82 @@ def _persist_analysis( result.warnings.append(WARNING_PERSISTENCE_FAILED) +def _dispatch_webhook(url: str, payload: dict, timeout: int, max_retries: int) -> None: + """POST a job-completion webhook from the worker subprocess; never raises. + + Called from ``_worker_run`` on both terminal branches (done + failed). + A webhook delivery failure must never change the job's own terminal + state, so every failure mode here is caught and logged, not propagated. + + Re-validates the URL with ``is_safe_webhook_url`` immediately before the + network call — defense in depth against the target becoming unsafe + between submit-time validation (the router) and job completion (e.g. a + DNS change). ``hardened_session`` does not itself block private IPs. + + Args: + url: Destination webhook URL (already submit-time validated). + payload: JSON-serializable envelope to POST. + timeout: Per-attempt request timeout, seconds. + max_retries: Additional attempts after the first, on non-2xx status + or request exception. + """ + # Every failure mode -- including the lazy imports below -- is caught by + # this outer guard so this function NEVER raises. If it did, the + # exception would propagate into _worker_run's `except Exception`, whose + # unconditional `update_job_status(..., FAILED, ...)` would revert an + # already-DONE job to FAILED. The imports live inside the try for exactly + # that reason. + try: + from app.security.opsec import hardened_session, redact + from app.utils.network_utils import is_safe_webhook_url + + if not is_safe_webhook_url(url): + logger.warning("Webhook dispatch refused: %s no longer passes the SSRF guard", redact(url)) + return + + body = json.dumps(payload).encode("utf-8") + headers = {"Content-Type": "application/json"} + + # Optional HMAC signing so the receiver can authenticate the callback. + secret = os.environ.get("PCAP_HUNTER_API_WEBHOOK_SECRET") or None + if secret: + import hashlib + import hmac + + signature = hmac.new(secret.encode("utf-8"), body, hashlib.sha256).hexdigest() + headers["X-PCAP-Hunter-Signature"] = f"sha256={signature}" + + attempts = max(1, max_retries + 1) + last_error = "unknown" + for attempt in range(1, attempts + 1): + try: + # allow_redirects=False: the SSRF guard only validated `url`. + # Following a redirect (hardened_session permits up to 3, and + # only blocks https->http downgrades) to an attacker-chosen + # Location -- e.g. http://169.254.169.254/ -- would reach an + # unvalidated internal host and reopen the SSRF hole. Webhooks + # do not need redirects. + resp = hardened_session(timeout=timeout).post(url, data=body, headers=headers, allow_redirects=False) + if 200 <= resp.status_code < 300: + logger.info( + "Webhook delivered to %s (attempt %d/%d, status %d)", + redact(url), + attempt, + attempts, + resp.status_code, + ) + return + last_error = f"http_{resp.status_code}" + except Exception as exc: + last_error = f"{type(exc).__name__}: {exc}" + if attempt < attempts: + time.sleep(0.25) + logger.warning("Webhook delivery to %s failed after %d attempt(s): %s", redact(url), attempts, last_error) + except Exception: + # No redact() here: the import that provides it may be what failed. + logger.exception("Webhook dispatch crashed unexpectedly") + + def _worker_run(job_id: str, db_path: str, pcap_path: str, options_dict: dict) -> None: """Top-level worker function (must be picklable for ProcessPoolExecutor).""" from app.utils.logger import get_logger @@ -261,6 +338,23 @@ def _on_event(event: ProgressEvent) -> None: result_blob = json.dumps(result.to_dict()).encode("utf-8") repo.complete_job(job_id, result_blob) + + webhook_url = options_dict.get("webhook_url") + if webhook_url: + from app.api.settings import APISettings + + settings = APISettings.from_env() + _dispatch_webhook( + webhook_url, + { + "job_id": job_id, + "case_id": job.case_id, + "status": "done", + "analysis_id": result.analysis_id or None, + }, + timeout=settings.webhook_timeout_seconds, + max_retries=settings.webhook_max_retries, + ) except Exception as exc: logger.exception("Job %s failed: %s", job_id, exc) repo.update_job_status( @@ -270,6 +364,23 @@ def _on_event(event: ProgressEvent) -> None: error_detail=str(exc)[:500], ) + webhook_url = options_dict.get("webhook_url") + if webhook_url: + from app.api.settings import APISettings + + settings = APISettings.from_env() + _dispatch_webhook( + webhook_url, + { + "job_id": job_id, + "case_id": job.case_id, + "status": "failed", + "analysis_id": None, + }, + timeout=settings.webhook_timeout_seconds, + max_retries=settings.webhook_max_retries, + ) + class InProcessJobQueue(JobQueue): """ProcessPoolExecutor-backed queue using SQLite for state.""" diff --git a/app/api/routers/pcaps.py b/app/api/routers/pcaps.py index 9cdd515..b713afb 100644 --- a/app/api/routers/pcaps.py +++ b/app/api/routers/pcaps.py @@ -7,12 +7,14 @@ import uuid from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile +from starlette.concurrency import run_in_threadpool from app.api.deps import get_queue, get_repo, get_settings, require_full_scope from app.api.models import JobLinks, PcapSubmissionForm, PcapSubmissionResponse from app.api.queue import JobSubmission, QueueFullError from app.api.validation import is_valid_pcap_magic from app.database.models import Case, CaseStatus, Severity +from app.utils.network_utils import is_safe_webhook_url router = APIRouter(prefix="/api/v1/pcaps", tags=["ingress"]) @@ -34,11 +36,18 @@ async def submit_pcap( osint_enabled: bool = Form(default=True), llm_enabled: bool = Form(default=True), pyshark_packet_limit: int | None = Form(default=None), + webhook_url: str | None = Form(default=None), _scope=Depends(require_full_scope), repo=Depends(get_repo), queue=Depends(get_queue), settings=Depends(get_settings), ) -> PcapSubmissionResponse: + # Fail fast, before any upload I/O: reject unsafe webhook targets up front. + # is_safe_webhook_url resolves DNS (socket.getaddrinfo, blocking), so run + # it off the event loop to avoid stalling other in-flight requests. + if webhook_url and not await run_in_threadpool(is_safe_webhook_url, webhook_url): + raise HTTPException(status_code=422, detail="invalid_webhook_url") + case_id = uuid.uuid4().hex[:8] out_path = _uploads_dir() / f"{case_id}.pcap" @@ -99,6 +108,7 @@ async def submit_pcap( "do_zeek": True, "pre_count": True, "pyshark_packet_limit": pyshark_packet_limit, + "webhook_url": webhook_url, } try: job_id = queue.enqueue( diff --git a/app/api/settings.py b/app/api/settings.py index 5e64cfc..1f52191 100644 --- a/app/api/settings.py +++ b/app/api/settings.py @@ -31,6 +31,9 @@ class APISettings: job_ttl_days: int require_https: bool cors_origins: list[str] + webhook_timeout_seconds: int + webhook_max_retries: int + webhook_secret: str | None = None @classmethod def from_env(cls) -> "APISettings": @@ -56,4 +59,7 @@ def from_env(cls) -> "APISettings": cors_origins=[ o.strip() for o in os.environ.get("PCAP_HUNTER_API_CORS_ORIGINS", "").split(",") if o.strip() ], + webhook_timeout_seconds=int(os.environ.get("PCAP_HUNTER_API_WEBHOOK_TIMEOUT_SECONDS", "10")), + webhook_max_retries=int(os.environ.get("PCAP_HUNTER_API_WEBHOOK_MAX_RETRIES", "2")), + webhook_secret=os.environ.get("PCAP_HUNTER_API_WEBHOOK_SECRET") or None, ) diff --git a/app/utils/network_utils.py b/app/utils/network_utils.py index 9ea84a6..46bf8af 100644 --- a/app/utils/network_utils.py +++ b/app/utils/network_utils.py @@ -244,6 +244,65 @@ def is_public_ipv4(s: str) -> bool: return False +def is_safe_webhook_url(url: str) -> bool: + """SSRF guard for user-supplied outbound webhook URLs. + + ``hardened_session`` (app/security/opsec.py) hardens TLS/redirect/timeout + behavior but does **not** block requests to private IPs — this is the + only SSRF protection for the completion-webhook feature (API v2). + + True only when: the scheme is ``http``/``https``, the host resolves + (``socket.getaddrinfo``), and *every* resolved address is a public/global + IP — not loopback, private, link-local, reserved, multicast, or + unspecified. Requiring all resolved addresses to be public (rather than + just the first) closes a DNS-rebinding-style bypass where a hostname + round-robins between a public and a private address. + + Args: + url: Candidate webhook URL. + + Returns: + False for non-http(s) schemes, missing host, resolution failures, or + any resolved address that is not public. + """ + try: + parsed = urllib.parse.urlparse(url) + except ValueError: + return False + if parsed.scheme not in ("http", "https"): + return False + host = parsed.hostname + if not host: + return False + + # getaddrinfo handles hostnames and IP literals uniformly and returns + # every resolved address (A + AAAA), which is what we need to check. + try: + infos = socket.getaddrinfo(host, None) + except (socket.gaierror, socket.herror, OSError, UnicodeError): + return False + if not infos: + return False + + for info in infos: + ip_str = info[4][0].split("%", 1)[0] # strip IPv6 zone id (e.g. "fe80::1%eth0") + try: + ip = ipaddress.ip_address(ip_str) + except ValueError: + return False + # ipaddress.is_global is True for some multicast ranges (e.g. IPv4 + # 224.0.0.1, IPv6 ff02::1) — reject those explicitly rather than + # trusting is_global alone. + if ip.is_multicast or ip.is_unspecified: + return False + if isinstance(ip, ipaddress.IPv4Address): + if not is_public_ipv4(ip_str): + return False + elif not ip.is_global or ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved: + return False + return True + + def pick_top_public_ips(features: dict, n: int) -> list[str]: """ Return top-N public IPv4s by packet volume across flows. diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index b5ed6b2..5ad4fa4 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -26,6 +26,8 @@ def _settings(main: str | None = "MAIN_KEY", feed: str | None = "FEED_KEY") -> A job_ttl_days=30, require_https=False, cors_origins=[], + webhook_timeout_seconds=10, + webhook_max_retries=2, ) diff --git a/tests/api/test_gc.py b/tests/api/test_gc.py index fd216e0..41c41fc 100644 --- a/tests/api/test_gc.py +++ b/tests/api/test_gc.py @@ -28,6 +28,8 @@ def _settings(tmp_path) -> APISettings: job_ttl_days=3, require_https=False, cors_origins=[], + webhook_timeout_seconds=10, + webhook_max_retries=2, ) diff --git a/tests/api/test_key_auth.py b/tests/api/test_key_auth.py index 5a67f59..d64f22b 100644 --- a/tests/api/test_key_auth.py +++ b/tests/api/test_key_auth.py @@ -32,6 +32,8 @@ def _settings(main="MAIN", feed="FEED") -> APISettings: job_ttl_days=3, require_https=False, cors_origins=[], + webhook_timeout_seconds=10, + webhook_max_retries=2, ) diff --git a/tests/api/test_pcaps.py b/tests/api/test_pcaps.py index 23c8d21..37a8acc 100644 --- a/tests/api/test_pcaps.py +++ b/tests/api/test_pcaps.py @@ -3,6 +3,7 @@ from __future__ import annotations +import json import pathlib import pytest @@ -77,6 +78,59 @@ def test_post_with_invalid_magic_returns_415(client): assert r.status_code == 415 +# --------------------------------------------------------------------------- +# Task 4.4: SSRF-safe completion webhook on submit +# --------------------------------------------------------------------------- + + +def test_post_with_private_webhook_url_returns_422(client): + """Fail fast (before upload I/O) on a webhook_url that fails the SSRF guard.""" + r = client.post( + "/api/v1/pcaps", + headers={"Authorization": "Bearer MAIN"}, + files={"pcap": ("a.pcap", b"\x00" * 100)}, + data={"webhook_url": "http://10.0.0.5/hook"}, + ) + assert r.status_code == 422 + assert r.json()["detail"] == "invalid_webhook_url" + + +def test_post_with_loopback_webhook_url_returns_422(client): + r = client.post( + "/api/v1/pcaps", + headers={"Authorization": "Bearer MAIN"}, + files={"pcap": ("a.pcap", b"\x00" * 100)}, + data={"webhook_url": "http://127.0.0.1:9000/hook"}, + ) + assert r.status_code == 422 + assert r.json()["detail"] == "invalid_webhook_url" + + +@pytest.mark.skipif(not FIXTURE.exists(), reason="fixture missing") +def test_post_with_public_webhook_url_returns_202_and_persists_option(client, monkeypatch): + """A public https webhook_url must be accepted and carried into the job's options_json.""" + monkeypatch.setattr("app.api.routers.pcaps.is_safe_webhook_url", lambda url: True) + + with FIXTURE.open("rb") as f: + r = client.post( + "/api/v1/pcaps", + headers={"Authorization": "Bearer MAIN"}, + files={"pcap": ("tiny.pcap", f, "application/vnd.tcpdump.pcap")}, + data={ + "osint_enabled": "false", + "llm_enabled": "false", + "webhook_url": "https://example.com/hook", + }, + ) + assert r.status_code == 202, r.text + job_id = r.json()["job_id"] + + from app.api.deps import get_repo + + job = get_repo().get_job(job_id) + assert json.loads(job.options_json)["webhook_url"] == "https://example.com/hook" + + @pytest.mark.skipif(not FIXTURE.exists(), reason="fixture missing") def test_post_returns_503_when_queue_full(client, monkeypatch): monkeypatch.setenv("PCAP_HUNTER_API_QUEUE_DEPTH", "1") diff --git a/tests/api/test_settings.py b/tests/api/test_settings.py index 681449c..145a7eb 100644 --- a/tests/api/test_settings.py +++ b/tests/api/test_settings.py @@ -50,6 +50,28 @@ def test_env_overrides(monkeypatch): assert s.require_https is True +def test_webhook_settings_defaults(monkeypatch): + monkeypatch.setenv("PCAP_HUNTER_API_KEY", "x") + monkeypatch.delenv("PCAP_HUNTER_API_WEBHOOK_TIMEOUT_SECONDS", raising=False) + monkeypatch.delenv("PCAP_HUNTER_API_WEBHOOK_MAX_RETRIES", raising=False) + monkeypatch.delenv("PCAP_HUNTER_API_WEBHOOK_SECRET", raising=False) + s = APISettings.from_env() + assert s.webhook_timeout_seconds == 10 + assert s.webhook_max_retries == 2 + assert s.webhook_secret is None + + +def test_webhook_settings_env_overrides(monkeypatch): + monkeypatch.setenv("PCAP_HUNTER_API_KEY", "x") + monkeypatch.setenv("PCAP_HUNTER_API_WEBHOOK_TIMEOUT_SECONDS", "5") + monkeypatch.setenv("PCAP_HUNTER_API_WEBHOOK_MAX_RETRIES", "4") + monkeypatch.setenv("PCAP_HUNTER_API_WEBHOOK_SECRET", "shh") + s = APISettings.from_env() + assert s.webhook_timeout_seconds == 5 + assert s.webhook_max_retries == 4 + assert s.webhook_secret == "shh" + + def test_create_app_refuses_start_without_any_auth(monkeypatch, tmp_path): """create_app() raises NoKeysConfiguredError when no auth sources exist.""" monkeypatch.delenv("PCAP_HUNTER_API_KEY", raising=False) diff --git a/tests/api/test_webhook.py b/tests/api/test_webhook.py new file mode 100644 index 0000000..eaa1f7f --- /dev/null +++ b/tests/api/test_webhook.py @@ -0,0 +1,301 @@ +# tests/api/test_webhook.py +"""Tests for the completion-webhook dispatch (Task 4.4, app/api/queue.py). + +`_dispatch_webhook` is unit-tested directly (patching `hardened_session` at +its source module so the worker's lazy `from app.security.opsec import +hardened_session` picks up the fake). `_worker_run` integration tests confirm +it is invoked from both terminal branches, guarded by `options["webhook_url"]`. +""" + +from __future__ import annotations + +import json +import pathlib +from unittest.mock import MagicMock + +import pytest + +import app.api.queue as queue_mod +from app.api.queue import _dispatch_webhook +from app.database.models import Case, CaseStatus, Job, JobStatus, Severity +from app.database.repository import CaseRepository + +FIXTURE_PCAP = pathlib.Path(__file__).parent.parent / "fixtures" / "tiny.pcap" + + +@pytest.fixture(autouse=True) +def _no_sleep(monkeypatch): + """Keep the in-loop retry sleep from slowing down the suite.""" + monkeypatch.setattr(queue_mod.time, "sleep", lambda *_a, **_kw: None) + + +@pytest.fixture(autouse=True) +def _always_safe_url(monkeypatch): + """Most tests here exercise dispatch/retry logic, not the SSRF guard itself.""" + monkeypatch.setattr("app.utils.network_utils.is_safe_webhook_url", lambda url: True) + + +# --------------------------------------------------------------------------- +# _dispatch_webhook unit tests +# --------------------------------------------------------------------------- + + +def test_dispatch_webhook_posts_expected_envelope(monkeypatch): + captured = {} + + def fake_hardened_session(timeout): + captured["timeout"] = timeout + session = MagicMock() + + def post(url, data=None, headers=None, **kwargs): + captured["url"] = url + captured["data"] = data + captured["headers"] = headers + return MagicMock(status_code=200) + + session.post = post + return session + + monkeypatch.setattr("app.security.opsec.hardened_session", fake_hardened_session) + + payload = {"job_id": "j_abc12345", "case_id": "cafe0001", "status": "done", "analysis_id": "an_1"} + _dispatch_webhook("https://example.com/hook", payload, timeout=7, max_retries=2) + + assert captured["url"] == "https://example.com/hook" + assert captured["timeout"] == 7 + assert json.loads(captured["data"]) == payload + assert captured["headers"]["Content-Type"] == "application/json" + + +def test_dispatch_webhook_does_not_follow_redirects(monkeypatch): + """A 302 to an internal host must NOT be followed -- the SSRF guard only + validated the original URL, so redirects would reopen the SSRF hole. + The POST must pass allow_redirects=False.""" + captured = {} + calls = {"n": 0} + + def fake_hardened_session(timeout): + session = MagicMock() + + def post(url, data=None, headers=None, allow_redirects=None, **kwargs): + calls["n"] += 1 + captured["allow_redirects"] = allow_redirects + captured["url"] = url + # A malicious receiver tries to bounce us to the cloud metadata endpoint. + return MagicMock(status_code=302, headers={"Location": "http://169.254.169.254/latest/meta-data/"}) + + session.post = post + return session + + monkeypatch.setattr("app.security.opsec.hardened_session", fake_hardened_session) + + _dispatch_webhook("https://example.com/hook", {"a": 1}, timeout=5, max_retries=0) + + assert captured["allow_redirects"] is False, "webhook POST must disable redirect-following" + # Only the original (validated) URL was ever requested; the 302 Location was not chased. + assert calls["n"] == 1 + assert captured["url"] == "https://example.com/hook" + + +def test_dispatch_webhook_retries_up_to_max_retries_on_failure(monkeypatch): + calls = {"n": 0} + + def fake_hardened_session(timeout): + session = MagicMock() + + def post(url, data=None, headers=None, **kwargs): + calls["n"] += 1 + return MagicMock(status_code=500) + + session.post = post + return session + + monkeypatch.setattr("app.security.opsec.hardened_session", fake_hardened_session) + + _dispatch_webhook("https://example.com/hook", {"a": 1}, timeout=5, max_retries=2) + + assert calls["n"] == 3 # 1 initial attempt + 2 retries + + +def test_dispatch_webhook_stops_retrying_once_a_2xx_is_seen(monkeypatch): + calls = {"n": 0} + + def fake_hardened_session(timeout): + session = MagicMock() + + def post(url, data=None, headers=None, **kwargs): + calls["n"] += 1 + return MagicMock(status_code=200) + + session.post = post + return session + + monkeypatch.setattr("app.security.opsec.hardened_session", fake_hardened_session) + + _dispatch_webhook("https://example.com/hook", {"a": 1}, timeout=5, max_retries=3) + + assert calls["n"] == 1 + + +def test_dispatch_webhook_never_raises_when_session_construction_throws(monkeypatch): + def fake_hardened_session(timeout): + raise RuntimeError("network is down") + + monkeypatch.setattr("app.security.opsec.hardened_session", fake_hardened_session) + + # Must not raise -- a webhook failure must never affect the job's own status. + _dispatch_webhook("https://example.com/hook", {"a": 1}, timeout=5, max_retries=1) + + +def test_dispatch_webhook_never_raises_when_post_throws_every_attempt(monkeypatch): + def fake_hardened_session(timeout): + session = MagicMock() + session.post.side_effect = ConnectionError("boom") + return session + + monkeypatch.setattr("app.security.opsec.hardened_session", fake_hardened_session) + + _dispatch_webhook("https://example.com/hook", {"a": 1}, timeout=5, max_retries=2) + + +def test_dispatch_webhook_never_raises_when_lazy_import_fails(monkeypatch): + """The lazy `from app.security.opsec import ...` lives INSIDE the try, so an + import failure is swallowed too. If it escaped, it would propagate into + _worker_run's `except` and flip an already-DONE job to FAILED.""" + import sys + + # Poisoning sys.modules makes `from app.security.opsec import ...` raise + # ModuleNotFoundError -- the very first statement inside _dispatch_webhook's try. + monkeypatch.setitem(sys.modules, "app.security.opsec", None) + + # Must return, not raise, even though the import blew up. + _dispatch_webhook("https://example.com/hook", {"a": 1}, timeout=5, max_retries=1) + + +def test_dispatch_webhook_refuses_unsafe_url_without_posting(monkeypatch): + monkeypatch.setattr("app.utils.network_utils.is_safe_webhook_url", lambda url: False) + + session = MagicMock() + monkeypatch.setattr("app.security.opsec.hardened_session", lambda timeout: session) + + _dispatch_webhook("http://10.0.0.5/hook", {"a": 1}, timeout=5, max_retries=2) + + session.post.assert_not_called() + + +# --------------------------------------------------------------------------- +# _worker_run integration: webhook fires on both terminal branches +# --------------------------------------------------------------------------- + + +def _fake_pipeline_result(case_id): + from app.pipeline.runner import PipelineResult + + return PipelineResult( + case_id=case_id, + packet_count=1, + features={"flows": [], "artifacts": {"ips": [], "domains": [], "urls": [], "hashes": [], "ja3": []}}, + ) + + +def test_worker_fires_webhook_on_success(tmp_path, monkeypatch): + import app.pipeline.runner as runner_mod + + captured = {} + + def fake_dispatch(url, payload, timeout, max_retries): + captured.update(url=url, payload=payload, timeout=timeout, max_retries=max_retries) + + monkeypatch.setattr(queue_mod, "_dispatch_webhook", fake_dispatch) + monkeypatch.setenv("PCAP_HUNTER_API_WEBHOOK_TIMEOUT_SECONDS", "3") + monkeypatch.setenv("PCAP_HUNTER_API_WEBHOOK_MAX_RETRIES", "1") + monkeypatch.setattr( + runner_mod, + "run_pipeline", + lambda pcap_path, case_id, options, progress, heartbeat=None: (_fake_pipeline_result(case_id)), + ) + + fake_pcap = tmp_path / "fake.pcap" + fake_pcap.write_bytes(b"\xd4\xc3\xb2\xa1" + b"\x00" * 20) + + db = str(tmp_path / "t.db") + repo = CaseRepository(db_path=db) + repo.create_case(Case(id="cafe0099", title="t", status=CaseStatus.IN_PROGRESS, severity=Severity.LOW)) + job_id = repo.create_job(Job(case_id="cafe0099", pcap_path=str(fake_pcap), options_json="{}")) + + queue_mod._worker_run( + job_id, + db, + str(fake_pcap), + {"osint_enabled": False, "llm_enabled": False, "webhook_url": "https://example.com/hook"}, + ) + + job = repo.get_job(job_id) + assert job.status == JobStatus.DONE + assert captured["url"] == "https://example.com/hook" + assert captured["payload"]["job_id"] == job_id + assert captured["payload"]["case_id"] == "cafe0099" + assert captured["payload"]["status"] == "done" + assert captured["timeout"] == 3 + assert captured["max_retries"] == 1 + + +def test_worker_fires_webhook_on_failure(tmp_path, monkeypatch): + import app.pipeline.runner as runner_mod + + captured = {} + + def fake_dispatch(url, payload, timeout, max_retries): + captured["payload"] = payload + + monkeypatch.setattr(queue_mod, "_dispatch_webhook", fake_dispatch) + + def boom(*a, **kw): + raise RuntimeError("pipeline exploded") + + monkeypatch.setattr(runner_mod, "run_pipeline", boom) + + fake_pcap = tmp_path / "fake.pcap" + fake_pcap.write_bytes(b"\xd4\xc3\xb2\xa1" + b"\x00" * 20) + + db = str(tmp_path / "t.db") + repo = CaseRepository(db_path=db) + repo.create_case(Case(id="cafe0098", title="t", status=CaseStatus.IN_PROGRESS, severity=Severity.LOW)) + job_id = repo.create_job(Job(case_id="cafe0098", pcap_path=str(fake_pcap), options_json="{}")) + + queue_mod._worker_run( + job_id, + db, + str(fake_pcap), + {"osint_enabled": False, "llm_enabled": False, "webhook_url": "https://example.com/hook"}, + ) + + job = repo.get_job(job_id) + assert job.status == JobStatus.FAILED + assert captured["payload"]["status"] == "failed" + assert captured["payload"]["analysis_id"] is None + assert captured["payload"]["case_id"] == "cafe0098" + + +def test_worker_does_not_fire_webhook_when_not_configured(tmp_path, monkeypatch): + import app.pipeline.runner as runner_mod + + called = {"n": 0} + monkeypatch.setattr(queue_mod, "_dispatch_webhook", lambda *a, **kw: called.__setitem__("n", called["n"] + 1)) + monkeypatch.setattr( + runner_mod, + "run_pipeline", + lambda pcap_path, case_id, options, progress, heartbeat=None: (_fake_pipeline_result(case_id)), + ) + + fake_pcap = tmp_path / "fake.pcap" + fake_pcap.write_bytes(b"\xd4\xc3\xb2\xa1" + b"\x00" * 20) + + db = str(tmp_path / "t.db") + repo = CaseRepository(db_path=db) + repo.create_case(Case(id="cafe0097", title="t", status=CaseStatus.IN_PROGRESS, severity=Severity.LOW)) + job_id = repo.create_job(Job(case_id="cafe0097", pcap_path=str(fake_pcap), options_json="{}")) + + queue_mod._worker_run(job_id, db, str(fake_pcap), {"osint_enabled": False, "llm_enabled": False}) + + assert called["n"] == 0 diff --git a/tests/test_network_utils.py b/tests/test_network_utils.py index 9360400..96630f7 100644 --- a/tests/test_network_utils.py +++ b/tests/test_network_utils.py @@ -11,6 +11,7 @@ bulk_resolve_ips, is_enrichable_domain, is_public_ipv4, + is_safe_webhook_url, pick_top_public_ips, resolve_ip, ) @@ -169,3 +170,111 @@ def test_pick_top_public_ips_ranks_by_packet_volume(): assert pick_top_public_ips(features, 1) == ["8.8.8.8"] # n <= 0 -> all public ips from artifacts assert set(pick_top_public_ips(features, 0)) == {"8.8.8.8", "1.1.1.1"} + + +class TestIsSafeWebhookUrl: + """SSRF guard for the API v2 completion-webhook feature (Task 4.4). + + hardened_session does NOT block private IPs, so this guard is the only + SSRF protection — the "public URL" and "unresolvable host" cases mock + socket.getaddrinfo so the suite never depends on real DNS/network access. + """ + + def test_rejects_non_http_scheme(self): + assert is_safe_webhook_url("ftp://example.com/hook") is False + + def test_rejects_missing_host(self): + assert is_safe_webhook_url("http:///hook") is False + + def test_rejects_empty_string(self): + assert is_safe_webhook_url("") is False + + def test_rejects_private_10(self): + assert is_safe_webhook_url("http://10.0.0.5/hook") is False + + def test_rejects_loopback(self): + assert is_safe_webhook_url("http://127.0.0.1/hook") is False + + def test_rejects_link_local(self): + assert is_safe_webhook_url("http://169.254.169.254/hook") is False + + def test_rejects_localhost_hostname(self): + # "localhost" resolves via the hosts file/nsswitch without hitting + # the network, so this is safe to assert without mocking. + assert is_safe_webhook_url("http://localhost/hook") is False + + def test_rejects_unresolvable_host(self, monkeypatch): + import socket as socket_mod + + def boom(host, port, *args, **kwargs): + raise socket_mod.gaierror("nodename nor servname provided") + + monkeypatch.setattr("app.utils.network_utils.socket.getaddrinfo", boom) + assert is_safe_webhook_url("https://does-not-exist.invalid/hook") is False + + def test_accepts_public_https_url(self, monkeypatch): + def fake_getaddrinfo(host, port, *args, **kwargs): + return [(2, 1, 6, "", ("93.184.216.34", 0))] + + monkeypatch.setattr("app.utils.network_utils.socket.getaddrinfo", fake_getaddrinfo) + assert is_safe_webhook_url("https://example.com/hook") is True + + def test_rejects_when_any_resolved_ip_is_private(self, monkeypatch): + """DNS-rebinding-style bypass: if ANY resolved address is private, refuse.""" + + def fake_getaddrinfo(host, port, *args, **kwargs): + return [ + (2, 1, 6, "", ("93.184.216.34", 0)), + (2, 1, 6, "", ("10.0.0.9", 0)), + ] + + monkeypatch.setattr("app.utils.network_utils.socket.getaddrinfo", fake_getaddrinfo) + assert is_safe_webhook_url("https://rebinding.example/hook") is False + + def test_rejects_ipv4_multicast(self, monkeypatch): + """ipaddress.is_global is (surprisingly) True for IPv4 multicast — must be rejected explicitly.""" + + def fake_getaddrinfo(host, port, *args, **kwargs): + return [(2, 1, 6, "", ("239.255.255.250", 0))] + + monkeypatch.setattr("app.utils.network_utils.socket.getaddrinfo", fake_getaddrinfo) + assert is_safe_webhook_url("http://multicast.example/hook") is False + + def test_rejects_ipv6_link_local_multicast(self, monkeypatch): + """ipaddress.is_global is also True for some IPv6 multicast ranges (e.g. ff02::1).""" + + def fake_getaddrinfo(host, port, *args, **kwargs): + return [(10, 1, 6, "", ("ff02::1", 0, 0, 0))] + + monkeypatch.setattr("app.utils.network_utils.socket.getaddrinfo", fake_getaddrinfo) + assert is_safe_webhook_url("http://[ff02::1]/hook") is False + + def test_rejects_ipv6_unspecified(self, monkeypatch): + def fake_getaddrinfo(host, port, *args, **kwargs): + return [(10, 1, 6, "", ("::", 0, 0, 0))] + + monkeypatch.setattr("app.utils.network_utils.socket.getaddrinfo", fake_getaddrinfo) + assert is_safe_webhook_url("http://[::]/hook") is False + + def test_accepts_public_ipv6_url(self, monkeypatch): + def fake_getaddrinfo(host, port, *args, **kwargs): + return [(10, 1, 6, "", ("2606:4700:4700::1111", 0, 0, 0))] + + monkeypatch.setattr("app.utils.network_utils.socket.getaddrinfo", fake_getaddrinfo) + assert is_safe_webhook_url("https://public-v6.example/hook") is True + + @pytest.mark.parametrize( + "url", + [ + "http://[::ffff:127.0.0.1]/", # IPv4-mapped IPv6 loopback + "http://2130706433/", # decimal-encoded 127.0.0.1 + "http://0x7f000001/", # hex-encoded 127.0.0.1 + "http://100.64.0.1/", # CGNAT shared address space (100.64.0.0/10) + "http://240.0.0.1/", # reserved (240.0.0.0/4) + ], + ) + def test_rejects_known_ssrf_bypass_vectors(self, url): + """Lock the guard against classic SSRF encoding/range bypasses across + Python versions -- these use the real resolver (getaddrinfo handles the + IP literals / integer forms without network access).""" + assert is_safe_webhook_url(url) is False From 17703f3305b5215bae4c33764388e33bad1ced10 Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Tue, 14 Jul 2026 03:03:30 +0700 Subject: [PATCH 25/35] docs: document case crud, ioc lookup, cef feed, and completion webhooks --- docs/API.md | 326 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 321 insertions(+), 5 deletions(-) diff --git a/docs/API.md b/docs/API.md index b01d6f0..e5903f6 100644 --- a/docs/API.md +++ b/docs/API.md @@ -236,11 +236,17 @@ All routes implemented by the server, by router group: | PCAP Ingestion | POST | `/api/v1/pcaps` | `full` | | Job Tracking | GET | `/api/v1/jobs/{job_id}` | `full` | | Job Tracking | GET | `/api/v1/jobs/{job_id}/result` | `full` | +| Case Management | GET | `/api/v1/cases` | `full` | | Case Management | GET | `/api/v1/cases/{case_id}` | `full` | | Case Management | GET | `/api/v1/cases/{case_id}/report.pdf` | `full` | | Case Management | DELETE | `/api/v1/cases/{case_id}` | `full` | +| Case Management | PATCH | `/api/v1/cases/{case_id}` | `full` | +| Case Management | POST | `/api/v1/cases/{case_id}/notes` | `full` | +| Case Management | GET | `/api/v1/cases/{case_id}/notes` | `full` | | IOC Feed | GET | `/api/v1/iocs.json` | `feed` | +| IOC Feed | GET | `/api/v1/iocs/lookup` | `feed` | | IOC Feed | GET | `/api/v1/iocs.csv` | `feed` | +| IOC Feed | GET | `/api/v1/iocs.cef` | `feed` | | IOC Feed | GET | `/api/v1/iocs.stix` (alias: `/api/v1/iocs/stix`) | `feed` | | Admin | POST | `/api/v1/admin/keys` | `full` | | Admin | GET | `/api/v1/admin/keys` | `full` | @@ -340,11 +346,35 @@ Submit a PCAP file for background analysis. The upload is streamed to disk in 1 | `osint_enabled` | boolean | No | `true` | Run OSINT enrichment after analysis (see below) | | `llm_enabled` | boolean | No | `true` | Accepted for forward compatibility — LLM reports are **not yet supported headless** (see below) | | `pyshark_packet_limit` | integer | No | server default (200,000) | Cap on packets to deep-parse | +| `webhook_url` | string | No | (none) | `http`/`https` callback URL to POST when the job reaches a terminal state (see [Completion webhook](#completion-webhook-webhook_url) below). Validated at submit time; rejected with `422` if unsafe | **OSINT enrichment (`osint_enabled`):** when enabled, the worker enriches the top public IPs after analysis using provider keys from the saved Streamlit config (`cfg_*_key` values) or, as a fallback, the environment (`OTX_KEY`, `VT_KEY`, `ABUSEIPDB_KEY`, `GREYNOISE_KEY`, `SHODAN_KEY`). If no provider keys are configured, the job still completes — with the warning code `osint_not_configured` in the result. Note: the API path always queries providers fresh; the OSINT response cache is not used headless. **LLM reports (`llm_enabled`):** LLM report generation is not yet supported on the API path. The field is accepted so existing clients keep working, but jobs complete with the warning code `llm_unsupported_on_api_path` in the result — including default submissions, since the field defaults to `true`. +#### Completion webhook (`webhook_url`) + +When `webhook_url` is supplied, the API POSTs a small JSON envelope to that URL once the job reaches a terminal state (`done` or `failed`) — useful for SOAR playbooks that would otherwise have to poll `GET /jobs/{job_id}`. + +- **SSRF-validated twice:** once at submit time (before any upload I/O — an unsafe URL fails fast with `422 invalid_webhook_url`), and again immediately before the callback POST fires (defense in depth against the target becoming unsafe between submission and job completion, e.g. a DNS change). A URL is safe only if its scheme is `http`/`https` and **every** address it resolves to is a public/global IP — not loopback, private, link-local, reserved, or multicast. `hardened_session` alone does not block private IPs, so this check is the only SSRF guard for this feature. +- **Delivery happens in the analysis worker subprocess**, after the job's own terminal state is already persisted. A webhook delivery failure (timeout, connection error, non-2xx status) is logged and retried but **never** changes the job's `done`/`failed` status — polling `GET /jobs/{job_id}` remains the source of truth. +- **Retries:** up to `PCAP_HUNTER_API_WEBHOOK_MAX_RETRIES` additional attempts (default `2`, so 3 attempts total) with a short fixed delay between attempts, each bounded by `PCAP_HUNTER_API_WEBHOOK_TIMEOUT_SECONDS` (default `10`). Any 2xx response stops retrying immediately. +- **Redirects are not followed** (`allow_redirects=False`) — only submit-time validation covers the URL you supplied; following a redirect to an attacker-chosen `Location` (e.g. a cloud metadata endpoint) would reopen the SSRF hole. +- **Optional HMAC signature:** if `PCAP_HUNTER_API_WEBHOOK_SECRET` is set on the server, every callback carries an `X-PCAP-Hunter-Signature: sha256=` header (HMAC-SHA256 of the raw JSON body, keyed with the secret) so the receiver can authenticate the request. + +**Callback envelope** (`Content-Type: application/json`): + +```json +{ + "job_id": "j_7d4e9f21", + "case_id": "c4a1b2d9", + "status": "done", + "analysis_id": "9c2d1e0f-4a7" +} +``` + +`status` is `"done"` or `"failed"`; `analysis_id` is `null` for a failed job, and also `null` on a done job whose analysis failed to persist (mirrors `analysis_id` on the [job result](#get-apiv1jobsjob_idresult) endpoint). + **Sample request:** ```bash @@ -355,7 +385,8 @@ curl -X POST http://localhost:8000/api/v1/pcaps \ -F 'tags=["soar:tines","source:edr_alert"]' \ -F "severity_hint=high" \ -F "osint_enabled=true" \ - -F "pyshark_packet_limit=100000" + -F "pyshark_packet_limit=100000" \ + -F "webhook_url=https://soar.example.com/hooks/pcap-hunter" ``` **Sample response — 202 Accepted:** @@ -382,6 +413,7 @@ curl -X POST http://localhost:8000/api/v1/pcaps \ | 413 | `pcap_too_large` | Upload exceeded `PCAP_HUNTER_API_MAX_PCAP_BYTES`; the partial file is deleted | | 415 | `pcap_invalid_format` | First bytes are not a known pcap/pcapng magic; the file is deleted | | 422 | `validation_error` | Malformed form field (e.g. non-boolean `osint_enabled`) | +| 422 | `invalid_webhook_url` | `webhook_url` is not `http`/`https`, its host doesn't resolve, or any resolved address is private/loopback/link-local/reserved/multicast — checked before any upload I/O | | 429 | `rate_limit_exceeded` | DB key over its per-minute limit (`Retry-After` header set) | | 503 | `queue_full` | Active jobs ≥ `PCAP_HUNTER_API_QUEUE_DEPTH`; response carries `Retry-After: 60` | @@ -611,6 +643,58 @@ Example `409`: ### Case Management +#### `GET /api/v1/cases` + +List cases with optional filters — the light-weight complement to `GET /api/v1/cases/{case_id}`: each entry omits embedded `analyses[]`/`notes[]` so a full case listing stays cheap even when individual cases carry large feature blobs. Read-only and idempotent. Requires `full` scope. Note: an unrecognized `status` value does not `422` — it silently falls back to filtering on `open` (the same lenient string→enum conversion `PATCH` uses internally), so double-check spelling rather than relying on a validation error to catch a typo. + +**Auth:** `full` scope required + +**Parameters:** + +| Param | In | Type | Required | Default | Description | +|-------|----|------|----------|---------|-------------| +| `status` | query | string | No | (none) | Filter to one status: `open`, `in_progress`, or `closed`. An unrecognized value is treated as `open`, not rejected (see note above) | +| `tag` | query | string | No | (none) | Filter to cases carrying this tag | +| `search` | query | string | No | (none) | Case-insensitive substring match against title and description | +| `limit` | query | integer | No | `100` | Results per page, 1–500 (422 outside that range) | +| `offset` | query | integer | No | `0` | Results to skip, ≥ 0 | + +**Sample request:** + +```bash +curl "http://localhost:8000/api/v1/cases?status=in_progress&limit=50" \ + -H "Authorization: Bearer phk_4f8a2b9c1d3e5f60718293a4b5c6d7e8" +``` + +**Sample response — 200 OK:** + +```json +{ + "cases": [ + { + "id": "c4a1b2d9", + "title": "Incident 2026-0042", + "description": "", + "status": "in_progress", + "severity": "high", + "created_at": "2026-06-12T09:14:02.731842", + "updated_at": "2026-06-12T09:15:41.557209", + "closed_at": null, + "tags": ["soar:tines", "source:edr_alert"] + } + ], + "count": 1 +} +``` + +**Error responses:** + +| Status | Code | When | +|--------|------|------| +| 401 | `missing_or_malformed_auth` / `invalid_key` | Auth failure | +| 403 | `insufficient_scope` | Feed-scope key | +| 422 | `validation_error` | Query constraint violated (e.g. `limit=0`, `limit=1000`) | + #### `GET /api/v1/cases/{case_id}` Fetch the full case record — title, status, severity, tags, plus every persisted analysis (with its extracted IOCs) and case notes embedded. API-submitted cases are created with `status = in_progress` and the requested `severity_hint`; the same record is visible in the Streamlit Cases tab. Read-only and idempotent. Requires `full` scope. Note that embedded `analyses[]` can be large (the `features` object holds the full feature extraction); fetch the job result instead if you only need pipeline output. @@ -786,13 +870,150 @@ Example `409`: } ``` +#### `PATCH /api/v1/cases/{case_id}` + +Partially update a case — title, description, status, severity, and/or tags. Omitted fields are left unchanged; there is no way to distinguish "field omitted" from "set to its current value" (both are no-ops). Setting `status` to `closed` also sets `closed_at` (via the same path the Streamlit "close case" action uses); setting any other status does not clear a previously-set `closed_at`. `tags` is a **full replace**, not a merge — send the complete desired tag list. Requires `full` scope. + +**Auth:** `full` scope required + +**Content-Type:** `application/json` + +**Parameters (JSON body, all optional):** + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `title` | string | No | New title (1–200 chars) | +| `description` | string | No | New description (max 5000 chars) | +| `status` | string | No | `open`, `in_progress`, or `closed` | +| `severity` | string | No | `low`, `medium`, `high`, or `critical` | +| `tags` | array of string | No | Full replacement tag list | + +**Sample request:** + +```bash +curl -X PATCH http://localhost:8000/api/v1/cases/c4a1b2d9 \ + -H "Authorization: Bearer phk_4f8a2b9c1d3e5f60718293a4b5c6d7e8" \ + -H "Content-Type: application/json" \ + -d '{"status": "closed", "severity": "critical", "tags": ["soar:tines", "confirmed"]}' +``` + +**Sample response — 200 OK** (full updated case record, same shape as `GET /api/v1/cases/{case_id}`): + +```json +{ + "id": "c4a1b2d9", + "title": "Incident 2026-0042", + "description": "", + "status": "closed", + "severity": "critical", + "created_at": "2026-06-12T09:14:02.731842", + "updated_at": "2026-06-12T09:20:11.004221", + "closed_at": "2026-06-12T09:20:11.004221", + "tags": ["soar:tines", "confirmed"], + "analyses": [], + "notes": [] +} +``` + +**Error responses:** + +| Status | Code | When | +|--------|------|------| +| 401 | `missing_or_malformed_auth` / `invalid_key` | Auth failure | +| 403 | `insufficient_scope` | Feed-scope key | +| 404 | `case_not_found` | Case ID does not exist | +| 422 | `validation_error` | `status`/`severity` not one of the accepted values, `title` empty, or a field exceeds its max length | + +#### `POST /api/v1/cases/{case_id}/notes` + +Add an analyst note to a case. Notes are plain text with no formatting; there is no update or delete endpoint for an individual note. Not idempotent — every call adds a new note. Requires `full` scope. + +**Auth:** `full` scope required + +**Content-Type:** `application/json` + +**Parameters (JSON body):** + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `content` | string | Yes | Note text, min length 1 | + +**Sample request:** + +```bash +curl -X POST http://localhost:8000/api/v1/cases/c4a1b2d9/notes \ + -H "Authorization: Bearer phk_4f8a2b9c1d3e5f60718293a4b5c6d7e8" \ + -H "Content-Type: application/json" \ + -d '{"content": "Confirmed beaconing to 198.51.100.42 matches known C2 infrastructure."}' +``` + +**Sample response — 201 Created:** + +```json +{ + "id": 17, + "content": "Confirmed beaconing to 198.51.100.42 matches known C2 infrastructure." +} +``` + +**Error responses:** + +| Status | Code | When | +|--------|------|------| +| 401 | `missing_or_malformed_auth` / `invalid_key` | Auth failure | +| 403 | `insufficient_scope` | Feed-scope key | +| 404 | `case_not_found` | Case ID does not exist | +| 422 | `validation_error` | `content` missing or empty | + +#### `GET /api/v1/cases/{case_id}/notes` + +List all notes on a case, in insertion order. Read-only and idempotent. Requires `full` scope. + +**Auth:** `full` scope required + +**Parameters:** + +| Param | In | Type | Required | Description | +|-------|----|------|----------|-------------| +| `case_id` | path | string | Yes | Case ID (8 hex chars) | + +**Sample request:** + +```bash +curl http://localhost:8000/api/v1/cases/c4a1b2d9/notes \ + -H "Authorization: Bearer phk_4f8a2b9c1d3e5f60718293a4b5c6d7e8" +``` + +**Sample response — 200 OK:** + +```json +{ + "notes": [ + { + "id": 17, + "content": "Confirmed beaconing to 198.51.100.42 matches known C2 infrastructure.", + "created_at": "2026-06-12T09:21:03.552104", + "updated_at": null + } + ] +} +``` + +**Error responses:** + +| Status | Code | When | +|--------|------|------| +| 401 | `missing_or_malformed_auth` / `invalid_key` | Auth failure | +| 403 | `insufficient_scope` | Feed-scope key | +| 404 | `case_not_found` | Case ID does not exist | + --- ### IOC Feed -All feed endpoints require `feed` scope (a `full`-scope key also works), are read-only/idempotent, and support conditional requests via ETag. All three formats accept the **same query parameters** and apply the same dedup/ordering semantics — they differ only in serialization. +All feed endpoints require `feed` scope (a `full`-scope key also works), are read-only/idempotent, and support conditional requests via ETag. The four bulk formats (`iocs.json`, `iocs.csv`, `iocs.cef`, `iocs.stix`) accept the **same query parameters** and apply the same dedup/ordering semantics — they differ only in serialization. `iocs/lookup` is the exception: it takes a single required `value` instead (see its own section below). -**Shared query parameters:** +**Shared query parameters** (`iocs.json` / `iocs.csv` / `iocs.cef` / `iocs.stix`): | Param | Type | Default | Description | |-------|------|---------|-------------| @@ -808,6 +1029,7 @@ All feed endpoints require `feed` scope (a `full`-scope key also works), are rea - **Scoring:** each indicator's `score` derives from the **worst** severity recorded across its sightings: `low` = 25, `medium` = 50, `high` = 75, `critical` = 100; `severity` is the matching label. - **Deduplication:** the same indicator appearing in multiple analyses collapses to a single row carrying that maximum severity/score; `first_seen`/`last_seen` span all sightings and `case_ids` lists every contributing case. +- **`mitre_techniques`:** the deduplicated union of MITRE ATT&CK technique IDs (e.g. `T1071.001`) from every analysis that produced the indicator, derived from each analysis's ATT&CK mapping. Empty for indicators whose contributing analyses matched no techniques — not a placeholder; it reflects real detections. - **Filtering:** `min_score` is applied in SQL (not post-filtered), so it composes correctly with `limit`/`cursor` — pages are always full up to `limit` and no matching rows are dropped at page boundaries. - **Ordering:** deterministic — `last_seen` descending, then indicator value ascending as a tie-breaker. Stable ordering makes cursor pagination reliable. - **Pagination:** `next_cursor` is non-null exactly when the page came back full (`count == limit`); pass it as `cursor` for the next page. (CSV/STIX responses don't carry a cursor — page by incrementing `cursor` by `limit` while pages stay full.) @@ -841,7 +1063,7 @@ curl "http://localhost:8000/api/v1/iocs.json?min_score=50&type=ip,domain&limit=1 "first_seen": "2026-06-10T08:02:11.402199", "last_seen": "2026-06-12T09:15:40.992103", "case_ids": ["c4a1b2d9"], - "mitre_techniques": [] + "mitre_techniques": ["T1071.001"] }, { "type": "domain", @@ -887,6 +1109,68 @@ Example `422`: } ``` +#### `GET /api/v1/iocs/lookup` + +Exact-match single-IOC lookup — the #1 SOAR enrichment pattern (SIEM alert fires on an IP/domain, playbook asks "have we seen this before, and how bad?"). Unlike the bulk feed endpoints, this normally aggregates to 0 or 1 row, so `next_cursor` is always `null` and there is no `since`/`tag`/`case_id`/`cursor` filter. Matching is **exact** (`value = ?`, not a substring or prefix match) — looking up `1.2.3.4` never also returns `1.2.3.40`. An empty `value` is rejected with `422` rather than silently returning the whole feed. Read-only and idempotent. + +**Auth:** `feed` scope (or `full`) + +**Parameters:** + +| Param | In | Type | Required | Default | Description | +|-------|----|------|----------|---------|-------------| +| `value` | query | string | Yes | | Exact IOC value to look up (min length 1) | +| `min_score` | query | integer | No | `0` | Minimum threat score, 0–100 (422 outside that range) | +| `type` | query | string | No | (none) | Comma-separated IOC types to restrict to: `ip`, `domain`, `url`, `hash` | +| `limit` | query | integer | No | `1000` | Results per page, 1–10000 (422 outside that range) — irrelevant in practice since a single value aggregates to at most one row | + +**Sample request:** + +```bash +curl "http://localhost:8000/api/v1/iocs/lookup?value=198.51.100.42" \ + -H "Authorization: Bearer phk_4f8a2b9c1d3e5f60718293a4b5c6d7e8" +``` + +**Sample response — 200 OK** (found): + +```json +{ + "iocs": [ + { + "type": "ip", + "value": "198.51.100.42", + "severity": "high", + "score": 75, + "tags": ["malware", "c2-beacon"], + "first_seen": "2026-06-10T08:02:11.402199", + "last_seen": "2026-06-12T09:15:40.992103", + "case_ids": ["c4a1b2d9"], + "mitre_techniques": ["T1071.001"] + } + ], + "count": 1, + "next_cursor": null +} +``` + +**Sample response — 200 OK** (not found — an empty result, not a `404`): + +```json +{ + "iocs": [], + "count": 0, + "next_cursor": null +} +``` + +**Error responses:** + +| Status | Code | When | +|--------|------|------| +| 401 | `missing_or_malformed_auth` / `invalid_key` | Auth failure | +| 422 | `validation_error` | `value` missing/empty, or `min_score`/`limit` outside their allowed range | +| 429 | `rate_limit_exceeded` | DB key over its per-minute limit | + #### `GET /api/v1/iocs.csv` The same feed as CSV (`Content-Type: text/csv`) — built for lookup tables: Splunk lookups, Graylog CSV adapters, Wazuh CDB lists. Same query parameters, dedup, ordering, and ETag caching as the JSON feed. List-valued fields (`tags`, `case_ids`, `mitre_techniques`) are `;`-joined inside one CSV column. Values that a spreadsheet would interpret as a formula (leading `=`, `+`, `-`, `@`) are prefixed with a single quote to prevent CSV injection. @@ -904,12 +1188,33 @@ curl "http://localhost:8000/api/v1/iocs.csv?min_score=25" \ ```csv type,value,score,severity,tags,first_seen,last_seen,case_ids,mitre_techniques -ip,198.51.100.42,75,high,malware;c2-beacon,2026-06-10T08:02:11.402199,2026-06-12T09:15:40.992103,c4a1b2d9, +ip,198.51.100.42,75,high,malware;c2-beacon,2026-06-10T08:02:11.402199,2026-06-12T09:15:40.992103,c4a1b2d9,T1071.001 domain,updates.evil-cdn.example,100,critical,malware,2026-06-11T17:44:03.215587,2026-06-11T17:44:03.215587,b91e0f2c;c4a1b2d9, ``` **Error responses:** same as `iocs.json` (401 / 422 / 429), plus `405 method_not_allowed` for non-GET methods (applies to every endpoint). +#### `GET /api/v1/iocs.cef` + +The same feed as CEF (Common Event Format) syslog lines (`Content-Type: text/plain`) — for ArcSight/QRadar/Sentinel-style SIEM ingestion. Same query parameters, dedup, ordering, and ETag caching as the JSON feed. Uses a dedicated feed-to-CEF adapter rather than the app's general-purpose correlation/beacon CEF exporter, so it faithfully emits **every** row the filters selected — including LOW-severity IOCs, which the general exporter's `priority_score >= 0.4` gate would otherwise silently drop. Reserved CEF characters (`=`, `\`, CR, LF) in IOC values are escaped per the CEF spec, so an attacker-influenced indicator value can't inject a second event into the ingesting SIEM. + +**Auth:** `feed` scope (or `full`) + +**Sample request:** + +```bash +curl "http://localhost:8000/api/v1/iocs.cef?min_score=25" \ + -H "Authorization: Bearer phk_4f8a2b9c1d3e5f60718293a4b5c6d7e8" +``` + +**Sample response — 200 OK** (`text/plain`, one line per indicator): + +``` +Jun 12 09:15:41 pcap-hunter CEF:0|PCAPHunter|ThreatWorkbench|1.0|IOC-FEED-001|IOC Feed Entry (high)|8|value=198.51.100.42 type=ip score=75 tags=malware;c2-beacon +``` + +**Error responses:** same as `iocs.json` (401 / 422 / 429). + #### `GET /api/v1/iocs.stix` The same feed as a **STIX 2.1 bundle** of `indicator` objects, for STIX-native platforms (OpenCTI, MISP, TAXII ingest scripts). Also served at the alias path **`GET /api/v1/iocs/stix`** (identical behavior). Same query parameters and caching as the other formats. Indicator IDs are deterministic (UUIDv5 of the indicator value), so re-pulls produce stable IDs; IOC types map to STIX patterns: @@ -1425,6 +1730,7 @@ Revoking a key clears its rate-limit window immediately. | `rate_limit_exceeded` | 429 | Per-key rate limit exceeded (`Retry-After` header) | | `pcap_too_large` | 413 | File exceeds `max_pcap_bytes` | | `pcap_invalid_format` | 415 | Missing valid PCAP/pcapng magic signature | +| `invalid_webhook_url` | 422 | `webhook_url` fails the SSRF guard (non-http(s) scheme, unresolvable host, or a resolved address that's private/loopback/link-local/reserved/multicast) | | `queue_full` | 503 | Job queue at capacity (`Retry-After: 60` header) | | `job_not_found` | 404 | Job ID does not exist | | `result_not_ready` | 409 | Job has not finished (or finished `failed`/`cancelled`); additive `current_status` field | @@ -1483,6 +1789,16 @@ All settings are read from environment variables at startup. Defaults are suitab | `PCAP_HUNTER_API_QUEUE_DEPTH` | `100` | Maximum active (queued + running) jobs | | `PCAP_HUNTER_API_UPLOAD_TIMEOUT_SEC` | `600` | Reserved — parsed but **not currently enforced** | +### Webhooks + +Settings for the optional completion webhook (see [Completion webhook](#completion-webhook-webhook_url) under `POST /api/v1/pcaps`). Only relevant when a submission includes `webhook_url`. + +| Variable | Default | Description | +|----------|---------|-------------| +| `PCAP_HUNTER_API_WEBHOOK_TIMEOUT_SECONDS` | `10` | Per-attempt HTTP timeout (seconds) for the completion callback POST | +| `PCAP_HUNTER_API_WEBHOOK_MAX_RETRIES` | `2` | Additional attempts after the first on a non-2xx response or request exception (default: 3 attempts total) | +| `PCAP_HUNTER_API_WEBHOOK_SECRET` | (none — signing disabled) | When set, every callback carries `X-PCAP-Hunter-Signature: sha256=`, an HMAC-SHA256 of the raw JSON body keyed with this secret | + ### Retention | Variable | Default | Description | From 0121cb50a42d91f1c6b7b2221a0cca354f83ae08 Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Tue, 14 Jul 2026 03:25:08 +0700 Subject: [PATCH 26/35] test: de-flake stale-date usage tests and timezone regex for green ci --- tests/api/test_key_repository.py | 27 +++++++++++++++++---------- tests/test_pdf_generator.py | 11 +++++++++-- tests/test_pdf_integration.py | 7 ++++++- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/tests/api/test_key_repository.py b/tests/api/test_key_repository.py index 39b029c..57456ad 100644 --- a/tests/api/test_key_repository.py +++ b/tests/api/test_key_repository.py @@ -100,11 +100,14 @@ def test_increment_usage_upsert(tmp_path): repo = _repo(tmp_path) _, key = _make_key() kid = repo.create_key(key) - repo.increment_usage(kid, "2026-05-23", 5) - repo.increment_usage(kid, "2026-05-23", 3) + # Use a date well within the 30-day retrieval window so this test doesn't + # rot as "today" drifts away from a hardcoded date (see get_usage's cutoff). + day = (datetime.now() - timedelta(days=2)).strftime("%Y-%m-%d") + repo.increment_usage(kid, day, 5) + repo.increment_usage(kid, day, 3) usage = repo.get_usage(kid, days=30) # Should have one entry with 8 total - day_data = [u for u in usage if u["date"] == "2026-05-23"] + day_data = [u for u in usage if u["date"] == day] assert len(day_data) == 1 assert day_data[0]["requests"] == 8 # total_requests on key should also be 8 @@ -116,13 +119,16 @@ def test_get_usage_returns_daily_data(tmp_path): repo = _repo(tmp_path) _, key = _make_key() kid = repo.create_key(key) - repo.increment_usage(kid, "2026-05-22", 10) - repo.increment_usage(kid, "2026-05-23", 20) + # Two distinct, consecutive days, both well within the 30-day window. + day_one = (datetime.now() - timedelta(days=3)).strftime("%Y-%m-%d") + day_two = (datetime.now() - timedelta(days=2)).strftime("%Y-%m-%d") + repo.increment_usage(kid, day_one, 10) + repo.increment_usage(kid, day_two, 20) usage = repo.get_usage(kid, days=30) assert len(usage) == 2 dates = [u["date"] for u in usage] - assert "2026-05-22" in dates - assert "2026-05-23" in dates + assert day_one in dates + assert day_two in dates def test_get_usage_summary(tmp_path): @@ -131,10 +137,11 @@ def test_get_usage_summary(tmp_path): _, k2 = _make_key(name="k2") id1 = repo.create_key(k1) id2 = repo.create_key(k2) - repo.increment_usage(id1, "2026-05-23", 10) - repo.increment_usage(id2, "2026-05-23", 5) + day = (datetime.now() - timedelta(days=2)).strftime("%Y-%m-%d") + repo.increment_usage(id1, day, 10) + repo.increment_usage(id2, day, 5) summary = repo.get_usage_summary(days=30) - day_data = [s for s in summary if s["date"] == "2026-05-23"] + day_data = [s for s in summary if s["date"] == day] assert len(day_data) == 1 assert day_data[0]["requests"] == 15 diff --git a/tests/test_pdf_generator.py b/tests/test_pdf_generator.py index c62b12e..dcb2fec 100644 --- a/tests/test_pdf_generator.py +++ b/tests/test_pdf_generator.py @@ -247,13 +247,20 @@ def test_cover_page_timestamp_is_timezone_aware(self): """The cover 'Generated:' stamp must carry a timezone abbreviation.""" gen = PDFReportGenerator() html = gen._render_cover_page(case_info=None) - assert re.search(r"Generated: \d{4}-\d{2}-\d{2} \d{2}:\d{2} [A-Z]{2,5}", html) + # %Z renders as an alpha abbreviation (e.g. "UTC") on some hosts/CI and + # as a numeric offset (e.g. "+07") on others (e.g. macOS) — either + # satisfies "carries a timezone indicator". + assert re.search( + r"Generated: \d{4}-\d{2}-\d{2} \d{2}:\d{2} (?:[A-Z]{2,5}|[+-]\d{2}(?::?\d{2})?)", html + ) def test_appendix_timestamp_is_timezone_aware(self): """The appendix 'Generated:' stamp must carry a timezone abbreviation.""" gen = PDFReportGenerator() html = gen._render_appendix({"flows": [], "artifacts": {}}) - assert re.search(r"Generated: \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [A-Z]{2,5}", html) + # See test_cover_page_timestamp_is_timezone_aware: accept both an alpha + # tz abbreviation and a numeric UTC offset. + assert re.search(r"Generated: \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} (?:[A-Z]{2,5}|[+-]\d{2}(?::?\d{2})?)", html) def test_generate_without_weasyprint(self): """Test generation when weasyprint is not available.""" diff --git a/tests/test_pdf_integration.py b/tests/test_pdf_integration.py index 185ea3e..7dd037a 100644 --- a/tests/test_pdf_integration.py +++ b/tests/test_pdf_integration.py @@ -528,7 +528,12 @@ def test_numbering_stays_sequential_when_conditional_sections_absent(self, reali assert toc_ids == ["summary", "iocs", "flows", "appendix"] def test_report_timestamp_is_timezone_aware(self, full_report_html): - assert re.search(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}(:\d{2})? [A-Z]{2,5}", full_report_html) + # %Z renders as an alpha abbreviation (e.g. "UTC") on some hosts/CI and + # as a numeric offset (e.g. "+07") on others (e.g. macOS) — either + # satisfies "carries a timezone indicator". + assert re.search( + r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}(:\d{2})? (?:[A-Z]{2,5}|[+-]\d{2}(?::?\d{2})?)", full_report_html + ) class TestLLMMarkdownTableRendering: From a37fb1659408a9aaa991046ff9d4b06c493e68e7 Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Tue, 14 Jul 2026 03:39:27 +0700 Subject: [PATCH 27/35] ci: enforce coverage floor and add pytest/coverage config Measured total coverage is 61% (12239 statements, 1189 passed / 22 skipped, 0 failed). Set --cov-fail-under=58 (measured minus 3 points) on the CI test step and the Makefile test target so CI enforces a floor without flaking on minor Linux/branch-coverage differences vs this host run. Also add [tool.pytest.ini_options] (permissive filterwarnings) and [tool.coverage.report] (exclude_lines for pragma/TYPE_CHECKING/NotImplementedError) to pyproject.toml. --- .github/workflows/ci.yml | 2 +- Makefile | 2 +- pyproject.toml | 6 ++++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae57209..8c9eb7c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,7 +51,7 @@ jobs: run: pip install -r requirements.txt - name: Run tests with coverage - run: PYTHONPATH=. pytest tests/ -v --cov=app + run: PYTHONPATH=. pytest tests/ -v --cov=app --cov-report=term-missing --cov-fail-under=58 - name: Ruff lint run: ruff check . diff --git a/Makefile b/Makefile index ee33bcc..6bb7b9d 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ check-deps doctor: # ------------------------------------------------------------------------- test: - PYTHONPATH=. pytest tests/ -v --cov=app + PYTHONPATH=. pytest tests/ -v --cov=app --cov-report=term-missing --cov-fail-under=58 # Focused smoke test: PDF generation with charts + correlations + beacon DF. # Run this after any change to pdf_generator.py, chart_images.py, main.py diff --git a/pyproject.toml b/pyproject.toml index e78fb42..aca347f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,3 +17,9 @@ quote-style = "double" indent-style = "space" skip-magic-trailing-comma = false line-ending = "auto" + +[tool.pytest.ini_options] +filterwarnings = ["ignore::DeprecationWarning"] + +[tool.coverage.report] +exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:", "raise NotImplementedError"] From c08e4b9629070af068b61aebb44ece7504da91b1 Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Tue, 14 Jul 2026 03:50:11 +0700 Subject: [PATCH 28/35] ci: add pip-audit job and dependabot config --- .github/dependabot.yml | 8 ++++++++ .github/workflows/ci.yml | 24 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..f4345a9 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ +version: 2 +updates: + - package-ecosystem: pip + directory: "/" + schedule: {interval: weekly} + - package-ecosystem: github-actions + directory: "/" + schedule: {interval: weekly} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c9eb7c..44757ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,3 +58,27 @@ jobs: - name: Ruff format check run: ruff format --check . + + audit: + runs-on: ubuntu-latest + timeout-minutes: 30 + # Advisory only: a discovered CVE surfaces here without blocking merges. + # The required check for branch protection stays `test`. + continue-on-error: true + steps: + - name: Check out repository + uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Set up Python 3.11 + uses: actions/setup-python@v6 + with: + python-version: "3.11" + cache: pip + + - name: Install pip-audit + run: pip install pip-audit + + - name: Audit dependencies + run: pip-audit -r requirements.txt From 99b5155db380476d62db5845d34f1c635d07e082 Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Tue, 14 Jul 2026 03:53:20 +0700 Subject: [PATCH 29/35] ci: build and test the docker image (with zeek) in ci --- .github/workflows/ci.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44757ee..565e4b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,3 +82,36 @@ jobs: - name: Audit dependencies run: pip-audit -r requirements.txt + + docker: + runs-on: ubuntu-latest + timeout-minutes: 40 + # Advisory only: the Dockerfile is the canonical deploy artifact (two + # Zeek OBS-repo installs, WeasyPrint/kaleido system libs, multi-stage + # builder->runtime->test) but the `test` job above never builds it, so a + # broken apt pin or requirements resolution failure would only surface + # when someone runs `make docker-build`. This job builds the `test` + # target and runs its suite, which also has real zeek installed (the + # `test` job above skips zeek-dependent e2e tests). It is deliberately + # NOT the required check — that stays `test`. + steps: + - name: Check out repository + uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build test-stage image (mirrors make docker-verify) + uses: docker/build-push-action@v6 + with: + context: . + target: test + tags: pcap-hunter:test + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Run in-image test suite (ruff format + lint + pytest, with real zeek) + run: docker run --rm pcap-hunter:test From 8683880b696f5c2238b4f0c5cc374822a526e7bd Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Tue, 14 Jul 2026 04:03:43 +0700 Subject: [PATCH 30/35] feat: add synthetic demo pcap (dns/http/beacon) for first-run experience --- .gitignore | 1 + Dockerfile | 4 + pcaps/_make_demo.py | 307 +++++++++++++++++++++++++++++++++++++++++ pcaps/demo.pcap | Bin 0 -> 3322 bytes tests/test_fixtures.py | 19 +++ 5 files changed, 331 insertions(+) create mode 100644 pcaps/_make_demo.py create mode 100644 pcaps/demo.pcap diff --git a/.gitignore b/.gitignore index c08213e..40e7f1d 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ htmlcov/ data/ *.pcap !tests/fixtures/*.pcap +!pcaps/demo.pcap *.log .DS_Store diff --git a/Dockerfile b/Dockerfile index 87b7866..1d8809e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,6 +61,10 @@ RUN pip install --no-cache-dir /wheels/* # The previous flattened COPY (app/ -> /app/) broke `uvicorn app.api.app`. COPY app/ ./app/ +# Synthetic first-run demo capture (see pcaps/_make_demo.py) — powers the +# "Load demo capture" button so a fresh install has something to analyze. +COPY pcaps/demo.pcap ./pcaps/demo.pcap + # Non-root + data dirs RUN useradd -m runner && mkdir -p /data /app/data && chown -R runner:runner /app /data USER runner diff --git a/pcaps/_make_demo.py b/pcaps/_make_demo.py new file mode 100644 index 0000000..e4c7e13 --- /dev/null +++ b/pcaps/_make_demo.py @@ -0,0 +1,307 @@ +"""Generate ``pcaps/demo.pcap`` deterministically. + +Run once locally to regenerate the fixture if it's lost or needs to evolve: + + python pcaps/_make_demo.py + +The committed ``demo.pcap`` is small (a few KB, dozens of packets) and +entirely SYNTHETIC — license-clean by construction. It exists so a first-run +user has something to load via the "Load demo capture" button instead of +facing an empty app, and so that loading it actually exercises real +detection paths in the pipeline: + +- DNS query/response pairs (UDP/53): two normal-looking domains plus one + high-entropy, DGA-looking domain that ``app.pipeline.dns_analysis.detect_dga`` + scores above its confirmation threshold. +- A minimal TLS ClientHello (TCP/443) with an SNI extension, for protocol + variety. +- Two HTTP GET requests (TCP/80): one with a normal browser User-Agent, one + with a scripting-tool User-Agent (``python-requests``) that + ``app.pipeline`` HTTP heuristics flag as suspicious. +- A periodic TCP beacon (13 packets, exactly 30s apart) to a well-known C2 + port (4444, see ``app.config.C2_SUSPECT_PORTS``) so + ``app.pipeline.beacon.rank_beaconing`` has a clear candidate. + +All destination IPs are from documentation/test ranges (RFC 5737 +TEST-NET-3 ``203.0.113.0/24`` and TEST-NET-2 ``198.51.100.0/24``) — publicly +reserved for exactly this purpose, never routed on the real Internet — so +the capture is obviously synthetic and safe to publish. The source is an +RFC 1918 client. Ethernet MACs and every packet timestamp are pinned (like +``tests/fixtures/_make_tiny.py``) so regenerating the fixture produces +byte-identical output, no matter where or when the script runs. + +The DGA domain resolves (in the synthetic DNS response) to the same IP the +beacon later targets, and the second "normal" domain resolves to the TLS +server IP — a small, deliberate narrative thread for anyone poking at the +demo data, not something the pipeline depends on. + +Tests must NOT import this module — they read the committed ``demo.pcap`` +file directly. This module is a developer tool, not test code. +""" + +from __future__ import annotations + +import pathlib +import struct + +import scapy.all as scapy + +FIXTURE_PATH = pathlib.Path(__file__).parent / "demo.pcap" + +# Pinned epoch for deterministic record timestamps: 2026-01-01T00:00:00 UTC +# (same anchor as tests/fixtures/_make_tiny.py). +BASE_EPOCH = 1767225600 + +# --- Deterministic L2/L3 addressing --- +CLIENT_MAC = "00:00:00:00:00:01" +SERVER_MAC = "00:00:00:00:00:02" +CLIENT_IP = "10.0.0.50" # RFC 1918 + +# Documentation/test-net destinations (RFC 5737) — never routed publicly. +DNS_SERVER_IP = "203.0.113.53" +NORMAL_HTTP_IP = "198.51.100.10" +TLS_SERVER_IP = "203.0.113.50" +SUSPICIOUS_HTTP_IP = "203.0.113.10" +BEACON_IP = "203.0.113.99" +BEACON_PORT = 4444 # in app.config.C2_SUSPECT_PORTS + +# --- Domains --- +DOMAIN_NORMAL_1 = "www.example.com" +DOMAIN_NORMAL_2 = "vault.example.net" +# High-entropy, digit-heavy, all-consonant-alpha synthetic name — not a real +# registered domain. Scores well above app.pipeline.dns_analysis's DGA +# confirmation threshold (0.5) via digit-ratio + consonant-ratio + no-vowels +# heuristics. +DOMAIN_DGA = "x7k2p9qz3v.com" + + +def _eth_ip(src_ip: str, dst_ip: str): + """Return an Ether/IP layer stack with MACs pinned by direction.""" + if src_ip == CLIENT_IP: + return scapy.Ether(src=CLIENT_MAC, dst=SERVER_MAC) / scapy.IP(src=src_ip, dst=dst_ip) + return scapy.Ether(src=SERVER_MAC, dst=CLIENT_MAC) / scapy.IP(src=src_ip, dst=dst_ip) + + +def _dns_pair(ts: float, query_id: int, domain: str, resolved_ip: str, sport: int) -> list: + """Build a DNS query + response packet pair for ``domain``.""" + query = ( + _eth_ip(CLIENT_IP, DNS_SERVER_IP) + / scapy.UDP(sport=sport, dport=53) + / scapy.DNS(id=query_id, rd=1, qd=scapy.DNSQR(qname=domain)) + ) + query.time = ts + + response = ( + _eth_ip(DNS_SERVER_IP, CLIENT_IP) + / scapy.UDP(sport=53, dport=sport) + / scapy.DNS( + id=query_id, + qr=1, + aa=1, + rd=1, + ra=1, + qd=scapy.DNSQR(qname=domain), + an=scapy.DNSRR(rrname=domain, type="A", ttl=300, rdata=resolved_ip), + ) + ) + response.time = ts + 0.05 + + return [query, response] + + +def _tcp_http_exchange( + ts: float, client_ip: str, server_ip: str, sport: int, dport: int, req_bytes: bytes, resp_bytes: bytes +) -> list: + """Build a minimal SYN/SYN-ACK/ACK + PSH request/response TCP exchange.""" + cli_seq = 1000 + srv_seq = 2000 + pkts = [] + + syn = _eth_ip(client_ip, server_ip) / scapy.TCP(sport=sport, dport=dport, flags="S", seq=cli_seq) + syn.time = ts + pkts.append(syn) + + synack = _eth_ip(server_ip, client_ip) / scapy.TCP( + sport=dport, dport=sport, flags="SA", seq=srv_seq, ack=cli_seq + 1 + ) + synack.time = ts + 0.01 + pkts.append(synack) + + ack = _eth_ip(client_ip, server_ip) / scapy.TCP( + sport=sport, dport=dport, flags="A", seq=cli_seq + 1, ack=srv_seq + 1 + ) + ack.time = ts + 0.02 + pkts.append(ack) + + request = ( + _eth_ip(client_ip, server_ip) + / scapy.TCP(sport=sport, dport=dport, flags="PA", seq=cli_seq + 1, ack=srv_seq + 1) + / scapy.Raw(load=req_bytes) + ) + request.time = ts + 0.03 + pkts.append(request) + + response = ( + _eth_ip(server_ip, client_ip) + / scapy.TCP(sport=dport, dport=sport, flags="PA", seq=srv_seq + 1, ack=cli_seq + 1 + len(req_bytes)) + / scapy.Raw(load=resp_bytes) + ) + response.time = ts + 0.04 + pkts.append(response) + + final_ack = _eth_ip(client_ip, server_ip) / scapy.TCP( + sport=sport, + dport=dport, + flags="A", + seq=cli_seq + 1 + len(req_bytes), + ack=srv_seq + 1 + len(resp_bytes), + ) + final_ack.time = ts + 0.05 + pkts.append(final_ack) + + return pkts + + +def _build_tls_client_hello(sni: str) -> bytes: + """Build a minimal, well-formed TLS 1.2 ClientHello record with an SNI extension. + + Not a real handshake (no key share, only two cipher suites) — just enough + structure for a dissector to recognize it as a TLS ClientHello carrying a + server name, for protocol variety in the demo capture. + """ + sni_bytes = sni.encode() + server_name_entry = b"\x00" + struct.pack(">H", len(sni_bytes)) + sni_bytes # type=host_name(0) + server_name_list = struct.pack(">H", len(server_name_entry)) + server_name_entry + ext_server_name = struct.pack(">HH", 0x0000, len(server_name_list)) + server_name_list + extensions = ext_server_name + + cipher_suites = bytes([0xC0, 0x2F, 0xC0, 0x30, 0x00, 0x9C, 0x00, 0x9D]) + random_bytes = bytes(range(32)) # deterministic filler, not real randomness + + body = ( + b"\x03\x03" # client_version: TLS 1.2 + + random_bytes + + b"\x00" # session_id length: 0 + + struct.pack(">H", len(cipher_suites)) + + cipher_suites + + b"\x01\x00" # compression methods: length 1, method null(0) + + struct.pack(">H", len(extensions)) + + extensions + ) + handshake = b"\x01" + len(body).to_bytes(3, "big") + body # handshake type 1 = ClientHello + return b"\x16\x03\x01" + struct.pack(">H", len(handshake)) + handshake # content type 22 = Handshake + + +def _tls_hello_exchange(ts: float, client_ip: str, server_ip: str, sport: int, dport: int, sni: str) -> list: + """Build a minimal SYN/SYN-ACK/ACK + ClientHello TCP exchange.""" + cli_seq = 3000 + srv_seq = 4000 + pkts = [] + + syn = _eth_ip(client_ip, server_ip) / scapy.TCP(sport=sport, dport=dport, flags="S", seq=cli_seq) + syn.time = ts + pkts.append(syn) + + synack = _eth_ip(server_ip, client_ip) / scapy.TCP( + sport=dport, dport=sport, flags="SA", seq=srv_seq, ack=cli_seq + 1 + ) + synack.time = ts + 0.01 + pkts.append(synack) + + ack = _eth_ip(client_ip, server_ip) / scapy.TCP( + sport=sport, dport=dport, flags="A", seq=cli_seq + 1, ack=srv_seq + 1 + ) + ack.time = ts + 0.02 + pkts.append(ack) + + hello_bytes = _build_tls_client_hello(sni) + hello = ( + _eth_ip(client_ip, server_ip) + / scapy.TCP(sport=sport, dport=dport, flags="PA", seq=cli_seq + 1, ack=srv_seq + 1) + / scapy.Raw(load=hello_bytes) + ) + hello.time = ts + 0.03 + pkts.append(hello) + + return pkts + + +def _beacon_packets( + start_ts: float, count: int, interval: float, client_ip: str, server_ip: str, sport: int, dport: int +) -> list: + """Build ``count`` small TCP packets at an exact ``interval`` — a periodic beacon.""" + pkts = [] + seq = 5000 + for i in range(count): + payload = f"chk-in-{i:02d}".encode() + pkt = ( + _eth_ip(client_ip, server_ip) + / scapy.TCP(sport=sport, dport=dport, flags="PA", seq=seq, ack=1) + / scapy.Raw(load=payload) + ) + pkt.time = start_ts + i * interval + seq += len(payload) + pkts.append(pkt) + return pkts + + +def build_packets() -> list: + """Return a deterministic, chronologically ordered list of demo packets.""" + pkts: list = [] + + # DNS: two normal lookups, one DGA-looking lookup. + pkts += _dns_pair(BASE_EPOCH + 0, 1, DOMAIN_NORMAL_1, NORMAL_HTTP_IP, sport=40001) + pkts += _dns_pair(BASE_EPOCH + 1, 2, DOMAIN_NORMAL_2, TLS_SERVER_IP, sport=40002) + pkts += _dns_pair(BASE_EPOCH + 2, 3, DOMAIN_DGA, BEACON_IP, sport=40003) + + # TLS: minimal ClientHello with SNI, for protocol variety. + pkts += _tls_hello_exchange(BASE_EPOCH + 5, CLIENT_IP, TLS_SERVER_IP, sport=50001, dport=443, sni=DOMAIN_NORMAL_2) + + # HTTP: suspicious tool User-Agent hitting a hardcoded IP (no prior DNS lookup). + suspicious_req = ( + b"GET /gate.php?id=1 HTTP/1.1\r\n" + b"Host: 203.0.113.10\r\n" + b"User-Agent: python-requests/2.31.0\r\n" + b"Accept: */*\r\n" + b"Connection: close\r\n\r\n" + ) + suspicious_resp = b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 2\r\n\r\nOK" + pkts += _tcp_http_exchange( + BASE_EPOCH + 10, CLIENT_IP, SUSPICIOUS_HTTP_IP, 51000, 80, suspicious_req, suspicious_resp + ) + + # HTTP: normal browser User-Agent to the site the client actually resolved. + normal_req = ( + b"GET /index.html HTTP/1.1\r\n" + b"Host: www.example.com\r\n" + b"User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36\r\n" + b"Accept: text/html\r\n" + b"Connection: close\r\n\r\n" + ) + normal_resp = b"HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: 13\r\n\r\nHello, world!" + pkts += _tcp_http_exchange(BASE_EPOCH + 15, CLIENT_IP, NORMAL_HTTP_IP, 51001, 80, normal_req, normal_resp) + + # Beacon: 13 packets, exactly 30s apart, to a well-known C2 port. + pkts += _beacon_packets( + BASE_EPOCH + 20, + count=13, + interval=30.0, + client_ip=CLIENT_IP, + server_ip=BEACON_IP, + sport=55000, + dport=BEACON_PORT, + ) + + pkts.sort(key=lambda p: p.time) + return pkts + + +def main() -> None: + pkts = build_packets() + scapy.wrpcap(str(FIXTURE_PATH), pkts) + size = FIXTURE_PATH.stat().st_size + print(f"wrote {FIXTURE_PATH} ({size} bytes, {len(pkts)} packets)") + + +if __name__ == "__main__": + main() diff --git a/pcaps/demo.pcap b/pcaps/demo.pcap new file mode 100644 index 0000000000000000000000000000000000000000..ec84e9c279ab560ec1f2c7e8ff5de10e0e18e863 GIT binary patch literal 3322 zcmbW3ZD?C%6vt0qx^{PFZP(h?Ip>`qw%c-ZZ{E7L8*NuwtLfGhQU)?4F}*Zt)0?!p zZL(2B7-EqM%DzcO$G(YEnRZ(pN};d{1938(eiH??`awZI3iF)jzAkGq-UBCT?#u5v z|8xG&y}i3}^^zOBV8s3V051BmHlC%gn4a`ExY?(x4h8{E&}Ds%;b9IScpGLTEBz3G zj&G9S!V?Su;3<_#wQ@N%Q^?7lbbf|TxCkBJ0GL7=B@68aH$$J*(H(>(JP>^N))u&4 zeT5P5mTl479>hhDg7;Qvf^+GjPT~EOEbO9s^_7QAQ72KK$I`VOdAnn#qI8X} zD2fYGval;^XY1|?PwR?;8)L4U0DErGm2o!@9v#YjlH@D{z-`@I$#(hoAkG8b7J+}Yx15X@$5}h(fGqg2)F%38LNbccGqFs@Ok*H^GUN*4BBzpMB9K!qZWg1!C&?egqkf)7llR9Prw z3ccBh6B0k1NW=q@Ao;lAd{OP@gJMV!1xX4ClIY{ci}IYme^OTPOyQiG$t(Ujd3Ihd zs>MK12uT87)1OYu1w4N^aM;HUhTRgA`{ z4%+(vxdkgm;Z36RnrIaBsG`0~a}6bF6!e_l?0V2#Z=!ahRAa88BoTCTb=l3e#A}pl z%=J($rn#05X%sTo_c$0z7It&p*5)(#d4(d=T)(XA=E^D)a#_fzGr5PHh3`&tkK()Y zL35qTpU>uUsX#;!`SvqN$(M@!=>#u{g4o03foQmgFGs^2e18F7zGvi@Vp%m133Upg zsAWOs0a|HB@4R|=QW{2O%+WiBQBE>xvA=mXcoa7YWPqJkr9XGTrPi< zFXiWQ6NjjzYCKsvLGLnp@*sKgH3^_u_hc_BqGVy8prr3Gi6)4 100, "demo pcap is suspiciously small; regenerate" + + +def test_demo_pcap_has_valid_magic(): + """First 4 bytes match a known pcap or pcapng magic.""" + head = DEMO_PCAP.read_bytes()[:4] + valid_magics = { + b"\xd4\xc3\xb2\xa1", # pcap classic, little-endian + b"\xa1\xb2\xc3\xd4", # pcap classic, byte-swapped + b"\x4d\x3c\xb2\xa1", # pcap nanosecond + b"\xa1\xb2\x3c\x4d", # pcap nanosecond, byte-swapped + b"\x0a\x0d\x0d\x0a", # pcapng + } + assert head in valid_magics, f"unexpected magic bytes: {head!r}" From 49f86384c4bf29a35bc45f46639db9a132d2cffd Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Tue, 14 Jul 2026 04:07:53 +0700 Subject: [PATCH 31/35] chore: dockerignore-proof pcaps/demo.pcap against Docker build context excludes Bare '*.pcap' currently only matches root-level files in the installed BuildKit version (verified empirically), so pcaps/demo.pcap already survives the runtime image build context. Add the negation anyway, mirroring the .gitignore fix, so a future tightening to a recursive pattern (e.g. '**/*.pcap') can't silently break the demo capture COPY. --- .dockerignore | 1 + pcaps/_make_demo.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.dockerignore b/.dockerignore index e2b45d7..a7ca768 100644 --- a/.dockerignore +++ b/.dockerignore @@ -16,3 +16,4 @@ docker-compose.yml *.db *.pcap *.pcapng +!pcaps/demo.pcap diff --git a/pcaps/_make_demo.py b/pcaps/_make_demo.py index e4c7e13..8b222c2 100644 --- a/pcaps/_make_demo.py +++ b/pcaps/_make_demo.py @@ -11,8 +11,9 @@ detection paths in the pipeline: - DNS query/response pairs (UDP/53): two normal-looking domains plus one - high-entropy, DGA-looking domain that ``app.pipeline.dns_analysis.detect_dga`` - scores above its confirmation threshold. + digit-heavy, all-consonant DGA-looking domain that + ``app.pipeline.dns_analysis.detect_dga`` scores above its confirmation + threshold (via the digit-ratio / consonant-ratio / no-vowel bonuses). - A minimal TLS ClientHello (TCP/443) with an SNI extension, for protocol variety. - Two HTTP GET requests (TCP/80): one with a normal browser User-Agent, one @@ -68,7 +69,7 @@ # --- Domains --- DOMAIN_NORMAL_1 = "www.example.com" DOMAIN_NORMAL_2 = "vault.example.net" -# High-entropy, digit-heavy, all-consonant-alpha synthetic name — not a real +# Digit-heavy, all-consonant-alpha synthetic name — not a real # registered domain. Scores well above app.pipeline.dns_analysis's DGA # confirmation threshold (0.5) via digit-ratio + consonant-ratio + no-vowels # heuristics. From 49a1bbcad07ca3d4c93e820513ba6695b47177a9 Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Tue, 14 Jul 2026 04:21:05 +0700 Subject: [PATCH 32/35] feat: add load-demo-capture button and point screenshots at demo pcap Lets a first-run user try the pipeline immediately via a one-click "Load demo capture" button on the Upload tab, wired to the committed pcaps/demo.pcap. Also repoints scripts/capture_screenshots.py's default sample pcap from the never-committed data/sample.pcap to pcaps/demo.pcap so the screenshot flow works out of the box. --- app/main.py | 15 ++++++++++++++- scripts/capture_screenshots.py | 18 +++++++++--------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/app/main.py b/app/main.py index 3c4f8cd..dd7f291 100644 --- a/app/main.py +++ b/app/main.py @@ -351,7 +351,8 @@ def _run_single_pcap_pipeline( with st.container(border=True): st.markdown("##### 🚀 Getting started") st.markdown( - "1. **Upload a PCAP** below (or type a container path).\n" + "1. **Upload a PCAP** below (or type a container path) — or click " + "**Load demo capture** to try a bundled synthetic sample.\n" "2. Click **Extract & Analyze** — the 10-stage pipeline parses packets (tshark/PyShark), " "runs Zeek, hunts DNS/TLS/beaconing anomalies, carves HTTP payloads, scans with YARA, " "and enriches IOCs via OSINT.\n" @@ -374,6 +375,18 @@ def _run_single_pcap_pipeline( with col_b: pcap_path_text = st.text_input("...or type a container path (e.g., /data/capture.pcap)", value="") + _demo_path = validate_pcap_path(str(pathlib.Path("pcaps/demo.pcap").resolve())) + if _demo_path: + if st.button( + "Load demo capture", + key="_load_demo", + help="Analyze a bundled synthetic capture (DNS/HTTP/beacon) to try the pipeline.", + ): + st.session_state["__pcap_path"] = _demo_path + st.session_state["__pcap_paths"] = [_demo_path] + st.session_state["__batch_mode"] = False + st.rerun() + ensure_dir(C.DATA_DIR) ensure_dir(C.ZEEK_DIR) ensure_dir(C.CARVE_DIR) diff --git a/scripts/capture_screenshots.py b/scripts/capture_screenshots.py index 887c4f8..14a9533 100644 --- a/scripts/capture_screenshots.py +++ b/scripts/capture_screenshots.py @@ -1,10 +1,10 @@ """Capture README screenshots of the PCAP Hunter UI and redact all IPs. Drives a headless Chromium via Playwright against a running Streamlit -instance (default http://localhost:8501), uploads ``data/sample.pcap`` -by entering its path into the "type a container path" text input, -clicks Extract & Analyze, waits for the pipeline to finish, then -snapshots each tab at 1440×900. +instance (default http://localhost:8501), uploads ``pcaps/demo.pcap`` +(the committed synthetic DNS/HTTP/beacon capture) by entering its path +into the "type a container path" text input, clicks Extract & Analyze, +waits for the pipeline to finish, then snapshots each tab at 1440×900. After capture, every PNG is post-processed with Pillow to redact IPv4 addresses. We do this at the pixel level using OCR-free regex scanning @@ -32,7 +32,7 @@ REPO_ROOT = Path(__file__).resolve().parent.parent OUT_DIR = REPO_ROOT / "docs" / "images" -SAMPLE_PCAP = REPO_ROOT / "data" / "sample.pcap" +SAMPLE_PCAP = REPO_ROOT / "pcaps" / "demo.pcap" # IPv4 pattern — matches anything that looks like a.b.c.d with 0-255 octets. # We redact any IP, including RFC1918/loopback — the goal is zero IPs visible. @@ -106,7 +106,7 @@ def upload_sample_pcap(page: Page, pcap_path: str) -> None: """Fill the 'type a container path' text input to load the sample. ``pcap_path`` must be valid for the SERVER process — when the app runs in - Docker that means a container-visible path like ``data/sample.pcap``, not + Docker that means a container-visible path like ``pcaps/demo.pcap``, not the host's absolute path. """ path_input = page.get_by_label("...or type a container path (e.g., /data/capture.pcap)") @@ -139,7 +139,7 @@ def run_extract_analyze(page: Page, wait_for_llm: bool = False, timeout_s: int = if "Please upload a PCAP or provide a valid path" in page.inner_text("body"): raise RuntimeError( "the server rejected the pcap path — pass a path the SERVER can see " - "(container-relative like data/sample.pcap when using Docker)" + "(container-relative like pcaps/demo.pcap when using Docker)" ) print(f" waiting for pipeline (up to {timeout_s}s)...") deadline = time.time() + timeout_s @@ -480,7 +480,7 @@ def main() -> int: ) parser.add_argument( "--pcap", - default="data/sample.pcap", + default="pcaps/demo.pcap", help="pcap path AS THE SERVER SEES IT (container-relative when the app runs in Docker)", ) parser.add_argument("--pipeline-timeout", type=int, default=600, help="seconds to wait for the data stages") @@ -498,7 +498,7 @@ def main() -> int: return 0 if not SAMPLE_PCAP.is_file() and not args.skip_analysis: - print(f"ERROR: {SAMPLE_PCAP} not found. Copy sample.pcap → data/ first.", file=sys.stderr) + print(f"ERROR: {SAMPLE_PCAP} not found. Demo capture should be committed at pcaps/demo.pcap.", file=sys.stderr) return 1 redact = not args.keep_ips From b9e7763b9e01232bef260670f25576cd14670b2a Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Tue, 14 Jul 2026 04:28:06 +0700 Subject: [PATCH 33/35] docs: refresh roadmap, add security/contributing, fix kaleido changelog note --- CHANGELOG.md | 2 +- CONTRIBUTING.md | 142 ++++++++++ SECURITY.md | 67 +++++ docs/FEATURE-ROADMAP.md | 554 +++++----------------------------------- 4 files changed, 274 insertions(+), 491 deletions(-) create mode 100644 CONTRIBUTING.md create mode 100644 SECURITY.md diff --git a/CHANGELOG.md b/CHANGELOG.md index f882c19..eee6186 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ First stable release. Production-ready installer, hardened pipeline, polished UX ### Changed - **PDF cover page redesign** — logo + tagline above the title, with classification banner and metadata block. -- **Kaleido upgraded to 1.x** — 0.x reaches end-of-life September 2025; 1.x is the active branch. +- **Kaleido pinned to `0.2.1`** — kaleido 1.x refuses to install alongside the pinned plotly 5.x ("not compatible"), so 0.2.1 is the only working pairing; it bundles its own headless Chromium and needs no system browser. See `requirements.txt` for the full rationale, and revisit together with a future `plotly>=6.1` upgrade. - **Testing discipline overhauled** — production-shape test data (real `CorrelationSignal` dataclasses, real DataFrames, nested dicts the pipeline actually produces) instead of simplified inputs. Documented in `CLAUDE.md` with bug-pattern history. New integration tests for every PDF section and chart. - **Version bumped to 1.0.0** with consolidated release notes. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..bd34310 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,142 @@ +# Contributing to PCAP Hunter + +Thanks for your interest in contributing. This is a concise guide to how the +repo actually works day-to-day — see `CLAUDE.md` for the full architecture +and conventions reference. + +## Getting set up + +```bash +git clone +cd pcap-hunter +make install # delegates to scripts/install.py (cross-platform) +``` + +`make install` installs both system binaries (tshark, Zeek, etc.) and Python +dependencies, then verifies them. Useful variants: + +```bash +make install-system # system binaries only +make install-python # Python packages only +make check-deps # or `make doctor` — verify everything is present +``` + +Run the app locally with: + +```bash +make run # streamlit run app/main.py (checks deps first) +``` + +## Before you commit: `make verify` + +**`make verify` is the pre-commit gate and it must pass before every +commit.** It runs, in order: + +1. `ruff format --check .` — formatting check +2. `ruff check .` — lint +3. `PYTHONPATH=. pytest tests/ -q` — full test suite + +```bash +make verify +``` + +CI (`.github/workflows/ci.yml`) runs the same three checks on every push and +PR to `main`, plus an advisory `pip-audit` dependency-scan job and a Docker +build/test job. If `make verify` passes locally, it will pass in CI. + +You can also run the pieces individually: + +```bash +make test # PYTHONPATH=. pytest tests/ -v --cov=app --cov-report=term-missing --cov-fail-under=58 +make test-pdf # focused PDF/chart smoke tests — run after touching pdf_generator.py, chart_images.py, or the charts module +make lint # ruff check . +make format # ruff format . +``` + +Always run tests with `PYTHONPATH=.` (or via `make test`/`make verify`, +which set it for you) — the codebase uses absolute imports +(`from app.pipeline.beacon import rank_beaconing`) and needs it on the path. + +### Host-Python caveat + +macOS machines frequently have multiple coexisting Python installs (Framework +Python, Homebrew Python, pyenv, etc.). `make test`/`make verify` use whichever +interpreter `streamlit` is installed under (see the `PYTHON` detection at the +top of the `Makefile`), but if your environment has `pytest` and other +dependencies split across interpreters, `make verify` on the host can fail +in ways that don't reflect a real problem. If you hit interpreter confusion: + +- confirm `pip show pytest` and `pip show streamlit` agree on the same + interpreter, or +- fall back to the canonical, environment-independent path below. + +### Docker: the canonical build-and-verify path + +Any verification that depends on a clean install (dependency changes, +install-path changes, or anything you want to be **certain** works outside +your local environment) should go through Docker rather than the host: + +```bash +make docker-verify # builds the `test` image and runs format+lint+tests inside it +make docker-up # build + run the UI at http://localhost:8501 +make docker-down +``` + +This mirrors what CI's `docker` job does and is the same environment the app +ships in, so it's the most trustworthy signal for anything build-shaped. + +## Code conventions + +- **Style**: Ruff, line length 120, double quotes, 4-space indent (see the + `select` list and per-file ignores in `pyproject.toml` for the exact rules) +- **Imports**: absolute only (`from app.pipeline.beacon import ...`), + stdlib → third-party → local, ordering enforced by ruff's `I` rule +- **Naming**: `snake_case.py` modules, `PascalCase` classes, `snake_case` + functions, `UPPER_SNAKE_CASE` constants, leading underscore for private + helpers +- **Data modeling**: prefer `dataclass` for structured data, `Enum` for + fixed categories +- **Errors/logging**: custom exceptions inherit from `Exception`; use the + `logging` module, never `print()` +- **Type hints**: used extensively, with `from __future__ import annotations` + for forward compatibility +- **Docstrings**: Google-style (`Args`/`Returns`) on public functions + +## Tests + +- One test file per major module: `tests/test_.py` +- Test classes `Test`, test functions `test_` +- Cover both the happy path and edge cases (empty, `None`, malformed input) +- **Use production-shape test data.** If a function consumes + `list[CorrelationSignal]` dataclasses, pass real dataclass instances in + tests, not dicts with similar-looking keys — simplified test inputs have + previously let real bugs ship. See `tests/test_pdf_integration.py` for the + expected shapes. +- No shared `conftest.py` fixtures — tests are independent +- New PDF sections need a corresponding assertion in + `tests/test_pdf_integration.py::test_html_contains_every_expected_section`; + new PDF charts need a kaleido smoke test in `tests/test_chart_rendering.py` + +## Commit messages + +Conventional-commits style, lowercase description after the prefix: + +``` +feat: add single-ioc exact-match lookup endpoint +fix: escape crlf in cef output to prevent syslog log injection +docs: refresh readme and user manuals +chore: cover all runtime deps in dependency check +``` + +Common prefixes: `feat:`, `fix:`, `docs:`, `style:`, `chore:`. + +## Submitting changes + +1. Make your change, keeping it focused. +2. Run `make verify` (and `make docker-verify` if the change touches + dependencies, install paths, or anything build-shaped). +3. Commit using the conventions above. +4. Open a pull request against `main` describing what changed and why. + +CI must pass (tests + coverage floor, lint, format check) before a PR can be +merged. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..16692f7 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,67 @@ +# Security Policy + +## Supported versions + +PCAP Hunter is developed on a single rolling `main` branch — there is no +long-term-support branch. Security fixes are applied to `main` and released +as the next version; only the latest released version is supported. + +| Version | Supported | +|---------|-----------| +| latest (`main`) | yes | +| older releases | no | + +## Reporting a vulnerability + +Please do not open a public GitHub issue for security vulnerabilities. + +Instead, report privately using one of: + +- Open a private GitHub Security Advisory for this repository + (repo → **Security** tab → **Advisories** → **Report a vulnerability**), or +- Email the maintainer directly (see the repository's commit history / + GitHub profile for a current contact) with a description of the issue, + affected version/commit, and reproduction steps. + +Please include enough detail to reproduce the issue (PCAP sample or steps, +affected module/endpoint, expected vs. actual behavior). We'll acknowledge +reports and follow up with next steps as the issue is triaged. + +## Scope and hardening expectations + +A few things are worth calling out explicitly because of what this tool does: + +- **PCAP Hunter parses hostile, untrusted input by design.** PCAP files come + from real (potentially adversary-controlled) network traffic, and the + pipeline runs several native tools (Zeek, tshark, PyShark) and file-format + parsers (TLS/X.509, YARA) against that input. Treat any parser crash, + memory-safety issue, or resource-exhaustion bug in these paths as a + security-relevant finding, not just a bug. +- **The app makes outbound network calls.** OSINT enrichment (VirusTotal, + AbuseIPDB, GreyNoise, OTX, Shodan, WHOIS, reverse DNS) and LLM calls (LM + Studio, OpenAI, Anthropic) send indicators and/or sanitized context to + third-party or self-hosted endpoints. If you operate in an environment + where that egress is sensitive, review `app/security/opsec.py` and the + Config tab before enabling providers, and consider network-level egress + controls. +- **The Streamlit UI has no built-in authentication.** It is designed to run + as a local, single-analyst tool. Do not expose it directly to an untrusted + network or the public internet. If remote/shared access is required, put + it behind a reverse proxy that terminates TLS and enforces authentication + (e.g., an OAuth2 proxy, SSO gateway, or VPN-only access) rather than + relying on Streamlit itself for access control. The same applies to the + optional integrations API (`app/api/`) — run it behind a proxy/firewall + and use its API-key auth; do not expose it unauthenticated to the internet. +- **Config secrets are encrypted at rest but machine-bound.** API keys saved + via the Config tab are encrypted with a PBKDF2 key derived from local + machine identifiers (see `ConfigManager` in `CLAUDE.md`). Don't commit + `~/.pcap_hunter_config.json` or `.env` files, and don't share them across + machines expecting the encryption to travel with them. + +## Dependency scanning + +Dependencies are scanned on every push/PR via an advisory `pip-audit` CI job +(see `.github/workflows/ci.yml`) and kept up to date via Dependabot +(`.github/dependabot.yml`, weekly for both `pip` and GitHub Actions). Findings +there don't block merges automatically but are reviewed as part of normal +maintenance. diff --git a/docs/FEATURE-ROADMAP.md b/docs/FEATURE-ROADMAP.md index ff41de9..cd39fd2 100644 --- a/docs/FEATURE-ROADMAP.md +++ b/docs/FEATURE-ROADMAP.md @@ -1,498 +1,72 @@ # PCAP Hunter Feature Roadmap -## Overview - -This document outlines the planned features for PCAP Hunter, prioritized by implementation complexity and user value. - ---- - -## Phase 1: Quick Wins (Low Effort, High Value) - -### 1.1 CSV/JSON Export - -**Goal**: Allow users to export analysis results for external processing or archival. - -**Scope**: -- Export flow data, Zeek logs, OSINT results, and beaconing scores -- Support CSV (for spreadsheets) and JSON (for programmatic use) -- Include filtered views (respect current dashboard filters) - -**Implementation**: - -| Component | File | Changes | -|-----------|------|---------| -| Export utilities | `app/utils/export.py` | New module | -| UI buttons | `app/ui/results_tab.py` | Add export buttons per section | -| Streamlit download | `app/main.py` | Wire `st.download_button` | - -**Technical Details**: -```python -# app/utils/export.py -def export_to_csv(data: list[dict], filename: str) -> bytes: - """Convert list of dicts to CSV bytes.""" - -def export_to_json(data: Any, filename: str, indent: int = 2) -> bytes: - """Convert data to formatted JSON bytes.""" -``` - -**Dependencies**: None (uses standard library) - ---- - -### 1.2 Configuration Persistence - -**Goal**: Save user settings (API keys, thresholds, preferences) across sessions. - -**Scope**: -- Persist: LLM endpoint, API keys, analysis toggles, threshold values -- Storage: Local JSON file (`.pcap_hunter_config.json`) -- Security: Encrypt sensitive values (API keys) - -**Implementation**: - -| Component | File | Changes | -|-----------|------|---------| -| Config manager | `app/utils/config_manager.py` | New module | -| Encryption | `app/security/crypto.py` | New module (Fernet) | -| UI integration | `app/ui/sidebar.py` | Load/save buttons | - -**Technical Details**: -```python -# app/utils/config_manager.py -class ConfigManager: - def load(self) -> dict: ... - def save(self, config: dict) -> None: ... - def get(self, key: str, default: Any = None) -> Any: ... -``` - -**Dependencies**: `cryptography` (for Fernet encryption) - ---- - -### 1.3 OSINT Response Caching - -**Goal**: Cache OSINT API responses to reduce API calls and improve response time. - -**Scope**: -- Cache by IP/domain with configurable TTL (default: 24 hours) -- Storage: SQLite database (`data/osint_cache.db`) -- Manual cache invalidation UI - -**Implementation**: - -| Component | File | Changes | -|-----------|------|---------| -| Cache layer | `app/pipeline/osint_cache.py` | New module | -| OSINT integration | `app/pipeline/osint.py` | Check cache before API call | -| UI controls | `app/ui/sidebar.py` | Cache stats & clear button | - -**Technical Details**: -```python -# app/pipeline/osint_cache.py -class OSINTCache: - def __init__(self, db_path: str, ttl_hours: int = 24): ... - def get(self, indicator: str, provider: str) -> dict | None: ... - def set(self, indicator: str, provider: str, data: dict) -> None: ... - def invalidate(self, indicator: str = None) -> int: ... -``` - -**Dependencies**: `sqlite3` (standard library) - ---- - -### 1.4 JA3/JA3S Fingerprint Lookup - -**Goal**: Identify TLS client/server implementations using JA3 fingerprints. - -**Scope**: -- Calculate JA3/JA3S from TLS handshake fields -- Lookup against known fingerprint databases -- Display in SSL/TLS analysis section - -**Implementation**: - -| Component | File | Changes | -|-----------|------|---------| -| JA3 calculator | `app/pipeline/ja3.py` | New module | -| Fingerprint DB | `app/data/ja3_fingerprints.json` | Static lookup table | -| Zeek integration | `app/pipeline/zeek.py` | Parse `ssl.log` JA3 fields | -| UI display | `app/ui/results_tab.py` | Add JA3 column to SSL table | - -**Technical Details**: -```python -# app/pipeline/ja3.py -def calculate_ja3(version: str, ciphers: list, extensions: list, - curves: list, point_formats: list) -> str: - """Calculate JA3 fingerprint hash.""" - -def lookup_ja3(ja3_hash: str) -> dict | None: - """Lookup JA3 in known fingerprint database.""" -``` - -**Dependencies**: None (Zeek already extracts JA3) - ---- - -## Phase 2: Medium Effort Features - -### 2.1 DNS Query/Response Carving - -**Goal**: Extract and analyze DNS queries and responses for threat detection. - -**Scope**: -- Parse DNS packets for query names, types, and responses -- Detect suspicious patterns (DGA, DNS tunneling, fast flux) -- Visualize DNS activity timeline - -**Implementation**: - -| Component | File | Changes | -|-----------|------|---------| -| DNS carver | `app/pipeline/dns_carve.py` | New module | -| DGA detection | `app/pipeline/dns_analysis.py` | Entropy/pattern analysis | -| Zeek DNS parsing | `app/pipeline/zeek.py` | Enhanced `dns.log` parsing | -| UI visualization | `app/ui/dns_tab.py` | New tab | - -**Technical Details**: -```python -# app/pipeline/dns_analysis.py -def detect_dga(domain: str) -> float: - """Return DGA probability score (0-1).""" - -def detect_tunneling(dns_records: list[dict]) -> dict: - """Analyze for DNS tunneling indicators.""" - -def detect_fast_flux(domain: str, responses: list[dict]) -> bool: - """Check for fast-flux DNS behavior.""" -``` - -**Dependencies**: None (uses existing Zeek/PyShark) - ---- - -### 2.2 Multi-PCAP Batch Analysis - -**Goal**: Analyze multiple PCAP files together with correlation across files. - -**Scope**: -- Upload multiple PCAPs -- Correlate IPs/domains across files -- Timeline aggregation -- Merged reporting - -**Implementation**: - -| Component | File | Changes | -|-----------|------|---------| -| Batch processor | `app/pipeline/batch.py` | New module | -| Session merger | `app/pipeline/merge.py` | New module | -| UI multi-upload | `app/ui/upload_tab.py` | Multi-file upload | -| Progress tracking | `app/pipeline/state.py` | Multi-file progress | - -**Technical Details**: -```python -# app/pipeline/batch.py -class BatchProcessor: - def __init__(self, pcap_paths: list[str]): ... - def process_all(self, phase: PhaseHandle) -> dict: ... - def correlate(self) -> dict: ... -``` - -**Dependencies**: None - ---- - -### 2.3 SSL/TLS Certificate Extraction - -**Goal**: Extract and display certificate details from TLS handshakes. - -**Scope**: -- Extract X.509 certificates from PCAP -- Parse certificate fields (subject, issuer, validity, SANs) -- Certificate chain validation -- Export certificates as PEM - -**Implementation**: - -| Component | File | Changes | -|-----------|------|---------| -| Cert extractor | `app/pipeline/tls_certs.py` | New module | -| PyShark integration | `app/pipeline/pyshark_pass.py` | Add TLS parsing | -| UI display | `app/ui/tls_tab.py` | New tab or section | - -**Technical Details**: -```python -# app/pipeline/tls_certs.py -@dataclass -class Certificate: - subject: dict - issuer: dict - not_before: datetime - not_after: datetime - serial: str - sans: list[str] - fingerprint_sha256: str - -def extract_certificates(pcap_path: str) -> list[Certificate]: ... -def validate_chain(certs: list[Certificate]) -> dict: ... -``` - -**Dependencies**: `cryptography` (for X.509 parsing) - ---- - -## Phase 3: High Effort Features - -### 3.1 YARA Rule Scanning - -**Goal**: Scan carved files against YARA rules for malware detection. - -**Scope**: -- Integrate YARA engine -- Include default rule sets (malware signatures) -- Support custom rule upload -- Display matches with context - -**Implementation**: - -| Component | File | Changes | -|-----------|------|---------| -| YARA scanner | `app/pipeline/yara_scan.py` | New module | -| Rule manager | `app/utils/yara_rules.py` | Load/manage rules | -| Default rules | `app/data/yara/` | Curated rule sets | -| UI integration | `app/ui/carve_tab.py` | Scan results display | - -**Technical Details**: -```python -# app/pipeline/yara_scan.py -class YARAScanner: - def __init__(self, rules_dir: str = None): ... - def add_rules(self, rules_path: str) -> None: ... - def scan_file(self, file_path: str) -> list[YARAMatch]: ... - def scan_directory(self, dir_path: str, phase: PhaseHandle) -> dict: ... - -@dataclass -class YARAMatch: - rule: str - tags: list[str] - strings: list[tuple[int, str, bytes]] - meta: dict -``` - -**Dependencies**: `yara-python` +## Status: Phases 1-3 delivered (v1.0.0, historical) + +This document originally laid out Phases 1-3 as *planned* work on the way to +a "v1.0.0 Enterprise Ready" milestone. All of it has since shipped: CSV/JSON +export, encrypted config persistence, OSINT response caching, JA3/JA3S +fingerprinting, DNS/DGA/tunneling analysis, TLS certificate extraction, +multi-PCAP batch analysis, YARA scanning, PDF report generation, and case +management are all in production (see `CHANGELOG.md` for the full history +through `1.0.0`). The section below is kept for historical context only — +it is **not** a live plan, and the module paths it originally proposed +(`app/ui/results_tab.py`, `app/db/cases.py`, `app/models/case.py`, etc.) were +early sketches; the actual implementation landed under different paths +(`app/utils/export.py`, `app/database/`, `app/pipeline/`, `app/ui/layout.py`, +`app/reports/pdf_generator.py`, and friends — see `CLAUDE.md` for the current +architecture map). --- -### 3.2 PDF Report Generation - -**Goal**: Generate professional PDF reports for documentation and sharing. - -**Scope**: -- Executive summary -- Detailed findings with visualizations -- IOC list -- Timeline of events -- Customizable branding - -**Implementation**: - -| Component | File | Changes | -|-----------|------|---------| -| Report generator | `app/reports/pdf_report.py` | New module | -| Templates | `app/reports/templates/` | Jinja2 templates | -| Chart export | `app/utils/chart_export.py` | Export Plotly as images | -| UI button | `app/ui/report_tab.py` | Generate PDF button | - -**Technical Details**: -```python -# app/reports/pdf_report.py -class PDFReportGenerator: - def __init__(self, analysis_data: dict, template: str = "default"): ... - def generate(self, output_path: str) -> None: ... - - def _render_executive_summary(self) -> str: ... - def _render_flow_analysis(self) -> str: ... - def _render_osint_findings(self) -> str: ... - def _render_timeline(self) -> str: ... - def _render_ioc_table(self) -> str: ... -``` - -**Dependencies**: `weasyprint` or `reportlab`, `jinja2` +## Post-1.0 — recently landed + +Work completed on top of the 1.0.0 baseline: + +- **Correctness fixes** — STIX export hash-type/IPv6/JA3 handling routed + through one shared helper; JA3 attribution de-duplicated via the + authoritative `lookup_ja3`; internal/private domains filtered out of OSINT + enrichment before egress; SQLite `cases.db` runs with WAL + busy-timeout to + avoid lock errors; attack-timeline persistence and beacon-penalty tuning + fixes for highly-regular HTTPS flows. +- **MITRE ATT&CK wired end-to-end** — the mapping engine now runs in the + pipeline runner and persists to `analyses.attack_json`; it's threaded + through the UI and API caller paths, rendered on the dashboard, included in + the PDF report, and reflected in the IOC feed's `mitre_techniques` field + (including for UI-saved analyses, so feed output stays consistent + regardless of how an analysis was produced). +- **HTTP analysis stage** — a new pipeline stage (`app/pipeline/http_analysis.py`) + adds user-agent/credential/URI heuristics, feeding cleartext-credential and + suspicious-UA signals into the correlation engine and the Raw Data tab, with + HTTPS beaconing detection tuned to reduce false positives on regular, + high-confidence flows. +- **Integrations API additions (tier-1)** — case management endpoints (list, + get, patch, delete, notes) under `/api/v1/cases`, a single-IOC exact-match + lookup endpoint, a CEF-formatted IOC feed (with CRLF escaping to prevent + syslog log injection), and an SSRF-safe job-completion webhook dispatched + from the queue worker. +- **CI hardening** — a coverage floor (`--cov-fail-under=58`) enforced in the + test job, an advisory `pip-audit` job plus Dependabot config for pip and + GitHub Actions updates, and a Docker build-and-test job that builds the + `test`-stage image (with real Zeek) and runs the in-image suite as a + second, non-required signal alongside the host-based `test` job. +- **Demo capture** — a synthetic `pcaps/demo.pcap` (DNS/HTTP/beacon traffic) + plus a "load demo capture" button for a first-run experience without + needing a real PCAP on hand. --- -### 3.3 Case Management - -**Goal**: Organize analyses into cases with notes, tags, and history. - -**Scope**: -- Create/manage cases -- Link multiple PCAPs to a case -- Add analyst notes and tags -- Search across cases -- Export case archive - -**Implementation**: - -| Component | File | Changes | -|-----------|------|---------| -| Case model | `app/models/case.py` | New module | -| Database | `app/db/cases.py` | SQLite persistence | -| Case API | `app/api/cases.py` | CRUD operations | -| UI | `app/ui/cases_tab.py` | Full case management UI | - -**Technical Details**: -```python -# app/models/case.py -@dataclass -class Case: - id: str - title: str - description: str - created_at: datetime - updated_at: datetime - tags: list[str] - pcaps: list[str] - notes: list[Note] - iocs: list[IOC] - status: CaseStatus - -# app/db/cases.py -class CaseDB: - def create(self, case: Case) -> str: ... - def get(self, case_id: str) -> Case | None: ... - def update(self, case: Case) -> None: ... - def delete(self, case_id: str) -> None: ... - def search(self, query: str, tags: list[str] = None) -> list[Case]: ... -``` - -**Dependencies**: `sqlite3`, possibly `sqlalchemy` - ---- - -## Implementation Priority Matrix - -| Feature | Effort | Value | Priority | -|---------|--------|-------|----------| -| CSV/JSON Export | Low | High | P0 | -| Config Persistence | Low | High | P0 | -| OSINT Caching | Low | Medium | P1 | -| JA3 Lookup | Low | Medium | P1 | -| DNS Carving | Medium | High | P1 | -| Multi-PCAP | Medium | High | P2 | -| TLS Cert Extraction | Medium | Medium | P2 | -| YARA Scanning | High | High | P2 | -| PDF Reports | High | Medium | P3 | -| Case Management | High | High | P3 | - ---- - -## Dependencies Summary - -**New packages required**: -``` -cryptography>=41.0.0 # Config encryption, TLS cert parsing -yara-python>=4.3.0 # YARA scanning (optional) -weasyprint>=60.0 # PDF generation (optional) -jinja2>=3.1.0 # Report templates (optional) -``` - ---- - -## Architecture Considerations - -### Module Organization - -``` -app/ -├── pipeline/ -│ ├── dns_carve.py # New: DNS carving -│ ├── dns_analysis.py # New: DGA/tunneling detection -│ ├── ja3.py # New: JA3 fingerprinting -│ ├── osint_cache.py # New: OSINT caching -│ ├── tls_certs.py # New: Certificate extraction -│ ├── yara_scan.py # New: YARA scanning -│ └── batch.py # New: Multi-PCAP processing -├── utils/ -│ ├── export.py # New: CSV/JSON export -│ ├── config_manager.py # New: Config persistence -│ └── yara_rules.py # New: YARA rule management -├── reports/ -│ ├── pdf_report.py # New: PDF generation -│ └── templates/ # New: Report templates -├── models/ -│ └── case.py # New: Case data model -├── db/ -│ └── cases.py # New: Case database -└── security/ - └── crypto.py # New: Encryption utilities -``` - -### Data Flow - -``` - ┌─────────────────┐ - │ PCAP Upload │ - └────────┬────────┘ - │ - ┌────────────────┼────────────────┐ - ▼ ▼ ▼ - ┌───────────┐ ┌───────────┐ ┌───────────┐ - │ Zeek │ │ PyShark │ │ Tshark │ - └─────┬─────┘ └─────┬─────┘ └─────┬─────┘ - │ │ │ - │ ┌───────────┴───────────┐ │ - │ ▼ ▼ │ - │ ┌───────┐ ┌───────┐ │ - │ │ JA3 │ │ DNS │ │ - │ └───────┘ │ Carve │ │ - │ └───────┘ │ - │ │ - └──────────────┬──────────────────┘ - ▼ - ┌─────────────────────┐ - │ OSINT Enrichment │◄──── Cache - └──────────┬──────────┘ - │ - ┌────────────┼────────────┐ - ▼ ▼ ▼ - ┌──────────┐ ┌──────────┐ ┌──────────┐ - │ Beaconing│ │ YARA │ │ TLS │ - │ Detection│ │ Scanning│ │ Certs │ - └──────────┘ └──────────┘ └──────────┘ - │ - ▼ - ┌─────────────────────┐ - │ LLM Analysis │ - └──────────┬──────────┘ - │ - ┌────────────┼────────────┐ - ▼ ▼ ▼ - ┌──────────┐ ┌──────────┐ ┌──────────┐ - │ Export │ │ PDF │ │ Case │ - │ CSV/JSON │ │ Report │ │ Mgmt │ - └──────────┘ └──────────┘ └──────────┘ -``` - ---- - -## Version Milestones - -### v0.3.0 - Export & Persistence -- CSV/JSON Export -- Configuration Persistence -- OSINT Caching - -### v0.4.0 - Enhanced Analysis -- JA3/JA3S Fingerprinting -- DNS Carving & Analysis -- SSL/TLS Certificate Extraction - -### v0.5.0 - Advanced Features -- Multi-PCAP Batch Analysis -- YARA Rule Scanning - -### v1.0.0 - Enterprise Ready -- PDF Report Generation -- Case Management System -- Full documentation +## Future ideas (not yet started) + +Genuine candidates for future work — none of these are in progress: + +- Lateral-movement / SMB traffic detection +- JA4 / JARM fingerprinting (JA3/JA3S only today) +- IPv6-aware OSINT enrichment (current provider integrations are IPv4-centric) +- MISP and Sigma export formats (STIX 2.0/2.1, ATT&CK Navigator, and CSV/JSON + are already supported) +- Per-case configuration profiles (today's config is global, one machine-wide + `ConfigManager` instance) +- Multi-user support with role-based access control and an audit trail + (the app currently assumes a single local analyst) +- Configurable case/analysis retention policy (currently manual "clear data" + actions only; no automatic expiry) From e6c827fdf1d55a36343b3fa67fff7d76f386de29 Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Tue, 14 Jul 2026 05:02:22 +0700 Subject: [PATCH 34/35] fix: address final-review cross-task findings (batch intel, timeline restore, beacon fp, host-port) --- app/analysis/correlation.py | 30 ++++++++++++++- app/main.py | 66 +++++++++++++++++++++++++++---- app/pipeline/batch.py | 77 +++++++++++++++++++++++++++++++++++++ app/pipeline/beacon.py | 18 ++++++++- tests/test_batch.py | 75 ++++++++++++++++++++++++++++++++++++ tests/test_beacon.py | 38 ++++++++++++++++++ tests/test_correlation.py | 33 ++++++++++++++++ 7 files changed, 327 insertions(+), 10 deletions(-) diff --git a/app/analysis/correlation.py b/app/analysis/correlation.py index 7221c77..4f4e6e3 100644 --- a/app/analysis/correlation.py +++ b/app/analysis/correlation.py @@ -80,12 +80,38 @@ def _get_verdict(score: float) -> str: return "low" +def _normalize_http_host(host: str) -> str: + """Strip a trailing ``:port`` suffix from an HTTP Host-header value. + + Zeek's ``http.log`` Host field may include an explicit port (e.g. + ``example.com:8443`` or ``203.0.113.5:8443``), but the IP/domain + indicators this lookup is later matched against are always bare. A naive + ``host.rsplit(":", 1)[0]`` would mangle a bracketed IPv6 literal like + ``[::1]`` (splitting on one of the address's own colons), so bracketed + hosts are only trimmed after the closing bracket and otherwise left + untouched. + """ + if not host: + return host + if host.startswith("["): + close = host.find("]") + return host[: close + 1] if close != -1 else host + if ":" in host: + head, _, tail = host.rpartition(":") + if tail.isdigit(): + return head + return host + + def _build_http_lookup(http_analysis: dict | None) -> dict[str, list[tuple[str, Any, float]]]: """Build a Host-header -> [(signal_name, value, score), ...] lookup from HTTP analysis. The Zeek ``http.log`` Host header may be a raw IP or a domain, so this lookup is matched against both IP and domain indicators by the callers - (``_collect_ip_signals`` / ``_collect_domain_signals``). + (``_collect_ip_signals`` / ``_collect_domain_signals``). Host values are + normalised (trailing ``:port`` stripped) before being used as keys so a + Host like ``example.com:8443`` still attaches its signal to the + ``example.com`` indicator. """ lookup: dict[str, list[tuple[str, Any, float]]] = {} if not http_analysis: @@ -94,11 +120,13 @@ def _build_http_lookup(http_analysis: dict | None) -> dict[str, list[tuple[str, for cred in http_analysis.get("cleartext_credentials", []) or []: host = cred.get("host") if isinstance(cred, dict) else None if host: + host = _normalize_http_host(host) lookup.setdefault(host, []).append(("http_cleartext_cred", cred.get("username", ""), 0.9)) for ua in http_analysis.get("suspicious_user_agents", []) or []: host = ua.get("host") if isinstance(ua, dict) else None if host: + host = _normalize_http_host(host) lookup.setdefault(host, []).append(("http_suspicious_ua", ua.get("reason", ""), 0.6)) return lookup diff --git a/app/main.py b/app/main.py index dd7f291..7e87bcc 100644 --- a/app/main.py +++ b/app/main.py @@ -640,13 +640,16 @@ def _run_single_pcap_pipeline( st.session_state["beacon_df"] = batch_result.merged_beacons st.session_state["dns_analysis"] = batch_result.aggregated_dns st.session_state["tls_analysis"] = batch_result.aggregated_tls - # No cross-file ATT&CK aggregation yet (batch.py has no merge helper for - # it) — mirror the first successful file's mapping, same fallback used - # for merged_features above. - st.session_state["attack_mapping"] = (first_ok.attack_mapping if first_ok else None) or {} - # No cross-file HTTP aggregation helper either (see attack_mapping - # above) — mirror the first successful file's HTTP findings. - st.session_state["http_analysis"] = (first_ok.http_analysis if first_ok else None) or {} + # HTTP findings merged across every file (app.pipeline.batch.aggregate_http_analysis) + # — previously only mirrored the first successful file's findings. + st.session_state["http_analysis"] = batch_result.aggregated_http + # Rebuilt further below, once merged features/dns/tls/beacon_df are + # all available, from the AGGREGATED batch inputs (mirrors the + # per-file ATTACKMapper call in app/pipeline/runner.py). Previously + # this mirrored only the first successful file's mapping, which + # under-reported ATT&CK techniques found in files 2..N. Placeholder + # here in case that rebuild raises. + st.session_state["attack_mapping"] = {} # Carved payloads concatenated across all successful files st.session_state["carved"] = [ item for r in batch_result.pcap_results if not r.error for item in r.carved_items @@ -716,6 +719,27 @@ def _run_single_pcap_pipeline( logger.debug("timeline precompute failed: %s", e) st.session_state["attack_timeline"] = [] + # Rebuild the ATT&CK mapping from the AGGREGATED batch inputs (same + # merged features/dns/tls/beacon_df used for the timeline above), + # mirroring the per-file ATTACKMapper call in app/pipeline/runner.py. + # Replaces the earlier first-successful-file-only placeholder so the + # dashboard ATT&CK panel and batch-quick-saved mitre_techniques + # reflect techniques detected across every file, not just file 1. + try: + from app.threat_intel import ATTACKMapper + + _mapping = ATTACKMapper().map_analysis( + features=st.session_state.get("features"), + dns_analysis=st.session_state.get("dns_analysis"), + tls_analysis=st.session_state.get("tls_analysis"), + beacon_results=( + get_df_state("beacon_df").to_dict("records") if not get_df_state("beacon_df").empty else [] + ), + ) + st.session_state["attack_mapping"] = _mapping.to_dict() + except Exception as e: + logger.warning("Batch ATT&CK mapping rebuild failed: %s", e) + batch_tracker.finish_all( f"Batch complete: {batch_result.summary['successful']}/{batch_result.summary['total_files']} files." ) @@ -1321,7 +1345,35 @@ def _run_single_pcap_pipeline( # Attack timeline (full-width, if available) — stored at pipeline-completion # time (see post-analysis blocks above) so it survives without needing to # recompute here, and so the PDF report can reuse the same data. + # + # A restored case resets attack_timeline to [] (it isn't a persisted Analysis + # column — see app/ui/cases_tab.py::_restore_analysis_to_session) even though + # features/dns/tls/beacon data ARE restored. Lazily recompute from the + # CURRENT restored state in that case so restored cases don't lose the + # timeline chart; a fresh run's own post-analysis block always populates + # attack_timeline directly, so this only fires for the restore path. timeline_dicts = st.session_state.get("attack_timeline") or [] + if not timeline_dicts and st.session_state.get("features"): + try: + from app.analysis.narrator import AttackNarrator + + _tl = AttackNarrator().create_timeline( + features=st.session_state.get("features"), + dns_analysis=st.session_state.get("dns_analysis"), + yara_results=st.session_state.get("yara_results"), + beacon_results=( + get_df_state("beacon_df").to_dict("records") if not get_df_state("beacon_df").empty else [] + ), + tls_analysis=st.session_state.get("tls_analysis"), + ) + timeline_dicts = [e.to_dict() for e in _tl] + # Cache back so we don't re-run the narrator on every dashboard + # rerun (filter toggles, etc.) for a restored case. Safe against the + # cross-run stale-leak: a fresh run's reset clears this key, and this + # value was just derived from the CURRENT restored features. + st.session_state["attack_timeline"] = timeline_dicts + except Exception as e: + logger.debug("timeline lazy recompute failed: %s", e) if timeline_dicts: st.plotly_chart( plot_attack_timeline(timeline_dicts), diff --git a/app/pipeline/batch.py b/app/pipeline/batch.py index 6d8cf60..ba5fffb 100644 --- a/app/pipeline/batch.py +++ b/app/pipeline/batch.py @@ -17,6 +17,8 @@ import pandas as pd +from app.pipeline.http_analysis import MAX_HTTP_RESULTS + logger = logging.getLogger(__name__) # --- Constants --- @@ -469,6 +471,78 @@ def aggregate_tls_analysis(results: list[PCAPResult]) -> dict[str, Any]: } +def aggregate_http_analysis(results: list[PCAPResult]) -> dict[str, Any]: + """ + Aggregate HTTP request analysis from multiple PCAPs. + + Mirrors the shape app.pipeline.http_analysis.analyze_http returns for a + single file, so downstream consumers (correlation, ATT&CK mapping, the + dashboard HTTP panel) don't need batch-specific handling. + + Args: + results: List of PCAPResult + + Returns: + Aggregated HTTP analysis dictionary. + """ + total_requests = 0 + # Summed per-file, like aggregate_dns_analysis's total_records — an + # approximation when the same host appears in multiple files, since only + # per-file counts (not raw host sets) are persisted on PCAPResult. + unique_hosts = 0 + methods: Counter = Counter() + status_codes: Counter = Counter() + all_uas = [] + all_creds = [] + all_uris = [] + + for r in results: + if r.error or not r.http_analysis: + continue + + http = r.http_analysis + if http.get("skipped") or http.get("error"): + continue + + total_requests += http.get("total_requests", 0) + unique_hosts += http.get("unique_hosts", 0) + + for method, count in (http.get("methods") or {}).items(): + methods[method] += count + for code, count in (http.get("status_codes") or {}).items(): + status_codes[code] += count + + for ua in http.get("suspicious_user_agents", []) or []: + ua_copy = dict(ua) + ua_copy["_source_file"] = r.filename + all_uas.append(ua_copy) + + for cred in http.get("cleartext_credentials", []) or []: + cred_copy = dict(cred) + cred_copy["_source_file"] = r.filename + all_creds.append(cred_copy) + + for uri in http.get("suspicious_uris", []) or []: + uri_copy = dict(uri) + uri_copy["_source_file"] = r.filename + all_uris.append(uri_copy) + + return { + "total_requests": total_requests, + "unique_hosts": unique_hosts, + "methods": dict(methods), + "status_codes": dict(status_codes), + "suspicious_user_agents": all_uas[:MAX_HTTP_RESULTS], + "cleartext_credentials": all_creds[:MAX_HTTP_RESULTS], + "suspicious_uris": all_uris[:MAX_HTTP_RESULTS], + "alerts": { + "suspicious_ua_count": len(all_uas), + "cleartext_cred_count": len(all_creds), + "suspicious_uri_count": len(all_uris), + }, + } + + @dataclass class BatchResult: """Complete batch processing result.""" @@ -480,6 +554,7 @@ class BatchResult: merged_beacons: pd.DataFrame aggregated_dns: dict[str, Any] aggregated_tls: dict[str, Any] + aggregated_http: dict[str, Any] summary: dict[str, Any] @@ -565,6 +640,7 @@ def merge_all(self) -> BatchResult: merged_beacons = merge_beacon_candidates(self.results) aggregated_dns = aggregate_dns_analysis(self.results) aggregated_tls = aggregate_tls_analysis(self.results) + aggregated_http = aggregate_http_analysis(self.results) # Build summary successful = sum(1 for r in self.results if not r.error) @@ -608,6 +684,7 @@ def merge_all(self) -> BatchResult: merged_beacons=merged_beacons, aggregated_dns=aggregated_dns, aggregated_tls=aggregated_tls, + aggregated_http=aggregated_http, summary=summary, ) diff --git a/app/pipeline/beacon.py b/app/pipeline/beacon.py index c187175..b243cf3 100644 --- a/app/pipeline/beacon.py +++ b/app/pipeline/beacon.py @@ -75,16 +75,30 @@ # That combination is exactly what genuine small-packet HTTPS C2 looks like. # Tuned against tests/test_beacon.py so a perfectly periodic small-payload # 443 flow (raw score ~1.0) clears BEACON_SCORE_THRESHOLD (0.6) with margin: -# 1.0 * 0.7 = 0.7 > 0.6. The naive "penalty * 4" (0.15 -> 0.6) is NOT enough: +# 1.0 * 0.69 = 0.69 > 0.6. The naive "penalty * 4" (0.15 -> 0.6) is NOT enough: # a 0.9-raw flow would land at 0.9 * 0.6 = 0.54, still below threshold. # +# 0.69, not 0.7: the ATT&CK mapper's beacon->C2 rule (DETECTION_RULES +# ["beacon_score"]["threshold"] in app/threat_intel/attack_mapping.py) fires +# at score >= 0.7 and auto-sets overall_severity="high". Since the max +# softened output is final_score(<=1.0) * SOFTENED_PENALTY, 0.7 would let a +# perfectly periodic, zero-jitter softened beacon land EXACTLY on the ATT&CK +# threshold — a benign small-payload HTTPS flow with flawless periodicity +# would then get flagged as T1071.001 C2 at HIGH severity purely from +# timing, which is exactly the false positive this softening bucket exists +# to avoid. 0.69 keeps the max softened score strictly below 0.7 (decoupled +# from the ATT&CK rule) while staying comfortably above the 0.6 beacon +# candidate floor and the 0.5 correlation ingest gate, so genuine HTTPS +# beacons still surface as candidates — they just don't trip the ATT&CK +# C2 technique on timing alone. +# # The small-payload condition (3) is load-bearing: a machine-regular CDN # heartbeat (zero jitter, LARGE 1200-byte payloads) is indistinguishable # from C2 by timing+jitter alone, so requiring small payloads keeps those # fully penalised. When pkt_lens is absent/empty we CANNOT confirm the flow # is C2-like, so we conservatively do NOT soften. Ordinary jittery HTTPS # keep-alives also fail the >=0.85 gate and keep the full 0.15 penalty. -SOFTENED_PENALTY = 0.7 +SOFTENED_PENALTY = 0.69 def periodicity_score(ts: list[float]) -> dict[str, object]: diff --git a/tests/test_batch.py b/tests/test_batch.py index f27d960..0267108 100644 --- a/tests/test_batch.py +++ b/tests/test_batch.py @@ -6,6 +6,7 @@ BatchProcessor, PCAPResult, aggregate_dns_analysis, + aggregate_http_analysis, aggregate_tls_analysis, correlate_results, merge_beacon_candidates, @@ -340,6 +341,80 @@ def test_deduplicate_by_fingerprint(self): assert aggregated["total_certificates"] == 2 # Deduplicated +class TestAggregateHTTP: + """Test aggregating HTTP request analysis results (FIX 1: batch mode must + not under-report intel from files 2..N).""" + + def test_empty_results(self): + aggregated = aggregate_http_analysis([]) + assert aggregated["total_requests"] == 0 + assert aggregated["cleartext_credentials"] == [] + + def test_skips_errored_and_skipped_files(self): + results = [ + PCAPResult(path="/data/1.pcap", filename="1.pcap", features={}, error="boom"), + PCAPResult(path="/data/2.pcap", filename="2.pcap", features={}, http_analysis={"skipped": True}), + PCAPResult( + path="/data/3.pcap", + filename="3.pcap", + features={}, + http_analysis={"error": "No HTTP log data", "records": 0}, + ), + ] + aggregated = aggregate_http_analysis(results) + assert aggregated["total_requests"] == 0 + + def test_aggregate_merges_findings_from_every_file(self): + # Regression for the bug where st.session_state["http_analysis"] only + # reflected the FIRST successful file in batch mode — this asserts the + # aggregate carries findings from file 2 as well. + results = [ + PCAPResult( + path="/data/1.pcap", + filename="1.pcap", + features={}, + http_analysis={ + "total_requests": 10, + "unique_hosts": 3, + "methods": {"GET": 8, "POST": 2}, + "status_codes": {"200": 10}, + "suspicious_user_agents": [], + "cleartext_credentials": [{"host": "1.2.3.4", "uri": "/login", "username": "admin"}], + "suspicious_uris": [], + "alerts": {"suspicious_ua_count": 0, "cleartext_cred_count": 1, "suspicious_uri_count": 0}, + }, + ), + PCAPResult( + path="/data/2.pcap", + filename="2.pcap", + features={}, + http_analysis={ + "total_requests": 5, + "unique_hosts": 2, + "methods": {"GET": 5}, + "status_codes": {"200": 4, "404": 1}, + "suspicious_user_agents": [ + {"host": "evil.example", "user_agent": "sqlmap/1.6", "uri": "/", "reason": "known tool"} + ], + "cleartext_credentials": [], + "suspicious_uris": [], + "alerts": {"suspicious_ua_count": 1, "cleartext_cred_count": 0, "suspicious_uri_count": 0}, + }, + ), + ] + aggregated = aggregate_http_analysis(results) + assert aggregated["total_requests"] == 15 + assert aggregated["methods"]["GET"] == 13 + assert aggregated["status_codes"]["200"] == 14 + # File 1's cred finding AND file 2's UA finding must both be present. + assert len(aggregated["cleartext_credentials"]) == 1 + assert aggregated["cleartext_credentials"][0]["host"] == "1.2.3.4" + assert len(aggregated["suspicious_user_agents"]) == 1 + assert aggregated["suspicious_user_agents"][0]["host"] == "evil.example" + assert aggregated["alerts"]["cleartext_cred_count"] == 1 + assert aggregated["alerts"]["suspicious_ua_count"] == 1 + + class TestBatchProcessor: """Test BatchProcessor class.""" diff --git a/tests/test_beacon.py b/tests/test_beacon.py index 7352f75..11dbacd 100644 --- a/tests/test_beacon.py +++ b/tests/test_beacon.py @@ -1,5 +1,6 @@ from app.config import BEACON_SCORE_THRESHOLD from app.pipeline.beacon import jitter_score, periodicity_score, rank_beaconing +from app.threat_intel.attack_mapping import DETECTION_RULES, ATTACKMapper def test_periodicity_score_empty(): @@ -170,6 +171,43 @@ def test_rank_beaconing_regular_443_no_pkt_lens_not_softened(): assert df.iloc[0]["score"] < BEACON_SCORE_THRESHOLD +def test_rank_beaconing_softened_443_score_decoupled_from_attack_mapper_threshold(): + # Locks in the FIX 3 invariant: a softened-443 beacon must clear the + # BEACON_SCORE_THRESHOLD candidate floor (so genuine HTTPS beacons still + # surface) but must NOT, on timing alone, reach the ATT&CK mapper's + # beacon->C2 rule threshold (score >= 0.7, which auto-sets HIGH + # severity with a T1071.001 false positive for a benign perfectly + # periodic small-payload flow). See app/pipeline/beacon.py's + # SOFTENED_PENALTY comment for the full rationale. + flows = [ + { + "src": "10.0.0.3", + "dst": "203.0.113.5", + "sport": "51000", + "dport": "443", + "proto": "tcp", + "pkt_times": [float(i) for i in range(1, 60)], # perfectly periodic + "pkt_lens": [80] * 59, # small, C2-like payloads + } + ] + df = rank_beaconing(flows, top_n=10) + assert len(df) == 1 + score = df.iloc[0]["score"] + + # Still a beacon candidate... + assert score > BEACON_SCORE_THRESHOLD + # ...but decoupled from the ATT&CK mapper's C2 threshold. + beacon_rule_threshold = DETECTION_RULES["beacon_score"]["threshold"] + assert score < beacon_rule_threshold + + # End-to-end: feeding this exact beacon candidate into the mapper must not + # produce the T1071.001 C2 technique (and therefore not force HIGH severity + # from timing alone). + mapping = ATTACKMapper().map_analysis(beacon_results=df.to_dict("records")) + ids = {t.technique_id for t in mapping.techniques} + assert "T1071.001" not in ids + + def test_rank_beaconing_jittery_443_stays_suppressed(): # An ordinary HTTPS keep-alive with real-world jitter must NOT be # promoted by the softening — only extremely regular, high-confidence diff --git a/tests/test_correlation.py b/tests/test_correlation.py index 2db4032..eea7cc5 100644 --- a/tests/test_correlation.py +++ b/tests/test_correlation.py @@ -243,6 +243,39 @@ def test_correlate_http_cleartext_cred_lights_up_domain_indicator(): assert any(s.name == "http_cleartext_cred" and s.source == "http" for s in results[0].signals) +def test_correlate_http_cleartext_cred_host_with_port_normalized(): + """A Host header carrying an explicit port (e.g. "example.com:8443") + must still attach its signal to the bare domain/IP indicator instead of + being silently dropped because the exact strings don't match.""" + features = { + "artifacts": {"ips": [], "domains": ["example.com"]}, + "flows": [], + } + http_analysis = { + "cleartext_credentials": [{"host": "example.com:8443", "uri": "/login", "username": "admin"}], + "suspicious_user_agents": [], + } + results = correlate_indicators(features=features, http_analysis=http_analysis) + assert len(results) == 1 + assert results[0].indicator == "example.com" + sig = next(s for s in results[0].signals if s.name == "http_cleartext_cred") + assert sig.source == "http" + + +def test_normalize_http_host_is_ipv6_safe(): + """Stripping the :port suffix must not mangle a bracketed IPv6 literal + (a naive rsplit on ':' would split one of the address's own colons).""" + from app.analysis.correlation import _normalize_http_host + + assert _normalize_http_host("example.com:8443") == "example.com" + assert _normalize_http_host("203.0.113.5:8443") == "203.0.113.5" + assert _normalize_http_host("example.com") == "example.com" + # Bracketed IPv6, with and without an explicit port — the address is preserved. + assert _normalize_http_host("[::1]") == "[::1]" + assert _normalize_http_host("[2001:db8::1]:8443") == "[2001:db8::1]" + assert _normalize_http_host("") == "" + + def test_correlate_http_suspicious_ua_signal(): features = { "artifacts": {"ips": ["5.6.7.8"], "domains": []}, From bce3ba68321ffa04939127b8df087b868327c47c Mon Sep 17 00:00:00 2001 From: Henry Hu Date: Tue, 14 Jul 2026 05:18:58 +0700 Subject: [PATCH 35/35] style: apply ruff 0.15 formatting to test_webhook.py (ci format gate) --- tests/api/test_webhook.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/api/test_webhook.py b/tests/api/test_webhook.py index eaa1f7f..6850696 100644 --- a/tests/api/test_webhook.py +++ b/tests/api/test_webhook.py @@ -212,7 +212,7 @@ def fake_dispatch(url, payload, timeout, max_retries): monkeypatch.setattr( runner_mod, "run_pipeline", - lambda pcap_path, case_id, options, progress, heartbeat=None: (_fake_pipeline_result(case_id)), + lambda pcap_path, case_id, options, progress, heartbeat=None: _fake_pipeline_result(case_id), ) fake_pcap = tmp_path / "fake.pcap" @@ -285,7 +285,7 @@ def test_worker_does_not_fire_webhook_when_not_configured(tmp_path, monkeypatch) monkeypatch.setattr( runner_mod, "run_pipeline", - lambda pcap_path, case_id, options, progress, heartbeat=None: (_fake_pipeline_result(case_id)), + lambda pcap_path, case_id, options, progress, heartbeat=None: _fake_pipeline_result(case_id), ) fake_pcap = tmp_path / "fake.pcap"