diff --git a/.gitignore b/.gitignore
index c08213e..3dc1c01 100644
--- a/.gitignore
+++ b/.gitignore
@@ -38,7 +38,9 @@ htmlcov/
# Project specific
data/
*.pcap
+*.pcapng
!tests/fixtures/*.pcap
+!tests/fixtures/*.pcapng
*.log
.DS_Store
@@ -50,4 +52,3 @@ CLAUDE.local.md
# Superpowers auto-generated specs and plans (session artifacts, not source)
docs/superpowers/
-
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f882c19..f947bad 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,43 @@
All notable changes to this project will be documented in this file.
+## [2.0.0] - 2026-07-14
+
+Version 2 turns PCAP Hunter into a more evidence-aware investigation workbench and strengthens both interactive and headless analysis paths.
+
+### Added
+
+- **Dedicated MITRE ATT&CK Analysis workspace** with versioned technique hypotheses, supporting evidence, applicable Detection Strategy context, analyst dispositions, coverage gaps, and ATT&CK Navigator export.
+- **Capture-quality telemetry** covering packet and flow scale, parse ratio, capture time window, sampling limits, pipeline warnings, completed stages, and detector availability.
+- **Persistent ATT&CK and capture context** in saved analyses and API results; IOC feeds now include related technique IDs across contributing analyses.
+- **Validated Streamlit uploads** that stream `.pcap` and `.pcapng` files in bounded chunks, enforce file and batch limits, validate magic bytes, and roll back partial batches.
+- **LLM-optional evidence snapshot** so parsed flows, artifacts, correlations, stage status, and warnings remain visible when the narrative is skipped or unavailable.
+- **Two additional real-app screenshots** for MITRE Analysis and API Key Management, with isolated capture data, IPv4/IPv6 pixel redaction, and a final OCR audit.
+
+### Changed
+
+- **Integrations API submission lifecycle** now initializes the worker queue only for accepted submissions and removes provisional uploads and cases if queueing or persistence fails.
+- **Readiness checks** no longer initialize the background job queue as a side effect.
+- **Docker LM Studio routing** adapts loopback and private host addresses to `host.docker.internal` for local-compatible providers.
+- **Case analysis updates** use an upsert that replaces stale IOC rows without deleting the parent analysis record.
+- **PDF timestamps** are emitted consistently in UTC.
+- **Navigator exports** now declare the current Navigator 5.3.2 application version while retaining layer schema 4.5.
+- **Version metadata** is unified at `2.0.0` for the package, FastAPI schema, documentation, tag, and GitHub release.
+
+### Fixed
+
+- HTTP payload carving now decodes compact and colon-separated tshark byte-array output before hashing and writing files.
+- Capture metrics tolerate flows with missing or malformed timestamps instead of failing the MITRE workspace.
+- ATT&CK mappings restored from SQLite remain normalized in session state, so analyst dispositions and notes survive Streamlit reruns.
+- The LLM evidence snapshot now reads the canonical `parsed_packet_count` capture metric.
+- Rejected API uploads no longer leave orphaned files or empty cases when the queue is full or cannot initialize.
+
+### Docs
+
+- Rewritten README visual tour covering all ten primary tabs with actual Docker UI captures.
+- Updated English and Traditional Chinese manuals for MITRE analysis, capture coverage, and Docker-hosted LM Studio routing.
+- Updated Integrations API examples for capture metrics, ATT&CK mappings, IOC technique IDs, and cleanup semantics.
+
## [1.0.0] - 2026-04-22
First stable release. Production-ready installer, hardened pipeline, polished UX, and brand identity.
diff --git a/README.md b/README.md
index f5b8ced..ed2c28c 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,7 @@
# PCAP Hunter
[](https://github.com/ninedter/pcap-hunter/actions/workflows/ci.yml)
+[](https://github.com/ninedter/pcap-hunter/releases/tag/v2.0.0)
[](https://www.python.org/downloads/)
[](LICENSE)
@@ -14,8 +15,20 @@ By combining industry-standard network analysis tools (**Zeek**, **Tshark**, **P
---
+## What's new in version 2
+
+- **Dedicated MITRE ATT&CK workspace** — evidence-backed technique hypotheses, ATT&CK v19.1 metadata, analyst dispositions, capture coverage, visibility gaps, and Navigator export.
+- **Capture-quality telemetry** — packet/flow scale, parse ratio, time window, sampling limits, completed stages, and warnings now travel with UI and API results and persist with cases.
+- **Safer PCAP intake** — Streamlit uploads are streamed in bounded chunks, preserve `.pcap`/`.pcapng`, validate magic bytes, enforce batch limits, and remove partial files after rejection.
+- **Stronger Integrations API** — headless jobs return ATT&CK mappings and capture metrics, IOC feeds carry related technique IDs, readiness checks avoid starting the worker queue, and failed submissions clean up provisional cases and files.
+- **Evidence without an LLM** — skipping or losing the optional AI narrative no longer hides deterministic packet, IOC, correlation, stage, and warning evidence.
+- **Runtime and export reliability** — Docker adapts local LM Studio addresses to the host bridge, HTTP carving decodes tshark byte arrays correctly, case re-saves replace stale IOCs, and PDF timestamps are consistently UTC.
+
+---
+
## Table of Contents
+- [What's new in version 2](#whats-new-in-version-2)
- [Visual Tour](#visual-tour)
- [Key Features](#key-features)
- [Integrations API](#integrations-api)
@@ -32,6 +45,10 @@ By combining industry-standard network analysis tools (**Zeek**, **Tshark**, **P
## Visual Tour
+These are captures of the real version-2 Streamlit app running in Docker against
+the bundled sample PCAP. IP addresses, API secrets, email addresses, and local
+user paths are masked in the image pixels before the files are committed.
+
### 1. Upload — load one or many PCAPs
Drag-and-drop `.pcap` / `.pcapng` files or paste a container path. Multiple files
@@ -59,7 +76,16 @@ timeline put the capture in visual context.
-### 4. LLM Analysis — AI-generated threat report
+### 4. MITRE Analysis — behaviors, evidence, and coverage
+
+The dedicated ATT&CK workspace treats mappings as analyst hypotheses rather than
+proof. It links network evidence to techniques and applicable detection context,
+lets analysts record a disposition and note, makes detector gaps explicit, and
+exports an ATT&CK Navigator layer with versioned metadata.
+
+
+
+### 5. LLM Analysis — AI-generated threat report
A nine-section narrative — Executive Summary through Recommended Actions, plus an
**IOC Summary table** and a **Risk Matrix rendered as a real Markdown table** — with
@@ -69,7 +95,11 @@ Reports in 9 languages, including Traditional Chinese (zh-TW).
-### 5. OSINT — multi-provider IOC enrichment
+When a report is skipped or unavailable, this tab still shows a deterministic
+snapshot of parsed packets, flows, IOCs, correlations, completed stages, and
+pipeline warnings.
+
+### 6. OSINT — multi-provider IOC enrichment
Prioritized IOC table with VirusTotal, AbuseIPDB, GreyNoise, Shodan, OTX, and
VT Domain signals merged into one view. **Provider-status pills** report each
@@ -80,7 +110,7 @@ Infrastructure ASN clustering, Export, Devices, and Notes.
-### 6. Raw Data — Zeek logs, flows, carved payloads, YARA matches
+### 7. Raw Data — Zeek logs, flows, carved payloads, YARA matches
Every underlying data source is available: the flow table (with explicit
**First/Last Seen (UTC)** timestamp columns), DNS and TLS analyses, NXDOMAIN
@@ -90,20 +120,23 @@ CSV-injection protection.
-### 7. Cases — persistent investigation tracking
+### 8. Cases — persistent investigation tracking
-Promote any capture and its findings into a case. Cases carry IOCs, severity, tags,
-investigation notes, status, and search — stored in a local SQLite database.
+Promote any capture and its findings into a case. Cases carry IOCs, severity,
+tags, investigation notes, ATT&CK mappings, capture-quality metrics, status, and
+search — stored in a local SQLite database.
-### 8. API Keys — manage programmatic access
+### 9. API Keys — manage programmatic access
Create, revoke, and monitor API keys for the Integrations API. Each key has its own
scope (full or feed-only), optional expiration, per-key rate limits, and a usage
sparkline. Environment-variable keys are shown as read-only bootstrap entries.
-### 9. Config — centralized settings
+
+
+### 10. Config — centralized settings
An **LLM Integration** section with three providers (LM Studio, OpenAI, Anthropic),
a **YARA Rules** section with a configurable rules directory, OSINT provider keys
@@ -132,8 +165,10 @@ and model picker.
- **OpenAI** (cloud) — single-shot report with the entire evidence corpus in one full-context call.
- **Anthropic** (cloud) — Claude via the official `anthropic` SDK (`claude-opus-4-8`, `claude-sonnet-4-6`, `claude-haiku-4-5`), single-shot with streaming.
- **Evidence-Grounded Reporting** — SOC-ready reports with severity-calibrated assessments, false-positive awareness, confidence qualifiers, a Risk Matrix rendered as a real Markdown table, and an IOC Summary table.
+- **LLM-Optional Evidence View** — parsed packet, flow, IOC, correlation, stage, and warning evidence remains visible when generation is skipped or the provider is unavailable.
- **Multi-Language Reports** — 9 languages with region-specific terminology: English, Traditional Chinese (Taiwan), Simplified Chinese, Japanese, Korean, Italian, Spanish, French, German.
-- **MITRE ATT&CK Mapping** — Automated mapping of detected behaviors and IOCs to ATT&CK techniques and Kill Chain phases.
+- **MITRE ATT&CK Analysis** — A separate Behaviors & Coverage workspace that maps network evidence to versioned ATT&CK hypotheses, links applicable Detection Strategies/Data Components, records analyst dispositions, and exports Navigator layers.
+- **Capture-quality telemetry** — Packet/flow scale, parse coverage, time window, sampling limits, pipeline warnings, and detector visibility gaps are recorded alongside findings.
- **Attack Narrative Synthesis** — Translates raw events into a coherent, actionable security story.
### IOC Priority Scoring
@@ -157,6 +192,7 @@ and model picker.
### Multi-PCAP Batch Processing
- **Multi-File Upload** — Upload and analyze multiple PCAP files simultaneously.
+- **Validated Streaming Intake** — `.pcap` and `.pcapng` uploads are written in bounded chunks, checked for file and batch limits, validated by magic bytes, and rolled back as a set after any failure.
- **Cross-File Correlation** — Detects shared IPs, domains, and JA3 fingerprints across files.
- **Merged Dashboard** — Aggregated results with per-file detail cards and batch summary.
- **Resource Limits** — Configurable limits: 1 GB per file, 50 files max, 5 GB total.
@@ -222,6 +258,8 @@ Integrates with leading threat intelligence providers:
### Case Management System
- Create, track, and close investigation cases.
- Store IOCs (IP, Domain, Hash, JA3, URL) with severity and context.
+- Persist ATT&CK hypotheses and capture-quality metrics with each analysis.
+- Replace stale IOC rows cleanly when an existing analysis is re-saved.
- Investigation notes, tag-based organization, and search.
### Professional PDF Export
@@ -246,6 +284,10 @@ platforms, SIEM systems, and custom scripts can submit PCAPs, poll job progress,
retrieve cases/PDF reports, and pull IOC feeds (JSON / CSV / STIX 2.1)
programmatically. It reuses the same 10-stage pipeline, SQLite case database, and
configuration as the UI; DB-backed API keys are managed from the API Keys tab.
+Headless results include capture-quality metrics and ATT&CK hypotheses, while IOC
+feeds include the technique IDs associated with contributing analyses. Uploads are
+streamed and validated before queueing; queue or persistence failures remove the
+provisional file and case instead of leaving orphans.
```bash
make run-api # http://localhost:8000
@@ -261,13 +303,13 @@ make smoke-api # end-to-end smoke test against the local API
```
app/
-├── analysis/ # Correlation engine, flow analysis, IOC scorer, narrator
+├── analysis/ # Correlation, flow/IOC scoring, narration, capture visibility
├── api/ # FastAPI integrations API (REST endpoints, auth, key mgmt)
│ ├── routers/ # health, pcaps, jobs, cases, iocs, admin
│ ├── key_auth.py # DB + env-var authentication pipeline
│ ├── key_repository.py # SQLite API key store
│ ├── rate_limiter.py # Sliding-window per-key rate limiter
-│ └── worker.py # Background pipeline execution (ProcessPoolExecutor)
+│ └── queue.py # Background pipeline execution (ProcessPoolExecutor)
├── database/ # Case management (SQLite)
├── llm/ # LLM client + multi-provider dispatch (providers.py)
├── pipeline/ # 10-stage analysis pipeline
@@ -286,7 +328,7 @@ app/
├── reports/ # PDF report generation (WeasyPrint + kaleido charts)
├── security/ # OPSEC hardening & data sanitization
├── threat_intel/ # MITRE ATT&CK mapping
-├── ui/ # Streamlit interface (9 tabs, severity color system)
+├── ui/ # Streamlit interface (10 tabs, upload validation, MITRE workspace)
├── utils/ # Export, GeoIP, config, binary discovery, CEF
├── config.py # Application defaults
└── main.py # Streamlit entry point
@@ -332,6 +374,7 @@ Compose notes:
- `./data` is mounted into the container, so PCAPs, carved files, Zeek logs, and
the case database live on the host. Put YARA rules under `./data/yara_rules`.
+ Set `PCAP_HUNTER_DATA_BIND` to use a different host data directory.
- API keys saved in the UI persist in the `pcap-hunter-home` volume; the compose
file pins `hostname:` so the config encryption key stays stable across
container recreation.
@@ -428,7 +471,7 @@ Open `http://localhost:8501` in your browser.
2. **Configure** — Pick an LLM provider (LM Studio / OpenAI / Anthropic), set your home location (Continent > Country > City), OSINT API keys, and optionally a YARA rules directory in the Config tab.
3. **Analyze** — Click **Extract & Analyze** to start the pipeline.
4. **Monitor** — Watch the Progress tab as stages execute: Packet Counting > Parsing + Zeek (parallel) > DNS / TLS / Beaconing / Carving (concurrent) > YARA > OSINT > LLM Report.
-5. **Review** — Explore results across Dashboard, LLM Analysis, OSINT, Raw Data, and Cases tabs.
+5. **Review** — Explore results across Dashboard, MITRE Analysis, LLM Analysis, OSINT, Raw Data, and Cases tabs.
6. **Export** — Download CSV/JSON data, PDF reports, STIX bundles, ATT&CK Navigator layers, or CEF syslog events.
### Re-run Reports
@@ -446,7 +489,7 @@ Use the granular **Clear** buttons in Config to independently wipe PCAP data, OS
- Defaults in `app/config.py` (thresholds, paths, URLs)
- Persistent config in `~/.pcap_hunter_config.json` (managed by `ConfigManager`)
- API keys encrypted at rest with machine-derived PBKDF2 key
-- Environment-variable overrides: `OTT_KEY`, `VT_KEY`, `SHODAN_KEY`, etc.
+- Environment-variable overrides: `OTX_KEY`, `VT_KEY`, `SHODAN_KEY`, etc.
- LLM defaults: LM Studio at `http://localhost:1234/v1`
- YARA rules: leave the directory blank to use `data/yara_rules/` when present
@@ -502,8 +545,22 @@ identical gate inside the runtime image, independent of the host Python setup.
### Regenerating doc screenshots
`scripts/capture_screenshots.py` re-captures every README/manual screenshot with
-Playwright headless Chromium (plus tesseract for OCR) and auto-redacts IP
-addresses before saving.
+the real Docker-hosted Streamlit UI in headless Chromium. It masks IP addresses,
+API secrets, email addresses, and local user paths at the pixel level; tesseract
+provides a fallback for canvas-rendered tables and a final privacy audit.
+
+```bash
+python3 -m pip install -r requirements-docs.txt
+python3 -m playwright install chromium
+DOCS_DATA="$(mktemp -d)"
+cp data/sample.pcap "$DOCS_DATA/sample.pcap"
+PCAP_HUNTER_DATA_BIND="$DOCS_DATA" make docker-up
+python3 scripts/capture_screenshots.py --seed-docs-key
+```
+
+The isolated bind prevents local cases, keys, cache entries, or prior captures
+from appearing in the documentation. The script creates its example API key
+through the real UI, then reloads away the one-time secret before capture.
### Testing discipline
diff --git a/app/__init__.py b/app/__init__.py
index 5becc17..8c0d5d5 100644
--- a/app/__init__.py
+++ b/app/__init__.py
@@ -1 +1 @@
-__version__ = "1.0.0"
+__version__ = "2.0.0"
diff --git a/app/analysis/visibility.py b/app/analysis/visibility.py
new file mode 100644
index 0000000..e214c08
--- /dev/null
+++ b/app/analysis/visibility.py
@@ -0,0 +1,117 @@
+"""Capture quality and detector-visibility metrics.
+
+PCAP analysis is only as trustworthy as the telemetry that made it through the
+capture and pipeline. These helpers turn the available flow/stage metadata
+into explicit, serializable metrics for the analyst workspace and case record.
+They intentionally distinguish an observed zero from an unavailable detector.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from datetime import datetime, timezone
+from typing import Any
+
+
+def _as_float(value: Any) -> float | None:
+ try:
+ return float(value)
+ except (TypeError, ValueError):
+ return None
+
+
+def _flow_bounds(flow: Mapping[str, Any]) -> tuple[float | None, float | None]:
+ first = _as_float(flow.get("first_ts"))
+ last = _as_float(flow.get("last_ts"))
+ samples = flow.get("pkt_times") or []
+ if first is None and samples:
+ sample_values = [value for item in samples if (value := _as_float(item)) is not None]
+ first = min(sample_values, default=None)
+ if last is None and samples:
+ sample_values = [value for item in samples if (value := _as_float(item)) is not None]
+ last = max(sample_values, default=None)
+ return first, last
+
+
+def _iso_utc(timestamp: float | None) -> str | None:
+ if timestamp is None:
+ return None
+ return datetime.fromtimestamp(timestamp, tz=timezone.utc).isoformat()
+
+
+def build_capture_metrics(state: Mapping[str, Any]) -> dict[str, Any]:
+ """Build quality, scale, and visibility metrics from an analysis state."""
+ features = state.get("features") if isinstance(state.get("features"), dict) else {}
+ flows = features.get("flows") or []
+ flows = [flow for flow in flows if isinstance(flow, Mapping)]
+ artifacts = features.get("artifacts") or {}
+
+ packet_count = state.get("__total_pkts")
+ try:
+ packet_count = int(packet_count) if packet_count is not None else 0
+ except (TypeError, ValueError):
+ packet_count = 0
+
+ parsed_packets = sum(int(flow.get("count") or 0) for flow in flows)
+ total_bytes = sum(int(flow.get("bytes") or 0) for flow in flows)
+ sources = {str(flow.get("src")) for flow in flows if flow.get("src")}
+ destinations = {str(flow.get("dst")) for flow in flows if flow.get("dst")}
+ protocols = {str(flow.get("proto")) for flow in flows if flow.get("proto")}
+ flow_bounds = [_flow_bounds(flow) for flow in flows]
+ first_values = [first for first, _ in flow_bounds if first is not None]
+ last_values = [last for _, last in flow_bounds if last is not None]
+ first_seen = min(first_values, default=None)
+ last_seen = max(last_values, default=None)
+ duration = max(0.0, last_seen - first_seen) if first_seen is not None and last_seen is not None else None
+
+ sampled_flows = sum(1 for flow in flows if int(flow.get("count") or 0) > len(flow.get("pkt_times") or []))
+ parse_ratio = parsed_packets / packet_count if packet_count > 0 else None
+ detector_status = {
+ "packet_flow": "available" if isinstance(state.get("features"), dict) else "unavailable",
+ "zeek": "available" if state.get("zeek_tables") else "unavailable",
+ "dns": "available"
+ if state.get("dns_analysis") and not state.get("dns_analysis", {}).get("error")
+ else "partial",
+ "tls": "available"
+ if state.get("tls_analysis") and not state.get("tls_analysis", {}).get("error")
+ else "partial",
+ "yara": "available" if isinstance(state.get("yara_results"), dict) else "unavailable",
+ "osint": "available" if state.get("osint") else "unavailable",
+ "correlation": "available" if isinstance(state.get("correlations"), list) else "unavailable",
+ }
+ warnings = [str(value) for value in (state.get("pipeline_warnings") or [])]
+ gaps = [name for name, status in detector_status.items() if status in {"partial", "unavailable"}]
+ limitations: list[str] = []
+ if packet_count and parsed_packets < packet_count:
+ limitations.append(
+ "Parsed packet count is below the capture count; packet parsing may be capped or incomplete."
+ )
+ if sampled_flows:
+ limitations.append(
+ "Per-flow packet samples are capped; flow totals and first/last timestamps remain authoritative."
+ )
+ if warnings:
+ limitations.append("One or more pipeline stages reported warnings: " + ", ".join(warnings[:5]))
+ if not state.get("zeek_tables"):
+ limitations.append("Zeek protocol logs are unavailable, so application-layer visibility is reduced.")
+
+ return {
+ "packet_count": packet_count,
+ "parsed_packet_count": parsed_packets,
+ "parse_ratio": round(parse_ratio, 4) if parse_ratio is not None else None,
+ "flow_count": len(flows),
+ "total_bytes": total_bytes,
+ "unique_sources": len(sources),
+ "unique_destinations": len(destinations),
+ "unique_protocols": len(protocols),
+ "unique_ips": len(artifacts.get("ips") or []),
+ "unique_domains": len(artifacts.get("domains") or []),
+ "sampled_flow_count": sampled_flows,
+ "first_seen": _iso_utc(first_seen),
+ "last_seen": _iso_utc(last_seen),
+ "duration_seconds": round(duration, 3) if duration is not None else None,
+ "detectors": detector_status,
+ "visibility_gaps": gaps,
+ "pipeline_warnings": warnings,
+ "limitations": limitations,
+ }
diff --git a/app/api/app.py b/app/api/app.py
index 4a22bbf..e802b32 100644
--- a/app/api/app.py
+++ b/app/api/app.py
@@ -19,6 +19,7 @@
from fastapi.responses import JSONResponse
from starlette.exceptions import HTTPException as StarletteHTTPException
+from app import __version__
from app.api.auth import Scope
from app.api.deps import get_key_repo, get_settings, get_usage_tracker
from app.api.queue import recover_stale_running_jobs
@@ -183,7 +184,7 @@ def create_app() -> FastAPI:
app = FastAPI(
title="PCAP Hunter Integrations API",
- version="1.0.0",
+ version=__version__,
docs_url="/docs",
redoc_url="/redoc",
openapi_url="/api/v1/openapi.json",
diff --git a/app/api/feed.py b/app/api/feed.py
index 63bd4e8..4f4e408 100644
--- a/app/api/feed.py
+++ b/app/api/feed.py
@@ -48,6 +48,7 @@ def query_iocs(repo: CaseRepository, filt: IOCFilter) -> list[dict[str, Any]]:
MIN(a.analyzed_at) AS first_seen,
MAX(a.analyzed_at) AS last_seen,
GROUP_CONCAT(DISTINCT a.case_id) AS case_ids,
+ GROUP_CONCAT(DISTINCT i.analysis_id) AS analysis_ids,
GROUP_CONCAT(DISTINCT t.name) AS tag_names
FROM iocs i
JOIN analyses a ON i.analysis_id = a.id
@@ -85,6 +86,20 @@ def query_iocs(repo: CaseRepository, filt: IOCFilter) -> list[dict[str, Any]]:
conn = repo._get_conn()
try:
rows = conn.execute(sql, params).fetchall()
+ analysis_ids = {
+ analysis_id for row in rows for analysis_id in (row["analysis_ids"] or "").split(",") if analysis_id
+ }
+ mapping_by_analysis: dict[str, dict] = {}
+ if analysis_ids:
+ placeholders = ",".join("?" * len(analysis_ids))
+ mapping_rows = conn.execute(
+ f"SELECT id, attack_mapping_json FROM analyses WHERE id IN ({placeholders})",
+ tuple(analysis_ids),
+ ).fetchall()
+ for mapping_row in mapping_rows:
+ mapping = repo._decompress_json(mapping_row["attack_mapping_json"])
+ if isinstance(mapping, dict):
+ mapping_by_analysis[mapping_row["id"]] = mapping
finally:
conn.close()
@@ -92,6 +107,14 @@ def query_iocs(repo: CaseRepository, filt: IOCFilter) -> list[dict[str, Any]]:
for row in rows:
d = dict(row)
score = int(d["score"] or 50)
+ techniques = sorted(
+ {
+ str(technique.get("technique_id"))
+ for analysis_id in (d.get("analysis_ids") or "").split(",")
+ for technique in (mapping_by_analysis.get(analysis_id, {}).get("techniques") or [])
+ if isinstance(technique, dict) and technique.get("technique_id")
+ }
+ )
out.append(
{
"type": d["ioc_type"],
@@ -102,7 +125,7 @@ def query_iocs(repo: CaseRepository, filt: IOCFilter) -> list[dict[str, Any]]:
"first_seen": d["first_seen"],
"last_seen": d["last_seen"],
"case_ids": [c for c in (d.get("case_ids") or "").split(",") if c],
- "mitre_techniques": [], # Future: derive from analysis features
+ "mitre_techniques": techniques,
}
)
return out
diff --git a/app/api/models.py b/app/api/models.py
index a6508a8..5e23f60 100644
--- a/app/api/models.py
+++ b/app/api/models.py
@@ -80,7 +80,6 @@ class PcapSubmissionForm(BaseModel):
tags: str | None = None # JSON-encoded array
severity_hint: str | None = None
osint_enabled: bool = True
- llm_enabled: bool = True
pyshark_packet_limit: int | None = None
def parsed_tags(self) -> list[str]:
diff --git a/app/api/queue.py b/app/api/queue.py
index 9666a24..1ec812e 100644
--- a/app/api/queue.py
+++ b/app/api/queue.py
@@ -27,7 +27,6 @@
WARNING_OSINT_NOT_CONFIGURED = "osint_not_configured"
WARNING_OSINT_FAILED = "osint_failed"
WARNING_YARA_FAILED = "yara_failed"
-WARNING_LLM_UNSUPPORTED = "llm_unsupported_on_api_path"
def _load_osint_keys() -> dict[str, str]:
@@ -168,9 +167,33 @@ def _persist_analysis(
"""
# Mirrors app/ui/cases_tab.py:_quick_save_analysis. Persistence failures
# must not lose the pipeline result -> warn, keep analysis_id None.
+ from app.analysis.visibility import build_capture_metrics
from app.database.models import Analysis
+ from app.threat_intel.attack_mapping import ATTACKMapper
try:
+ mapping = ATTACKMapper().map_analysis(
+ features=result.features,
+ dns_analysis=result.dns_analysis or {},
+ tls_analysis=result.tls_analysis or {},
+ yara_results=yara_results or {},
+ beacon_results=result.beacon_df_records,
+ osint=osint_data or {},
+ )
+ result.attack_mapping = mapping.to_dict()
+ result.mitre_techniques = [technique.technique_id for technique in mapping.techniques]
+ result.capture_metrics = build_capture_metrics(
+ {
+ "features": result.features,
+ "__total_pkts": result.packet_count,
+ "dns_analysis": result.dns_analysis,
+ "tls_analysis": result.tls_analysis,
+ "zeek_tables": result.zeek_tables,
+ "yara_results": yara_results,
+ "osint": osint_data,
+ "pipeline_warnings": result.warnings,
+ }
+ )
analysis = Analysis(
case_id=job.case_id,
pcap_path=pcap_path,
@@ -181,6 +204,8 @@ def _persist_analysis(
yara_results=yara_results,
dns_analysis=result.dns_analysis or None,
tls_analysis=result.tls_analysis or None,
+ attack_mapping=result.attack_mapping,
+ capture_metrics=result.capture_metrics,
)
if result.beacon_df_records:
analysis.features["beacon_records"] = result.beacon_df_records
@@ -246,10 +271,6 @@ def _on_event(event: ProgressEvent) -> None:
yara_results = _run_yara_stage(result, opts, job_id, repo)
osint_data = _run_osint_stage(result, opts, job_id, repo)
- # --- LLM report: not yet supported headless (needs UI correlation context) ---
- if opts.get("llm_enabled", True):
- result.warnings.append(WARNING_LLM_UNSUPPORTED)
-
_persist_analysis(result, job, pcap_path, osint_data, yara_results, repo)
result_blob = json.dumps(result.to_dict()).encode("utf-8")
diff --git a/app/api/routers/health.py b/app/api/routers/health.py
index 8fd065b..43094a7 100644
--- a/app/api/routers/health.py
+++ b/app/api/routers/health.py
@@ -6,7 +6,7 @@
from fastapi import APIRouter, Depends, HTTPException
-from app.api.deps import get_queue, get_repo
+from app.api.deps import get_repo
router = APIRouter()
@@ -17,7 +17,7 @@ def healthz() -> dict:
@router.get("/readyz")
-def readyz(repo=Depends(get_repo), queue=Depends(get_queue)) -> dict:
+def readyz(repo=Depends(get_repo)) -> dict:
failures: list[str] = []
# DB check
diff --git a/app/api/routers/iocs.py b/app/api/routers/iocs.py
index f50f478..aca3f0a 100644
--- a/app/api/routers/iocs.py
+++ b/app/api/routers/iocs.py
@@ -167,8 +167,8 @@ def iocs_csv(
# ── STIX 2.1 feed ──────────────────────────────────────────────────────────
-@router.get("/iocs.stix")
-@router.get("/iocs/stix")
+@router.get("/iocs.stix", operation_id="iocs_stix_dot")
+@router.get("/iocs/stix", operation_id="iocs_stix_path")
def iocs_stix(
request: Request,
since: str | None = Query(default=None),
diff --git a/app/api/routers/pcaps.py b/app/api/routers/pcaps.py
index 9cdd515..ccf0f14 100644
--- a/app/api/routers/pcaps.py
+++ b/app/api/routers/pcaps.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import logging
import os
import pathlib
import uuid
@@ -17,6 +18,7 @@
router = APIRouter(prefix="/api/v1/pcaps", tags=["ingress"])
UPLOADS_DIR_DEFAULT = pathlib.Path("data/api_uploads")
+logger = logging.getLogger(__name__)
def _uploads_dir() -> pathlib.Path:
@@ -25,6 +27,18 @@ def _uploads_dir() -> pathlib.Path:
return p
+def _cleanup_failed_submission(repo, case_id: str, out_path: pathlib.Path, *, case_created: bool) -> None:
+ try:
+ out_path.unlink(missing_ok=True)
+ except Exception:
+ logger.warning("Failed to remove rejected API upload %s", out_path, exc_info=True)
+ if case_created:
+ try:
+ repo.delete_case(case_id)
+ except Exception:
+ logger.warning("Failed to remove rejected API case %s", case_id, exc_info=True)
+
+
@router.post("", status_code=202, response_model=PcapSubmissionResponse)
async def submit_pcap(
pcap: UploadFile = File(...),
@@ -32,11 +46,9 @@ async def submit_pcap(
tags: str | None = Form(default=None),
severity_hint: str | None = Form(default=None),
osint_enabled: bool = Form(default=True),
- llm_enabled: bool = Form(default=True),
pyshark_packet_limit: int | None = Form(default=None),
_scope=Depends(require_full_scope),
repo=Depends(get_repo),
- queue=Depends(get_queue),
settings=Depends(get_settings),
) -> PcapSubmissionResponse:
case_id = uuid.uuid4().hex[:8]
@@ -69,7 +81,6 @@ async def submit_pcap(
tags=tags,
severity_hint=severity_hint,
osint_enabled=osint_enabled,
- llm_enabled=llm_enabled,
pyshark_packet_limit=pyshark_packet_limit,
)
case = Case(
@@ -79,20 +90,8 @@ async def submit_pcap(
severity=Severity.from_str(form.severity_hint or "medium"),
tags=form.parsed_tags(),
)
- repo.create_case(case)
-
- # Mark source as 'api'
- conn = repo._get_conn()
- try:
- conn.execute("UPDATE cases SET source='api' WHERE id=?", (case_id,))
- conn.commit()
- finally:
- conn.close()
-
- # Enqueue
options = {
"osint_enabled": osint_enabled,
- "llm_enabled": llm_enabled,
"do_yara": True,
"do_carve": True,
"do_pyshark": True,
@@ -100,7 +99,21 @@ async def submit_pcap(
"pre_count": True,
"pyshark_packet_limit": pyshark_packet_limit,
}
+ case_created = False
try:
+ repo.create_case(case)
+ case_created = True
+
+ # Mark source as 'api'
+ conn = repo._get_conn()
+ try:
+ conn.execute("UPDATE cases SET source='api' WHERE id=?", (case_id,))
+ conn.commit()
+ finally:
+ conn.close()
+
+ # Enqueue
+ queue = get_queue()
job_id = queue.enqueue(
JobSubmission(
case_id=case_id,
@@ -108,8 +121,12 @@ async def submit_pcap(
options=options,
)
)
- except QueueFullError:
- raise HTTPException(status_code=503, detail="queue_full", headers={"Retry-After": "60"})
+ except QueueFullError as exc:
+ _cleanup_failed_submission(repo, case_id, out_path, case_created=case_created)
+ raise HTTPException(status_code=503, detail="queue_full", headers={"Retry-After": "60"}) from exc
+ except Exception:
+ _cleanup_failed_submission(repo, case_id, out_path, case_created=case_created)
+ raise
return PcapSubmissionResponse(
job_id=job_id,
diff --git a/app/api/validation.py b/app/api/validation.py
index 6d6694e..8a50c44 100644
--- a/app/api/validation.py
+++ b/app/api/validation.py
@@ -2,19 +2,6 @@
from __future__ import annotations
-# pcap classic, pcap classic byte-swapped, pcap nanosecond, pcapng
-PCAP_MAGICS: tuple[bytes, ...] = (
- b"\xd4\xc3\xb2\xa1", # classic
- b"\xa1\xb2\xc3\xd4", # classic byte-swapped
- b"\x4d\x3c\xb2\xa1", # nanosecond
- b"\xa1\xb2\x3c\x4d", # nanosecond byte-swapped
- b"\x0a\x0d\x0d\x0a", # pcapng (block type for SHB)
-)
+from app.utils.pcap_validation import PCAP_MAGICS, is_valid_pcap_magic
-
-def is_valid_pcap_magic(prefix: bytes) -> bool:
- """Return True if prefix begins with a known pcap or pcapng magic."""
- if len(prefix) < 4:
- return False
- head = prefix[:4]
- return any(head == m for m in PCAP_MAGICS)
+__all__ = ["PCAP_MAGICS", "is_valid_pcap_magic"]
diff --git a/app/database/models.py b/app/database/models.py
index 7d7e30b..19f0c70 100644
--- a/app/database/models.py
+++ b/app/database/models.py
@@ -156,6 +156,8 @@ class Analysis:
yara_results: dict | None = None
dns_analysis: dict | None = None
tls_analysis: dict | None = None
+ attack_mapping: dict | None = None
+ capture_metrics: dict | None = None
iocs: list[IOC] = field(default_factory=list)
def to_dict(self) -> dict:
@@ -172,6 +174,8 @@ def to_dict(self) -> dict:
"yara_results": self.yara_results,
"dns_analysis": self.dns_analysis,
"tls_analysis": self.tls_analysis,
+ "attack_mapping": self.attack_mapping,
+ "capture_metrics": self.capture_metrics,
"iocs": [ioc.to_dict() for ioc in self.iocs],
}
@@ -196,6 +200,8 @@ def from_dict(cls, data: dict) -> "Analysis":
yara_results=data.get("yara_results"),
dns_analysis=data.get("dns_analysis"),
tls_analysis=data.get("tls_analysis"),
+ attack_mapping=data.get("attack_mapping"),
+ capture_metrics=data.get("capture_metrics"),
iocs=iocs,
)
diff --git a/app/database/repository.py b/app/database/repository.py
index 7ec6b0f..1f4694f 100644
--- a/app/database/repository.py
+++ b/app/database/repository.py
@@ -71,7 +71,9 @@ def _init_schema(self):
report_md TEXT,
yara_json TEXT,
dns_json TEXT,
- tls_json TEXT
+ tls_json TEXT,
+ attack_mapping_json TEXT,
+ capture_metrics_json TEXT
);
-- IOCs extracted from analyses
@@ -135,6 +137,15 @@ def _init_schema(self):
CREATE INDEX IF NOT EXISTS idx_jobs_case ON jobs(case_id);
"""
)
+ # Existing case databases predate ATT&CK and capture-quality
+ # persistence. Add the columns in place so upgrades do not erase
+ # prior investigations.
+ for column in ("attack_mapping_json", "capture_metrics_json"):
+ try:
+ conn.execute(f"ALTER TABLE analyses ADD COLUMN {column} TEXT") # noqa: S608 — fixed column names
+ except sqlite3.OperationalError as exc:
+ if "duplicate column name" not in str(exc).lower():
+ raise
conn.commit()
# Idempotent column additions (ALTER TABLE ADD COLUMN errors if column exists)
@@ -406,31 +417,52 @@ def save_analysis(self, analysis: Analysis) -> str:
yara_json = self._compress_json(analysis.yara_results) if analysis.yara_results else None
dns_json = self._compress_json(analysis.dns_analysis) if analysis.dns_analysis else None
tls_json = self._compress_json(analysis.tls_analysis) if analysis.tls_analysis else None
-
+ attack_mapping_json = self._compress_json(analysis.attack_mapping) if analysis.attack_mapping else None
+ capture_metrics_json = self._compress_json(analysis.capture_metrics) if analysis.capture_metrics else None
+
+ params = (
+ analysis.id,
+ analysis.case_id,
+ analysis.pcap_path,
+ analysis.pcap_hash,
+ analysis.packet_count,
+ analysis.analyzed_at.isoformat(),
+ features_json,
+ osint_json,
+ analysis.report,
+ yara_json,
+ dns_json,
+ tls_json,
+ attack_mapping_json,
+ capture_metrics_json,
+ )
conn.execute(
"""
- INSERT OR REPLACE INTO analyses
+ INSERT INTO analyses
(id, case_id, pcap_path, pcap_hash, packet_count, analyzed_at,
- features_json, osint_json, report_md, yara_json, dns_json, tls_json)
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ features_json, osint_json, report_md, yara_json, dns_json, tls_json,
+ attack_mapping_json, capture_metrics_json)
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+ ON CONFLICT(id) DO UPDATE SET
+ case_id = excluded.case_id,
+ pcap_path = excluded.pcap_path,
+ pcap_hash = excluded.pcap_hash,
+ packet_count = excluded.packet_count,
+ analyzed_at = excluded.analyzed_at,
+ features_json = excluded.features_json,
+ osint_json = excluded.osint_json,
+ report_md = excluded.report_md,
+ yara_json = excluded.yara_json,
+ dns_json = excluded.dns_json,
+ tls_json = excluded.tls_json,
+ attack_mapping_json = excluded.attack_mapping_json,
+ capture_metrics_json = excluded.capture_metrics_json
""",
- (
- analysis.id,
- analysis.case_id,
- analysis.pcap_path,
- analysis.pcap_hash,
- analysis.packet_count,
- analysis.analyzed_at.isoformat(),
- features_json,
- osint_json,
- analysis.report,
- yara_json,
- dns_json,
- tls_json,
- ),
+ params,
)
- # Save IOCs
+ # Save IOCs as a replacement set for this analysis ID.
+ conn.execute("DELETE FROM iocs WHERE analysis_id = ?", (analysis.id,))
for ioc in analysis.iocs:
self._save_ioc(conn, analysis.id, ioc)
@@ -702,6 +734,8 @@ def _row_to_analysis(self, row: dict, conn: sqlite3.Connection) -> Analysis:
yara_results = self._decompress_json(row.get("yara_json"))
dns_analysis = self._decompress_json(row.get("dns_json"))
tls_analysis = self._decompress_json(row.get("tls_json"))
+ attack_mapping = self._decompress_json(row.get("attack_mapping_json"))
+ capture_metrics = self._decompress_json(row.get("capture_metrics_json"))
# Load IOCs
ioc_rows = conn.execute("SELECT * FROM iocs WHERE analysis_id = ?", (row["id"],)).fetchall()
@@ -729,6 +763,8 @@ def _row_to_analysis(self, row: dict, conn: sqlite3.Connection) -> Analysis:
yara_results=yara_results,
dns_analysis=dns_analysis,
tls_analysis=tls_analysis,
+ attack_mapping=attack_mapping,
+ capture_metrics=capture_metrics,
iocs=iocs,
)
diff --git a/app/llm/client.py b/app/llm/client.py
index ffba66a..ac006c3 100644
--- a/app/llm/client.py
+++ b/app/llm/client.py
@@ -1,7 +1,10 @@
+import ipaddress
import json
import logging
+import os
import re
from typing import Any
+from urllib.parse import urlparse, urlunparse
from openai import OpenAI
@@ -10,7 +13,7 @@
logger = logging.getLogger(__name__)
-def _normalize_base_url(base_url: str) -> str:
+def _normalize_base_url(base_url: str, *, local_compatible: bool = False) -> str:
"""Ensure OpenAI-compatible base URLs carry an API version path.
The OpenAI SDK appends ``/chat/completions`` to ``base_url`` verbatim, so
@@ -18,11 +21,38 @@ def _normalize_base_url(base_url: str) -> str:
which it logs as "Unexpected endpoint". Bare host:port URLs get ``/v1``
appended; URLs that already carry any path are respected as-is.
"""
- from urllib.parse import urlparse
-
url = (base_url or "").strip().rstrip("/")
if not url:
return url
+ parsed = urlparse(url)
+
+ # Docker Desktop containers cannot reliably route to the host's LAN IP
+ # (for example 192.168.2.114), while host.docker.internal is explicitly
+ # provided for host services. The compose runtime opts into this rewrite
+ # for LM Studio only; cloud OpenAI-compatible endpoints are never changed.
+ if local_compatible and os.getenv("PCAP_HUNTER_DOCKER_HOST_FALLBACK", "").lower() in {"1", "true", "yes"}:
+ hostname = parsed.hostname or ""
+ is_local = hostname.lower() in {"localhost", "host.docker.internal"}
+ if not is_local:
+ try:
+ is_local = ipaddress.ip_address(hostname).is_private
+ except ValueError:
+ pass
+ if is_local and hostname.lower() != "host.docker.internal":
+ # Preserve credentials, port, path, and query while replacing only
+ # the host component. IPv6 literals need brackets in netloc.
+ replacement_host = "host.docker.internal"
+ if parsed.port:
+ replacement_host += f":{parsed.port}"
+ netloc = replacement_host
+ if parsed.username:
+ auth = parsed.username
+ if parsed.password:
+ auth += f":{parsed.password}"
+ netloc = f"{auth}@{netloc}"
+ parsed = parsed._replace(netloc=netloc)
+ url = urlunparse(parsed)
+
if not urlparse(url).path:
return f"{url}/v1"
return url
@@ -1024,7 +1054,7 @@ def generate_report(
if lang_instruction:
msg_system += f"\n\n{lang_instruction}"
- client = OpenAI(base_url=_normalize_base_url(base_url), api_key=api_key, timeout=120.0)
+ client = OpenAI(base_url=_normalize_base_url(base_url, local_compatible=True), api_key=api_key, timeout=120.0)
# --- Generate each section ---
full_report_parts = []
@@ -1431,7 +1461,7 @@ def _get_translations() -> dict:
}
-def test_connection(base_url: str, api_key: str, model: str) -> str:
+def test_connection(base_url: str, api_key: str, model: str, *, local_compatible: bool = False) -> str:
"""
Test connectivity to the LLM endpoint by performing a minimal API call.
Returns an empty string on success, or an error message on failure.
@@ -1441,7 +1471,9 @@ def test_connection(base_url: str, api_key: str, model: str) -> str:
try:
client = OpenAI(
- base_url=_normalize_base_url(base_url), api_key=api_key or "lm-studio", timeout=C.LLM_PROBE_TIMEOUT_SECONDS
+ base_url=_normalize_base_url(base_url, local_compatible=local_compatible),
+ api_key=api_key or "lm-studio",
+ timeout=C.LLM_PROBE_TIMEOUT_SECONDS,
)
client.chat.completions.create(
model=model,
@@ -1453,7 +1485,7 @@ def test_connection(base_url: str, api_key: str, model: str) -> str:
return str(e)
-def fetch_models(base_url: str, api_key: str) -> list[str]:
+def fetch_models(base_url: str, api_key: str, *, local_compatible: bool = False) -> list[str]:
"""
Fetch available models from the LLM endpoint.
Returns a list of model IDs. Returns an empty list on failure.
@@ -1463,7 +1495,9 @@ def fetch_models(base_url: str, api_key: str) -> list[str]:
try:
client = OpenAI(
- base_url=_normalize_base_url(base_url), api_key=api_key or "lm-studio", timeout=C.LLM_PROBE_TIMEOUT_SECONDS
+ base_url=_normalize_base_url(base_url, local_compatible=local_compatible),
+ api_key=api_key or "lm-studio",
+ timeout=C.LLM_PROBE_TIMEOUT_SECONDS,
)
models = client.models.list()
return [m.id for m in models]
diff --git a/app/llm/providers.py b/app/llm/providers.py
index d9e7cdf..0778371 100644
--- a/app/llm/providers.py
+++ b/app/llm/providers.py
@@ -473,7 +473,14 @@ def probe_provider(
else:
probe_url = base_url
- err = _client.test_connection(probe_url, api_key, model)
+ # LM Studio is local to the Docker host. The client rewrites loopback/LAN
+ # host addresses to host.docker.internal when the compose runtime opts in.
+ err = _client.test_connection(
+ probe_url,
+ api_key,
+ model,
+ local_compatible=provider == PROVIDER_LMSTUDIO,
+ )
if err:
return False, err
return True, f"{provider_label(provider)} connection OK."
diff --git a/app/main.py b/app/main.py
index fceee9c..6a33280 100644
--- a/app/main.py
+++ b/app/main.py
@@ -15,6 +15,7 @@
from app import config as C
from app.analysis.flow_aggregates import compute_flow_aggregates
+from app.analysis.visibility import build_capture_metrics
from app.llm import providers as llm_providers
from app.pipeline.batch import BatchProcessor, PCAPResult
from app.pipeline.geoip import GeoIP
@@ -46,6 +47,7 @@
make_results_panel,
make_tabs,
render_active_filters,
+ render_analysis_snapshot,
render_batch_summary,
render_carved,
render_chart_hint,
@@ -71,6 +73,8 @@
render_zeek,
resolve_logo_path,
)
+from app.ui.mitre_page import build_attack_mapping, render_mitre_page
+from app.ui.upload import UploadValidationError, save_uploaded_pcaps
from app.utils.common import ensure_dir, find_bin, is_public_ipv4, make_slug, uniq_sorted
from app.utils.network_utils import pick_top_public_ips
@@ -227,6 +231,9 @@ def _run_single_pcap_pipeline(
dns_analysis=result.dns_analysis or {},
tls_analysis=result.tls_analysis or {},
packet_count=result.packet_count,
+ duration_seconds=result.duration_seconds,
+ stages_run=list(result.stages_run),
+ warnings=list(result.warnings),
)
@@ -269,6 +276,7 @@ def _run_single_pcap_pipeline(
st.session_state[f"done_{llm_slug}"] = False
st.session_state[f"skip_{llm_slug}"] = False
st.session_state["report"] = None
+ st.session_state["llm_status"] = None
# Consume the trigger
st.session_state["trigger_llm_rerun"] = False
st.rerun()
@@ -308,15 +316,25 @@ def _run_single_pcap_pipeline(
)
# Tabs
-tab_upload, tab_progress, tab_dashboard, tab_llm, tab_osint, tab_results, tab_cases, tab_api_keys, tab_config = (
- make_tabs()
-)
+(
+ tab_upload,
+ tab_progress,
+ tab_dashboard,
+ tab_mitre,
+ tab_llm,
+ tab_osint,
+ tab_results,
+ tab_cases,
+ tab_api_keys,
+ tab_config,
+) = make_tabs()
# Defaults
for k, v in [
("features", None),
("osint", None),
("report", None),
+ ("llm_status", None),
("beacon_df", pd.DataFrame()),
("zeek_tables", {}),
("carved", []),
@@ -325,6 +343,10 @@ def _run_single_pcap_pipeline(
("map_reset_counter", 0),
("dns_analysis", None),
("tls_analysis", None),
+ ("attack_mapping", None),
+ ("capture_metrics", None),
+ ("pipeline_warnings", []),
+ ("pipeline_stages", []),
("yara_results", None),
("correlations", None),
("flow_asymmetry", None),
@@ -375,20 +397,22 @@ def _run_single_pcap_pipeline(
pcap_path = None
pcap_paths: list[str] = []
if uploaded_files:
- ts = int(time.time())
- for i, uploaded in enumerate(uploaded_files):
- save_path = str((C.DATA_DIR / f"upload_{ts}_{i}.pcap").resolve())
- pathlib.Path(save_path).write_bytes(uploaded.read())
- pcap_paths.append(save_path)
- pcap_path = pcap_paths[0]
- st.session_state["__pcap_path"] = pcap_path
- st.session_state["__pcap_paths"] = pcap_paths
- st.session_state["__batch_mode"] = len(pcap_paths) > 1
- if len(pcap_paths) > 1:
- names = ", ".join(u.name for u in uploaded_files)
- source_msg = f"Uploaded {len(pcap_paths)} files: {names}"
+ try:
+ saved_uploads = save_uploaded_pcaps(uploaded_files, C.DATA_DIR, timestamp=int(time.time()))
+ except UploadValidationError as exc:
+ st.error(str(exc))
+ pcap_path = None
else:
- source_msg = f"Uploaded: {uploaded_files[0].name}"
+ pcap_paths = [item.path for item in saved_uploads]
+ pcap_path = pcap_paths[0]
+ st.session_state["__pcap_path"] = pcap_path
+ st.session_state["__pcap_paths"] = pcap_paths
+ st.session_state["__batch_mode"] = len(pcap_paths) > 1
+ if len(pcap_paths) > 1:
+ names = ", ".join(item.original_name for item in saved_uploads)
+ source_msg = f"Uploaded {len(pcap_paths)} files: {names}"
+ else:
+ source_msg = f"Uploaded: {saved_uploads[0].original_name}"
elif pcap_path_text.strip():
validated = validate_pcap_path(pcap_path_text.strip())
if validated:
@@ -455,6 +479,7 @@ def _run_single_pcap_pipeline(
"features": None,
"osint": None,
"report": None,
+ "llm_status": None,
"beacon_df": pd.DataFrame(),
"zeek_tables": {},
"carved": [],
@@ -463,6 +488,10 @@ def _run_single_pcap_pipeline(
"__pcap_paths": pcap_paths or [pcap_path],
"dns_analysis": None,
"tls_analysis": None,
+ "attack_mapping": None,
+ "capture_metrics": None,
+ "pipeline_warnings": [],
+ "pipeline_stages": [],
"yara_results": None,
"correlations": None,
"flow_asymmetry": None,
@@ -616,6 +645,13 @@ def _run_single_pcap_pipeline(
st.session_state["beacon_df"] = batch_result.merged_beacons
st.session_state["dns_analysis"] = batch_result.aggregated_dns
st.session_state["tls_analysis"] = batch_result.aggregated_tls
+ st.session_state["__total_pkts"] = batch_result.correlation.total_packets
+ st.session_state["pipeline_warnings"] = sorted(
+ {warning for item in batch_result.pcap_results for warning in item.warnings}
+ )
+ st.session_state["pipeline_stages"] = sorted(
+ {stage for item in batch_result.pcap_results for stage in item.stages_run}
+ )
# Carved payloads concatenated across all successful files
st.session_state["carved"] = [
item for r in batch_result.pcap_results if not r.error for item in r.carved_items
@@ -702,8 +738,11 @@ def _run_single_pcap_pipeline(
st.session_state["beacon_df"] = beacon_df
st.session_state["osint"] = osint_data
st.session_state["carved"] = result.carved_items
+ st.session_state["__total_pkts"] = result.packet_count
st.session_state["dns_analysis"] = result.dns_analysis or None
st.session_state["tls_analysis"] = result.tls_analysis or None
+ st.session_state["pipeline_warnings"] = list(result.warnings)
+ st.session_state["pipeline_stages"] = list(result.stages_run)
_precompute_dash_aggregates(features.get("flows"))
# a fresh run supersedes any restored case
@@ -741,6 +780,17 @@ def _run_single_pcap_pipeline(
except Exception as e:
logger.warning("Post-analysis failed: %s", e)
+ # Build the ATT&CK view only after all available UI stages have joined.
+ # Keeping this here prevents the dedicated MITRE page from showing a
+ # partial mapping that predates YARA, OSINT, or correlation results.
+ try:
+ st.session_state["attack_mapping"] = build_attack_mapping(st.session_state)
+ st.session_state["capture_metrics"] = build_capture_metrics(st.session_state)
+ except Exception as exc:
+ logger.warning("MITRE mapping failed: %s", exc)
+ st.session_state["attack_mapping"] = None
+ st.session_state["capture_metrics"] = build_capture_metrics(st.session_state)
+
# ---- LLM REPORT (shared for single & batch) ----
features = st.session_state.get("features") or {}
zeek_tables = st.session_state.get("zeek_tables") or {}
@@ -786,6 +836,8 @@ def _run_single_pcap_pipeline(
"flow_asymmetry": st.session_state.get("flow_asymmetry"),
"port_anomalies": st.session_state.get("port_anomalies"),
"ja3_analysis": st.session_state.get("ja3_analysis"),
+ "attack_mapping": st.session_state.get("attack_mapping"),
+ "capture_metrics": st.session_state.get("capture_metrics"),
"rdns_map": st.session_state.get("rdns_map"),
"config": {
"limit_packets": limit_packets,
@@ -820,7 +872,12 @@ def _run_single_pcap_pipeline(
st.error(f"LLM call failed: {e}")
report_md = "_LLM generation failed. Check server/model settings._"
else:
- report_md = "_Report skipped by user._"
+ # Skipping the optional narrative must not replace the
+ # deterministic evidence with a placeholder report.
+ report_md = None
+ st.session_state["llm_status"] = "skipped"
+ if not llm_skip and report_md:
+ st.session_state["llm_status"] = "generated"
p.done(
"LLM report generated."
if not st.session_state.get(f"skip_{make_slug('LLM report')}", False)
@@ -1376,10 +1433,20 @@ def _run_single_pcap_pipeline(
st.markdown("---")
-# 4) LLM Analysis ----------------------
+# 4) MITRE ATT&CK Analysis ----------------------
+with tab_mitre:
+ render_mitre_page(st.session_state)
+
+# 5) LLM Analysis ----------------------
with tab_llm:
st.markdown("### LLM Analysis & Report")
- render_report(st.container(), st.session_state.get("report"))
+ render_report(
+ st.container(),
+ st.session_state.get("report"),
+ status=st.session_state.get("llm_status"),
+ )
+ if not st.session_state.get("report") and analysis_has_run():
+ render_analysis_snapshot(st.container(), st.session_state)
# PDF Export Section
st.markdown("---")
@@ -1467,7 +1534,7 @@ def _run_single_pcap_pipeline(
key="download_pdf",
)
-# 5) OSINT ----------------------
+# 6) OSINT ----------------------
with tab_osint:
st.markdown("### OSINT Investigation")
render_osint(
@@ -1478,7 +1545,7 @@ def _run_single_pcap_pipeline(
beacon_df=st.session_state.get("beacon_df"),
)
-# 5) Raw Data ----------------------
+# 7) Raw Data ----------------------
with tab_results:
results_panel = make_results_panel(st.container())
with results_panel:
@@ -1500,26 +1567,27 @@ def _run_single_pcap_pipeline(
render_flow_asymmetry(results_panel, st.session_state.get("flow_asymmetry"))
render_port_anomalies(results_panel, st.session_state.get("port_anomalies"))
-# 6) Cases ----------------------
+# 8) Cases ----------------------
with tab_cases:
from app.ui.cases_tab import render_cases_tab
render_cases_tab()
-# 7) API Keys --------------------
+# 9) API Keys --------------------
with tab_api_keys:
from app.ui.api_keys_tab import render_api_keys_tab
render_api_keys_tab()
-# 8) Config ----------------------
+# 10) Config ----------------------
with tab_config:
render_config_tab()
st.markdown("---")
with st.expander("Notes & OPSEC"):
st.markdown("""
-- **Tabs**: Upload → Progress → Results → Config.
+- **Tabs**: Upload → Progress → Dashboard → MITRE Analysis → LLM Analysis → OSINT → Raw Data → Cases →
+ API Keys → Config.
- **Skip** is non-blocking; pipeline continues to next phase.
- **OSINT limit**: configurable Top-N IPs by traffic; 0 = enrich all.
- Zeek JSON-first with ASCII fallback; OSINT calls have safe timeouts.
diff --git a/app/pipeline/batch.py b/app/pipeline/batch.py
index d1d2d1c..bf7fb75 100644
--- a/app/pipeline/batch.py
+++ b/app/pipeline/batch.py
@@ -86,6 +86,9 @@ class PCAPResult:
dns_analysis: dict[str, Any] = field(default_factory=dict)
tls_analysis: dict[str, Any] = field(default_factory=dict)
packet_count: int = 0
+ duration_seconds: float = 0.0
+ stages_run: list[str] = field(default_factory=list)
+ warnings: list[str] = field(default_factory=list)
error: str | None = None
diff --git a/app/pipeline/carve.py b/app/pipeline/carve.py
index 95d1d93..8dec2d3 100644
--- a/app/pipeline/carve.py
+++ b/app/pipeline/carve.py
@@ -2,6 +2,26 @@ class CarveError(Exception):
"""Raised when HTTP payload carving fails."""
+def _decode_tshark_file_data(value: str | bytes) -> bytes:
+ """Decode tshark field output for http.file_data into payload bytes.
+
+ Tshark byte-array fields are commonly rendered as hexadecimal, sometimes
+ colon-separated. Tests also feed plain text for readability, so fall back to
+ UTF-8 bytes when the field is not valid hex.
+ """
+ if isinstance(value, bytes):
+ return value
+
+ text = value.strip()
+ compact = text.replace(":", "").replace(" ", "")
+ if compact and len(compact) % 2 == 0 and all(c in "0123456789abcdefABCDEF" for c in compact):
+ try:
+ return bytes.fromhex(compact)
+ except ValueError:
+ pass
+ return value.encode("utf-8", "surrogateescape")
+
+
def carve_http_payloads(pcap_path: str, out_dir: str, phase=None) -> list[dict]:
import hashlib
import logging
@@ -84,10 +104,7 @@ def _kill() -> None:
if len(parts) < 5:
continue
ts, stream_id, ctype, clen, body = parts[:5]
- if isinstance(body, str):
- data_bytes = body.encode("utf-8", "surrogateescape")
- else:
- data_bytes = body
+ data_bytes = _decode_tshark_file_data(body)
h = hashlib.sha256(data_bytes).hexdigest()
fname = f"stream{stream_id}_{h[:10]}.bin"
fpath = pathlib.Path(out_dir) / fname
diff --git a/app/pipeline/runner.py b/app/pipeline/runner.py
index 814e230..52230b0 100644
--- a/app/pipeline/runner.py
+++ b/app/pipeline/runner.py
@@ -35,6 +35,7 @@
import pandas as pd
from app import config as C
+from app.analysis.visibility import build_capture_metrics
from app.pipeline.beacon import rank_beaconing
from app.pipeline.carve import CarveError, carve_http_payloads
from app.pipeline.dns_analysis import analyze_dns
@@ -43,6 +44,7 @@
from app.pipeline.pyshark_pass import parse_pcap_pyshark
from app.pipeline.tls_certs import analyze_certificates
from app.pipeline.zeek import load_zeek_any, merge_zeek_dns, run_zeek
+from app.threat_intel.attack_mapping import ATTACKMapper
from app.utils.string_utils import uniq_sorted
logger = logging.getLogger(__name__)
@@ -119,6 +121,8 @@ class PipelineResult:
warnings: list[str] = field(default_factory=list)
summary_narrative: str | None = None
mitre_techniques: list[str] = field(default_factory=list)
+ attack_mapping: dict | None = None
+ capture_metrics: dict | None = None
dns_analysis: dict = field(default_factory=dict)
tls_analysis: dict = field(default_factory=dict)
beacon_df_records: list[dict] = field(default_factory=list)
@@ -143,6 +147,8 @@ def to_dict(self) -> dict:
"warnings": list(self.warnings),
"summary_narrative": self.summary_narrative,
"mitre_techniques": list(self.mitre_techniques),
+ "attack_mapping": self.attack_mapping,
+ "capture_metrics": self.capture_metrics,
"dns_analysis": dict(self.dns_analysis),
"tls_analysis": dict(self.tls_analysis),
"beacon_df_records": list(self.beacon_df_records),
@@ -397,6 +403,23 @@ def _run_carve(h) -> None:
features["artifacts"]["hashes"].append(sha)
features["artifacts"]["hashes"] = uniq_sorted(features["artifacts"]["hashes"])
+ partial_mapping = ATTACKMapper().map_analysis(
+ features=features,
+ dns_analysis=dns_result,
+ tls_analysis=tls_result,
+ beacon_results=beacon_records,
+ )
+ capture_metrics = build_capture_metrics(
+ {
+ "features": features,
+ "__total_pkts": total_pkts,
+ "dns_analysis": dns_result,
+ "tls_analysis": tls_result,
+ "zeek_tables": zeek_tables,
+ "pipeline_warnings": warnings,
+ }
+ )
+
return PipelineResult(
case_id=case_id,
analysis_id=None, # caller writes the Analysis row and fills this in
@@ -411,4 +434,7 @@ def _run_carve(h) -> None:
zeek_tables=zeek_tables,
zeek_log_paths=zeek_log_paths,
carved_items=carved if options.do_carve else [],
+ mitre_techniques=[technique.technique_id for technique in partial_mapping.techniques],
+ attack_mapping=partial_mapping.to_dict(),
+ capture_metrics=capture_metrics,
)
diff --git a/app/reports/pdf_generator.py b/app/reports/pdf_generator.py
index 5dab5d2..94c6118 100644
--- a/app/reports/pdf_generator.py
+++ b/app/reports/pdf_generator.py
@@ -7,7 +7,7 @@
import re
import sys
from dataclasses import dataclass
-from datetime import datetime
+from datetime import datetime, timezone
from io import BytesIO
from pathlib import Path
from typing import TYPE_CHECKING
@@ -348,9 +348,9 @@ def _build_html(
def _render_cover_page(self, case_info: dict | None) -> str:
"""Render the cover page."""
- # Timezone-aware local time — analysts need to know which clock the
- # report was generated against (e.g. "2026-06-11 09:30 CST").
- date_str = datetime.now().astimezone().strftime("%Y-%m-%d %H:%M %Z")
+ # Use an explicit UTC abbreviation rather than platform-dependent
+ # numeric offsets such as ``+07`` returned by macOS ``%Z``.
+ date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
case_section = ""
if case_info:
@@ -1071,7 +1071,7 @@ def _render_appendix(self, features: dict) -> str:
Report Generation
- - Generated: {datetime.now().astimezone().strftime("%Y-%m-%d %H:%M:%S %Z")}
+ - Generated: {datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")}
- Classification: {self._escape(self.config.classification)}
- Tool: PCAP Hunter
diff --git a/app/threat_intel/attack_mapping.py b/app/threat_intel/attack_mapping.py
index 953ec4a..dc36ee9 100644
--- a/app/threat_intel/attack_mapping.py
+++ b/app/threat_intel/attack_mapping.py
@@ -6,6 +6,8 @@
import logging
from dataclasses import dataclass, field
+from app import config as C
+
logger = logging.getLogger(__name__)
# Valid IOC types for validation
@@ -17,6 +19,9 @@
MAX_TLS_ALERTS = 20
MAX_JA3_FINGERPRINTS = 50
MAX_FLOWS = 1000
+ATTACK_VERSION = "19.1"
+MAPPING_SCHEMA_VERSION = 2
+VALID_DISPOSITIONS = {"unreviewed", "confirmed", "dismissed"}
# Average packet size estimate (bytes) when only packet count is available
AVG_PACKET_SIZE_ESTIMATE = 800
@@ -31,6 +36,12 @@ class TechniqueMatch:
tactic: str # e.g., command-and-control
confidence: float # 0.0 - 1.0
evidence: list[str] = field(default_factory=list) # What triggered this detection
+ analytic_id: str | None = None
+ data_components: list[str] = field(default_factory=list)
+ limitations: list[str] = field(default_factory=list)
+ references: list[str] = field(default_factory=list)
+ disposition: str = "unreviewed"
+ analyst_note: str = ""
def to_dict(self) -> dict:
"""Convert to dictionary."""
@@ -40,6 +51,12 @@ def to_dict(self) -> dict:
"tactic": self.tactic,
"confidence": self.confidence,
"evidence": self.evidence,
+ "analytic_id": self.analytic_id,
+ "data_components": self.data_components,
+ "limitations": self.limitations,
+ "references": self.references,
+ "disposition": self.disposition,
+ "analyst_note": self.analyst_note,
}
@@ -51,16 +68,51 @@ class AttackMapping:
tactics_summary: dict[str, int] = field(default_factory=dict) # tactic -> count
kill_chain_phase: str = "unknown" # Most advanced phase detected
overall_severity: str = "low" # low, medium, high, critical
+ attack_version: str = ATTACK_VERSION
+ mapping_schema_version: int = MAPPING_SCHEMA_VERSION
def to_dict(self) -> dict:
"""Convert to dictionary."""
return {
+ "attack_version": self.attack_version,
+ "mapping_schema_version": self.mapping_schema_version,
"techniques": [t.to_dict() for t in self.techniques],
"tactics_summary": self.tactics_summary,
"kill_chain_phase": self.kill_chain_phase,
"overall_severity": self.overall_severity,
}
+ @classmethod
+ def from_dict(cls, data: dict) -> "AttackMapping":
+ """Restore a mapping from a session or persisted JSON payload."""
+ techniques = [
+ TechniqueMatch(
+ technique_id=str(item.get("technique_id", "")),
+ technique_name=str(item.get("technique_name", "")),
+ tactic=str(item.get("tactic", "")),
+ confidence=float(item.get("confidence", 0.0)),
+ evidence=[str(value) for value in item.get("evidence", [])],
+ analytic_id=item.get("analytic_id") or None,
+ data_components=[str(value) for value in item.get("data_components", [])],
+ limitations=[str(value) for value in item.get("limitations", [])],
+ references=[str(value) for value in item.get("references", [])],
+ disposition=item.get("disposition", "unreviewed")
+ if item.get("disposition", "unreviewed") in VALID_DISPOSITIONS
+ else "unreviewed",
+ analyst_note=str(item.get("analyst_note", "")),
+ )
+ for item in data.get("techniques", [])
+ if isinstance(item, dict)
+ ]
+ return cls(
+ techniques=techniques,
+ tactics_summary={str(key): int(value) for key, value in (data.get("tactics_summary") or {}).items()},
+ kill_chain_phase=str(data.get("kill_chain_phase", "unknown")),
+ overall_severity=str(data.get("overall_severity", "low")),
+ attack_version=str(data.get("attack_version", ATTACK_VERSION)),
+ mapping_schema_version=int(data.get("mapping_schema_version", MAPPING_SCHEMA_VERSION)),
+ )
+
# Kill chain phases in order of advancement
KILL_CHAIN_ORDER = [
@@ -142,11 +194,6 @@ def to_dict(self) -> dict:
# TLS/Certificate anomalies
"self_signed_cert": {
"techniques": [
- {
- "id": "T1587.003",
- "name": "Develop Capabilities: Digital Certificates",
- "tactic": "resource-development",
- },
{
"id": "T1573.002",
"name": "Encrypted Channel: Asymmetric Cryptography",
@@ -240,9 +287,80 @@ def to_dict(self) -> dict:
},
}
+# ATT&CK context is kept separate from the detector rules so the UI can show
+# what an analytic actually supports without pretending that every heuristic
+# is a complete ATT&CK detection. IDs are only populated where the current
+# ATT&CK site has a relevant network analytic; otherwise the match remains an
+# unlinked technique hypothesis.
+TECHNIQUE_METADATA = {
+ "T1071.001": {
+ "data_components": ["Network Traffic: Web Protocols"],
+ "analytic_id": "DET0027",
+ "references": ["https://attack.mitre.org/detectionstrategies/DET0027/"],
+ "limitations": ["Beacon periodicity alone does not prove HTTP or web-protocol C2."],
+ },
+ "T1573": {
+ "data_components": ["Network Traffic Flow"],
+ "limitations": ["Beacon periodicity alone does not prove an encrypted channel."],
+ },
+ "T1071.004": {
+ "data_components": ["Network Traffic: DNS"],
+ "limitations": ["DNS anomaly scoring does not establish operator intent or exfiltration."],
+ },
+ "T1568.001": {
+ "data_components": ["Network Traffic: DNS"],
+ "limitations": ["Fast-flux indicators require infrastructure corroboration to distinguish benign CDNs."],
+ },
+ "T1568.002": {
+ "data_components": ["Network Traffic: DNS"],
+ "limitations": ["DGA scoring is probabilistic and should be confirmed with domain-age or endpoint evidence."],
+ },
+ "T1571": {
+ "analytic_id": "DET0227",
+ "data_components": ["Network Traffic Flow"],
+ "references": ["https://attack.mitre.org/detectionstrategies/DET0227/"],
+ "limitations": ["A non-standard port is not malicious without protocol and asset context."],
+ },
+ "T1573.002": {
+ "data_components": ["Network Traffic: SSL/TLS"],
+ "limitations": ["Certificate anomalies do not prove encrypted C2 or attacker-controlled keys."],
+ },
+ "T1041": {
+ "data_components": ["Network Traffic Flow"],
+ "limitations": ["High outbound volume alone does not establish exfiltration or C2 use."],
+ },
+ "T1048": {
+ "data_components": ["Network Traffic Flow"],
+ "limitations": ["High outbound volume alone does not identify an alternative exfiltration protocol."],
+ },
+ "T1027": {
+ "limitations": [
+ "YARA severity alone does not identify obfuscation; rule semantics and file context are required."
+ ],
+ },
+ "T1059": {
+ "limitations": ["A carved-file YARA severity does not prove command or scripting execution."],
+ },
+ "T1055": {
+ "limitations": ["A carved-file YARA severity does not prove process injection."],
+ },
+ "T1105": {
+ "limitations": ["A YARA match does not prove tool transfer without transfer lineage and endpoint evidence."],
+ },
+ "T1095": {
+ "limitations": ["JA3 reputation alone does not prove a non-application-layer protocol."],
+ },
+}
+
class ATTACKMapper:
- """Maps analysis results to MITRE ATT&CK techniques."""
+ """Maps analysis results to ATT&CK v19.1 technique hypotheses.
+
+ The mapper deliberately distinguishes a technique hypothesis from direct
+ analytic coverage. Network-only evidence cannot prove endpoint execution,
+ identity, or intent, so each match carries limitations and may omit an
+ analytic ID when the required protocol context is missing.
+ """
def __init__(self):
"""Initialize the mapper."""
@@ -301,9 +419,11 @@ def map_analysis(
# Check for large data transfers
if features:
techniques.extend(self._check_data_transfer(features))
+ techniques.extend(self._check_suspicious_ports(features))
# Deduplicate techniques
techniques = self._deduplicate_techniques(techniques)
+ self._annotate_techniques(techniques)
# Calculate tactics summary
tactics_summary = self._calculate_tactics_summary(techniques)
@@ -331,7 +451,11 @@ def _check_beaconing(self, beacon_results: list) -> list[TechniqueMatch]:
score = beacon.get("score", 0) if isinstance(beacon, dict) else 0
if score >= threshold:
for tech in rule["techniques"]:
- evidence = f"Beaconing detected with score {score:.2f} to {beacon.get('dst', 'unknown')}"
+ proto = beacon.get("proto") or beacon.get("protocol")
+ protocol_note = f" over {proto}" if proto else ""
+ evidence = (
+ f"Beaconing detected with score {score:.2f} to {beacon.get('dst', 'unknown')}{protocol_note}"
+ )
techniques.append(
TechniqueMatch(
technique_id=tech["id"],
@@ -401,8 +525,15 @@ def _check_tls(self, tls_analysis: dict) -> list[TechniqueMatch]:
"""Check TLS certificate analysis for anomalies."""
techniques = []
alerts = tls_analysis.get("alerts", [])
+ # The TLS stage uses ``alerts`` for aggregate counters while older
+ # callers may provide a list of detailed alert objects. Only the
+ # latter can support a certificate-level ATT&CK hypothesis.
+ if isinstance(alerts, dict):
+ alerts = tls_analysis.get("certificate_alerts", [])
for alert in alerts:
+ if not isinstance(alert, dict):
+ continue
alert_type = alert.get("type", "")
if alert_type == "self_signed":
@@ -557,6 +688,33 @@ def _check_data_transfer(self, features: dict) -> list[TechniqueMatch]:
return techniques
+ def _check_suspicious_ports(self, features: dict) -> list[TechniqueMatch]:
+ """Map configured C2-suspect ports without treating every high port as C2."""
+ matches: list[TechniqueMatch] = []
+ seen: set[tuple[str, int]] = set()
+ rule = self.detection_rules["non_standard_port"]
+ for flow in (features.get("flows") or [])[:MAX_FLOWS]:
+ try:
+ port = int(flow.get("dport"))
+ except (TypeError, ValueError):
+ continue
+ dst = str(flow.get("dst") or "unknown")
+ key = (dst, port)
+ if port not in C.C2_SUSPECT_PORTS or key in seen:
+ continue
+ seen.add(key)
+ for tech in rule["techniques"]:
+ matches.append(
+ TechniqueMatch(
+ technique_id=tech["id"],
+ technique_name=tech["name"],
+ tactic=tech["tactic"],
+ confidence=0.7,
+ evidence=[f"C2-suspect destination port {port} observed to {dst}"],
+ )
+ )
+ return matches
+
def _deduplicate_techniques(self, techniques: list[TechniqueMatch]) -> list[TechniqueMatch]:
"""Deduplicate techniques, keeping highest confidence and merging evidence."""
seen: dict[str, TechniqueMatch] = {}
@@ -568,6 +726,9 @@ def _deduplicate_techniques(self, techniques: list[TechniqueMatch]) -> list[Tech
existing = seen[key]
existing.confidence = max(existing.confidence, tech.confidence)
existing.evidence.extend(tech.evidence)
+ existing.data_components.extend(tech.data_components)
+ existing.limitations.extend(tech.limitations)
+ existing.references.extend(tech.references)
else:
seen[key] = TechniqueMatch(
technique_id=tech.technique_id,
@@ -575,10 +736,41 @@ def _deduplicate_techniques(self, techniques: list[TechniqueMatch]) -> list[Tech
tactic=tech.tactic,
confidence=tech.confidence,
evidence=list(tech.evidence),
+ analytic_id=tech.analytic_id,
+ data_components=list(tech.data_components),
+ limitations=list(tech.limitations),
+ references=list(tech.references),
+ disposition=tech.disposition,
+ analyst_note=tech.analyst_note,
)
return list(seen.values())
+ def _annotate_techniques(self, techniques: list[TechniqueMatch]) -> None:
+ """Attach ATT&CK analytic context and de-duplicate evidence metadata."""
+ for technique in techniques:
+ metadata = TECHNIQUE_METADATA.get(technique.technique_id, {})
+ analytic_id = metadata.get("analytic_id")
+ evidence_text = " ".join(technique.evidence).lower()
+ # DET0027 is specifically a web-protocol analytic. A generic
+ # periodic flow may still be a T1071.001 hypothesis, but it must
+ # not be presented as coverage of that analytic without HTTP/S
+ # evidence.
+ if technique.technique_id == "T1071.001" and not any(
+ token in evidence_text for token in ("http", "https", "web protocol")
+ ):
+ analytic_id = None
+ technique.analytic_id = technique.analytic_id or analytic_id
+ metadata_components = metadata.get("data_components", [])
+ if technique.technique_id == "T1071.001" and analytic_id is None:
+ metadata_components = ["Network Traffic Flow"]
+ technique.data_components = list(dict.fromkeys(technique.data_components + metadata_components))
+ technique.limitations = list(dict.fromkeys(technique.limitations + metadata.get("limitations", [])))
+ technique.references = list(dict.fromkeys(technique.references + metadata.get("references", [])))
+ technique.disposition = (
+ technique.disposition if technique.disposition in VALID_DISPOSITIONS else "unreviewed"
+ )
+
def _calculate_tactics_summary(self, techniques: list[TechniqueMatch]) -> dict[str, int]:
"""Calculate tactics summary from techniques."""
summary: dict[str, int] = {}
diff --git a/app/ui/cases_tab.py b/app/ui/cases_tab.py
index 09f1033..60eedc1 100644
--- a/app/ui/cases_tab.py
+++ b/app/ui/cases_tab.py
@@ -52,24 +52,33 @@ def _restore_analysis_to_session(analysis: Analysis) -> None:
Mirrors the session keys the pipeline populates after a live run. Keys that
are not persisted on Analysis (beacon_df, zeek_tables, carved files, JA3,
correlation/anomaly results) are reset instead of leaving stale data from a
- previously analyzed capture. Dashboard filters are cleared because they
- reference IPs/time ranges from the prior capture and would otherwise filter
- the restored flows down to nothing.
+ previously analyzed capture. ATT&CK mapping and capture-quality metrics are
+ persisted when the analysis was saved. Dashboard filters are cleared because
+ they reference IPs/time ranges from the prior capture and would otherwise
+ filter the restored flows down to nothing.
Args:
analysis: The saved analysis to load into the current session.
"""
features = analysis.features or {}
st.session_state["features"] = features
+ st.session_state["__total_pkts"] = analysis.packet_count
# Must match the dashboard fast path: top_n=10, weight="flows"
# (see _precompute_dash_aggregates in app.main).
st.session_state["dash_aggregates"] = compute_flow_aggregates(features.get("flows"), top_n=10, weight="flows")
st.session_state["osint"] = analysis.osint or {}
st.session_state["dns_analysis"] = analysis.dns_analysis
st.session_state["tls_analysis"] = analysis.tls_analysis
+ # Restore the evidence-backed mapping when available. Older cases without
+ # the new column are handled by the dedicated page's lazy recomputation.
+ st.session_state["attack_mapping"] = analysis.attack_mapping
+ st.session_state["capture_metrics"] = analysis.capture_metrics
+ st.session_state["pipeline_warnings"] = []
+ st.session_state["pipeline_stages"] = []
st.session_state["yara_results"] = analysis.yara_results
# Model default for report is "" but the app's no-report sentinel is None.
st.session_state["report"] = analysis.report or None
+ st.session_state["llm_status"] = "generated" if analysis.report else None
# Everything below isn't persisted on Analysis — reset it all, otherwise the
# dashboard mixes this case's data with leftovers from the previous live run.
st.session_state["beacon_df"] = pd.DataFrame()
@@ -583,6 +592,9 @@ def _quick_save_analysis():
return
repo = _get_repo()
+ attack_mapping = st.session_state.get("attack_mapping")
+ if hasattr(attack_mapping, "to_dict"):
+ attack_mapping = attack_mapping.to_dict()
# Create case
timestamp = datetime.now().strftime("%Y%m%d_%H%M")
@@ -603,6 +615,8 @@ def _quick_save_analysis():
yara_results=st.session_state.get("yara_results"),
dns_analysis=st.session_state.get("dns_analysis"),
tls_analysis=st.session_state.get("tls_analysis"),
+ attack_mapping=attack_mapping,
+ capture_metrics=st.session_state.get("capture_metrics"),
)
# Extract IOCs
@@ -625,6 +639,9 @@ def _add_current_analysis_to_case(case: Case):
return
repo = _get_repo()
+ attack_mapping = st.session_state.get("attack_mapping")
+ if hasattr(attack_mapping, "to_dict"):
+ attack_mapping = attack_mapping.to_dict()
analysis = Analysis(
case_id=case.id,
@@ -636,6 +653,8 @@ def _add_current_analysis_to_case(case: Case):
yara_results=st.session_state.get("yara_results"),
dns_analysis=st.session_state.get("dns_analysis"),
tls_analysis=st.session_state.get("tls_analysis"),
+ attack_mapping=attack_mapping,
+ capture_metrics=st.session_state.get("capture_metrics"),
)
analysis.iocs = repo.extract_iocs(analysis)
diff --git a/app/ui/config_ui.py b/app/ui/config_ui.py
index 1f55e7f..74d2b2a 100644
--- a/app/ui/config_ui.py
+++ b/app/ui/config_ui.py
@@ -225,6 +225,11 @@ def _render_llm_integration():
# ---- Provider-specific fields (only the active provider's are shown) ----
if selected_provider == llm_providers.PROVIDER_LMSTUDIO:
+ st.caption(
+ "Docker note: when LM Studio runs on this computer, "
+ "`host.docker.internal` is the reliable container route. "
+ "LAN/loopback addresses are automatically adapted in the Docker runtime."
+ )
c1, c2 = st.columns([2, 1])
with c1:
st.session_state["cfg_lm_base_url"] = st.text_input(
@@ -250,6 +255,7 @@ def _render_llm_integration():
models = fetch_models(
st.session_state.get("cfg_lm_base_url"),
st.session_state.get("cfg_lm_api_key"),
+ local_compatible=True,
)
if models:
st.session_state["available_models_lmstudio"] = models
diff --git a/app/ui/layout.py b/app/ui/layout.py
index dabfd3c..459ce6b 100644
--- a/app/ui/layout.py
+++ b/app/ui/layout.py
@@ -131,11 +131,12 @@ def render_export_buttons(data, prefix: str, key_suffix: str = "", is_dataframe:
def make_tabs():
- """Top tabs: Upload • Progress • Dashboard • LLM Analysis • OSINT • Results • Cases • API Keys • Config."""
+ """Top tabs, including a dedicated MITRE analysis workspace."""
tab_names = [
"📤 Upload",
"📈 Progress",
"📊 Dashboard",
+ "🧭 MITRE Analysis",
"🤖 LLM Analysis",
"🕵️ OSINT",
"📋 Raw Data",
@@ -144,7 +145,7 @@ def make_tabs():
"⚙️ Config",
]
tabs = st.tabs(tab_names)
- return tabs[0], tabs[1], tabs[2], tabs[3], tabs[4], tabs[5], tabs[6], tabs[7], tabs[8]
+ return tuple(tabs)
def make_progress_panel(container):
@@ -311,7 +312,14 @@ def render_threat_summary(
def analysis_has_run() -> bool:
"""True once a pipeline run (or case restore) has populated session state."""
feats = st.session_state.get("features") or {}
- return bool(feats.get("flows") or st.session_state.get("zeek_tables"))
+ return bool(
+ feats.get("flows")
+ or st.session_state.get("zeek_tables")
+ or st.session_state.get("__total_pkts") is not None
+ or st.session_state.get("capture_metrics") is not None
+ or st.session_state.get("attack_mapping") is not None
+ or st.session_state.get("correlations") is not None
+ )
# Backward-compatible alias for internal call sites that predate the public name.
@@ -1635,17 +1643,101 @@ def render_ja3(result_col, ja3_df, ja3_analysis: dict | None):
st.caption("No TLS/JA3 data available. Run analysis with PCAP containing TLS traffic.")
-def render_report(result_col, report_md):
+def render_report(result_col, report_md, *, status: str | None = None):
with result_col:
st.markdown("#### LLM Report")
if report_md:
st.markdown(report_md)
+ elif status == "skipped":
+ st.info("⏭️ LLM report generation was skipped. The parsed and analyzed evidence remains available below.")
elif _analysis_has_run():
st.info("📝 No LLM report yet — generate one from the LLM Analysis tab.")
else:
st.info("📭 Upload a PCAP and run analysis to populate this section.")
+def render_analysis_snapshot(result_col, state: dict) -> None:
+ """Show deterministic pipeline evidence when no LLM narrative is available.
+
+ The LLM report is an optional presentation layer. Skipping it must never
+ make the packet parse, protocol analysis, detections, or extracted IOCs
+ appear to have disappeared. This compact view keeps the most useful
+ evidence visible in the LLM tab while the Dashboard and Raw Data tabs retain
+ their full drill-down views.
+ """
+ with result_col:
+ features = state.get("features") or {}
+ flows = features.get("flows") or []
+ artifacts = features.get("artifacts") or {}
+ correlations = state.get("correlations") or []
+ metrics = state.get("capture_metrics") or {}
+ warnings = state.get("pipeline_warnings") or []
+ stages = state.get("pipeline_stages") or []
+
+ st.markdown("#### Parsed & Analyzed Evidence")
+ st.caption("This evidence is produced by the deterministic pipeline and does not require an AI provider.")
+
+ artifact_total = sum(len(value) for value in artifacts.values() if isinstance(value, list))
+ packet_count = metrics.get("packet_count", state.get("__total_pkts"))
+ parsed_packets = metrics.get("parsed_packet_count")
+ metric_values = [
+ ("Packets", packet_count if packet_count is not None else "—"),
+ ("Parsed flows", len(flows)),
+ ("Extracted IOCs", artifact_total),
+ ("Correlations", len(correlations)),
+ ]
+ cols = st.columns(len(metric_values))
+ for col, (label, value) in zip(cols, metric_values):
+ with col:
+ st.metric(label, value)
+
+ if parsed_packets is not None:
+ st.caption(f"Parser coverage: {parsed_packets:,} packets represented in the stored flow evidence.")
+ if warnings:
+ st.warning("Pipeline warnings: " + "; ".join(str(item) for item in warnings[:8]))
+ if stages:
+ st.caption("Completed stages: " + ", ".join(str(stage) for stage in stages))
+
+ detail_cols = st.columns(2)
+ with detail_cols[0]:
+ with st.expander(f"Artifacts ({artifact_total})", expanded=True):
+ shown = False
+ for kind, values in artifacts.items():
+ if isinstance(values, list) and values:
+ shown = True
+ st.markdown(f"**{kind.title()}** ({len(values)})")
+ st.code("\n".join(str(value) for value in values[:50]))
+ if not shown:
+ st.caption("No extracted artifacts were reported.")
+ with detail_cols[1]:
+ with st.expander(f"Correlations ({len(correlations)})", expanded=True):
+ rows = []
+ for item in correlations[:50]:
+ if hasattr(item, "to_dict"):
+ item = item.to_dict()
+ if isinstance(item, dict):
+ rows.append(
+ {
+ "Indicator": item.get("indicator", ""),
+ "Type": item.get("indicator_type", item.get("type", "")),
+ "Verdict": item.get("verdict", ""),
+ "Score": item.get("composite_score", ""),
+ }
+ )
+ if rows:
+ st.dataframe(pd.DataFrame(rows), hide_index=True, width="stretch")
+ else:
+ st.caption("No cross-indicator correlations were reported.")
+
+ with st.expander(f"Parsed flows ({len(flows)})", expanded=False):
+ if flows:
+ st.dataframe(pd.DataFrame(flows[:100]), hide_index=True, width="stretch")
+ if len(flows) > 100:
+ st.caption(f"Showing the first 100 of {len(flows)} flows. See Raw Data for the full export.")
+ else:
+ st.caption("No parsed flow rows were reported.")
+
+
def render_dns_analysis(result_col, dns_analysis: dict | None):
"""Render DNS analysis results with DGA, tunneling, and fast flux detection."""
with result_col:
diff --git a/app/ui/mitre_page.py b/app/ui/mitre_page.py
new file mode 100644
index 0000000..70518d7
--- /dev/null
+++ b/app/ui/mitre_page.py
@@ -0,0 +1,310 @@
+"""Dedicated MITRE ATT&CK analysis workspace.
+
+The original Dashboard is intended for network findings and exploratory charts.
+This module keeps the ATT&CK view separate and deliberately frames technique
+matches as evidence-backed hypotheses until an analyst confirms them.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Any
+
+import pandas as pd
+import streamlit as st
+
+from app.analysis.visibility import build_capture_metrics
+from app.threat_intel.attack_mapping import ATTACKMapper, AttackMapping
+
+
+def _coerce_mapping(value: Any) -> AttackMapping | None:
+ """Normalize session-state or persisted mapping values."""
+ if isinstance(value, AttackMapping):
+ return value
+ if isinstance(value, Mapping):
+ return AttackMapping.from_dict(dict(value))
+ return None
+
+
+def build_attack_mapping(state: Mapping[str, Any]) -> AttackMapping | None:
+ """Build an ATT&CK mapping from the completed analysis state.
+
+ This function is intentionally UI-agnostic so the Streamlit path and tests
+ consume the same production-shaped detector inputs.
+ """
+ features = state.get("features")
+ if not isinstance(features, dict):
+ return None
+
+ beacon_df = state.get("beacon_df")
+ if isinstance(beacon_df, pd.DataFrame):
+ beacon_results = beacon_df.to_dict("records")
+ elif isinstance(beacon_df, list):
+ beacon_results = beacon_df
+ else:
+ beacon_results = []
+
+ return ATTACKMapper().map_analysis(
+ features=features,
+ dns_analysis=state.get("dns_analysis") or {},
+ tls_analysis=state.get("tls_analysis") or {},
+ yara_results=state.get("yara_results") or {},
+ beacon_results=beacon_results,
+ osint=state.get("osint") or {},
+ )
+
+
+def build_visibility_rows(state: Mapping[str, Any]) -> list[dict[str, str]]:
+ """Return explicit detector visibility status for the coverage subview."""
+ features = state.get("features")
+ zeek_tables = state.get("zeek_tables")
+ yara = state.get("yara_results")
+ osint = state.get("osint")
+ correlations = state.get("correlations")
+
+ rows: list[dict[str, str]] = []
+ if isinstance(features, dict):
+ rows.append(
+ {
+ "Detector": "Packet / flow telemetry",
+ "Status": "available",
+ "Evidence": f"{len(features.get('flows') or []):,} flows",
+ }
+ )
+ else:
+ rows.append(
+ {"Detector": "Packet / flow telemetry", "Status": "unavailable", "Evidence": "Run an analysis first"}
+ )
+
+ if isinstance(zeek_tables, dict) and zeek_tables:
+ names = ", ".join(sorted(zeek_tables.keys()))
+ rows.append({"Detector": "Zeek protocol telemetry", "Status": "available", "Evidence": names})
+ else:
+ rows.append(
+ {
+ "Detector": "Zeek protocol telemetry",
+ "Status": "unavailable",
+ "Evidence": "No Zeek tables in this session",
+ }
+ )
+
+ dns = state.get("dns_analysis")
+ rows.append(
+ {
+ "Detector": "DNS analytics",
+ "Status": "available" if isinstance(dns, dict) and dns and not dns.get("error") else "partial",
+ "Evidence": f"{(dns or {}).get('total_records', 0):,} DNS records"
+ if isinstance(dns, dict)
+ else "No DNS result",
+ }
+ )
+ tls = state.get("tls_analysis")
+ rows.append(
+ {
+ "Detector": "TLS / certificate analytics",
+ "Status": "available" if isinstance(tls, dict) and tls and not tls.get("error") else "partial",
+ "Evidence": f"{(tls or {}).get('total_certificates', 0):,} certificates"
+ if isinstance(tls, dict)
+ else "No TLS result",
+ }
+ )
+ rows.append(
+ {
+ "Detector": "YARA file analytics",
+ "Status": "available" if isinstance(yara, dict) else "unavailable",
+ "Evidence": f"{(yara or {}).get('matched', 0):,} matched files"
+ if isinstance(yara, dict)
+ else "Not run or not restored",
+ }
+ )
+ rows.append(
+ {
+ "Detector": "OSINT enrichment",
+ "Status": "available" if isinstance(osint, dict) and osint else "unavailable",
+ "Evidence": f"{len(osint or {}):,} provider result groups"
+ if isinstance(osint, dict)
+ else "Not run or not configured",
+ }
+ )
+ rows.append(
+ {
+ "Detector": "Cross-indicator correlation",
+ "Status": "available" if isinstance(correlations, list) else "unavailable",
+ "Evidence": f"{len(correlations or []):,} correlated entities"
+ if isinstance(correlations, list)
+ else "Not available",
+ }
+ )
+ return rows
+
+
+def _confidence_band(confidence: float) -> str:
+ if confidence >= 0.8:
+ return "strong support"
+ if confidence >= 0.6:
+ return "supported"
+ return "hypothesis"
+
+
+def render_mitre_page(state: Mapping[str, Any]) -> None:
+ """Render the dedicated MITRE ATT&CK / Behaviors & Coverage page."""
+ st.markdown("### MITRE ATT&CK Analysis")
+ st.caption(
+ "This workspace maps observed network evidence to ATT&CK hypotheses. "
+ "It does not prove endpoint execution, user identity, or authorization from PCAP alone."
+ )
+
+ raw_mapping = state.get("attack_mapping")
+ mapping = _coerce_mapping(raw_mapping)
+ if mapping is not None and mapping is not raw_mapping and hasattr(state, "__setitem__"):
+ # Persist the normalized object in session state. Otherwise a mapping
+ # restored from SQLite as a dict is reconstructed on every rerun and
+ # analyst dispositions/notes disappear immediately.
+ state["attack_mapping"] = mapping
+ if mapping is None:
+ mapping = build_attack_mapping(state)
+ if mapping is not None and hasattr(state, "__setitem__"):
+ # Keep the lazily-built result available to the LLM/export paths on
+ # the next rerun without requiring a second mapper invocation.
+ state["attack_mapping"] = mapping
+
+ if mapping is None:
+ st.info("Run a PCAP analysis first. ATT&CK findings will appear here after the analysis stages complete.")
+ return
+
+ techniques = mapping.techniques
+ capture_metrics = state.get("capture_metrics")
+ if not isinstance(capture_metrics, dict):
+ capture_metrics = build_capture_metrics(state)
+ if hasattr(state, "__setitem__"):
+ state["capture_metrics"] = capture_metrics
+ visibility = build_visibility_rows(state)
+ gaps = sum(1 for row in visibility if row["Status"] in {"unavailable", "partial"})
+
+ st.info(
+ "Scope: network capture only. Empty or unavailable detector results are visibility states, "
+ "not proof that the behavior did not occur."
+ )
+ st.warning(
+ "ATT&CK matches are analyst hypotheses generated from network evidence. "
+ "Validate the raw flows and supporting telemetry before treating a technique as confirmed."
+ )
+
+ metric_cols = st.columns(5)
+ metric_cols[0].metric("ATT&CK hypotheses", len(techniques))
+ metric_cols[1].metric("Observed tactics", len(mapping.tactics_summary))
+ metric_cols[2].metric("Visibility gaps", gaps)
+ metric_cols[3].metric("Parsed flows", f"{capture_metrics.get('flow_count', 0):,}")
+ metric_cols[4].metric("Mapping severity", mapping.overall_severity.upper())
+ st.caption("Mapping severity is a heuristic prioritization signal, not an incident severity verdict.")
+
+ findings_tab, coverage_tab, export_tab = st.tabs(["Findings", "Coverage & Gaps", "Exports"])
+
+ with findings_tab:
+ if not techniques:
+ st.success("No ATT&CK hypotheses were produced from the available evidence.")
+ else:
+ rows = [
+ {
+ "Assessment": "Hypothesis",
+ "ID": technique.technique_id,
+ "Technique": technique.technique_name,
+ "Tactic": technique.tactic,
+ "Analytic": technique.analytic_id or "Not linked",
+ "Confidence": f"{technique.confidence:.0%}",
+ "Disposition": technique.disposition.title(),
+ "Evidence strength": _confidence_band(technique.confidence),
+ "Evidence": "; ".join(technique.evidence[:2]),
+ }
+ for technique in techniques
+ ]
+ st.dataframe(pd.DataFrame(rows), use_container_width=True, hide_index=True)
+
+ st.markdown("#### Evidence detail")
+ for technique in techniques:
+ with st.expander(f"{technique.technique_id} · {technique.technique_name}"):
+ disposition = st.selectbox(
+ "Analyst disposition",
+ ["unreviewed", "confirmed", "dismissed"],
+ index=["unreviewed", "confirmed", "dismissed"].index(technique.disposition),
+ key=f"mitre_disposition_{technique.technique_id}",
+ )
+ technique.disposition = disposition
+ technique.analyst_note = st.text_input(
+ "Analyst note (optional)",
+ value=technique.analyst_note,
+ key=f"mitre_note_{technique.technique_id}",
+ )
+ st.write(
+ f"**Assessment:** consistent with this technique ({_confidence_band(technique.confidence)})."
+ )
+ st.write(f"**Tactic:** {technique.tactic}")
+ st.write(f"**ATT&CK analytic:** {technique.analytic_id or 'No direct analytic linked'}")
+ if technique.data_components:
+ st.write(f"**Data components:** {', '.join(technique.data_components)}")
+ if technique.evidence:
+ for evidence in technique.evidence:
+ st.markdown(f"- {evidence}")
+ else:
+ st.caption("No structured evidence reference was retained for this hypothesis.")
+ if technique.limitations:
+ st.caption("Limitations: " + " ".join(technique.limitations))
+ if technique.references:
+ st.caption("References: " + ", ".join(technique.references))
+
+ with coverage_tab:
+ st.markdown("#### Capture profile")
+ profile_rows = [
+ {"Metric": "Packets counted", "Value": f"{capture_metrics.get('packet_count', 0):,}"},
+ {"Metric": "Packets parsed into flows", "Value": f"{capture_metrics.get('parsed_packet_count', 0):,}"},
+ {
+ "Metric": "Parse ratio",
+ "Value": f"{capture_metrics['parse_ratio']:.1%}"
+ if capture_metrics.get("parse_ratio") is not None
+ else "Unavailable",
+ },
+ {"Metric": "Total flow bytes", "Value": f"{capture_metrics.get('total_bytes', 0):,}"},
+ {
+ "Metric": "Unique IPs / domains",
+ "Value": f"{capture_metrics.get('unique_ips', 0):,} / {capture_metrics.get('unique_domains', 0):,}",
+ },
+ {
+ "Metric": "Capture window",
+ "Value": f"{capture_metrics.get('duration_seconds'):.1f}s"
+ if capture_metrics.get("duration_seconds") is not None
+ else "Unavailable",
+ },
+ ]
+ st.dataframe(pd.DataFrame(profile_rows), use_container_width=True, hide_index=True)
+ review_counts = {
+ status: sum(1 for tech in techniques if tech.disposition == status)
+ for status in ("confirmed", "dismissed", "unreviewed")
+ }
+ st.caption(
+ f"Analyst review: {review_counts['confirmed']} confirmed · {review_counts['dismissed']} dismissed · "
+ f"{review_counts['unreviewed']} unreviewed"
+ )
+ st.markdown("#### Detector coverage")
+ st.dataframe(pd.DataFrame(visibility), use_container_width=True, hide_index=True)
+ st.markdown("#### What this capture cannot establish")
+ st.markdown(
+ "- Process lineage, logged-in user, asset owner, MFA outcome, and authorization state.\n"
+ "- Host-side persistence, credential theft, registry changes, or process injection.\n"
+ "- Traffic outside the capture sensor, including asymmetric or dropped packets."
+ )
+
+ with export_tab:
+ st.markdown("#### ATT&CK Navigator")
+ st.caption("The export reflects the current local mapping and preserves the evidence text for analyst review.")
+ try:
+ from app.utils.navigator_export import export_navigator_json
+
+ st.download_button(
+ "Download Navigator layer",
+ data=export_navigator_json(mapping),
+ file_name="pcap_hunter_attack_layer.json",
+ mime="application/json",
+ key="mitre_navigator_export",
+ )
+ except Exception as exc: # pragma: no cover - defensive UI guard
+ st.error(f"Navigator export unavailable: {exc}")
diff --git a/app/ui/upload.py b/app/ui/upload.py
new file mode 100644
index 0000000..dbe79ce
--- /dev/null
+++ b/app/ui/upload.py
@@ -0,0 +1,106 @@
+"""Upload helpers for the Streamlit PCAP intake flow."""
+
+from __future__ import annotations
+
+import pathlib
+import time
+import uuid
+from dataclasses import dataclass
+from typing import BinaryIO, Iterable
+
+from app import config as C
+from app.utils.common import ensure_dir
+from app.utils.pcap_validation import is_valid_pcap_magic
+
+CHUNK_SIZE = 1024 * 1024
+ALLOWED_UPLOAD_SUFFIXES = frozenset({".pcap", ".pcapng"})
+
+
+class UploadValidationError(ValueError):
+ """Raised when an uploaded PCAP should be rejected before analysis."""
+
+
+@dataclass(frozen=True)
+class SavedUpload:
+ """Metadata for a PCAP uploaded through Streamlit."""
+
+ path: str
+ original_name: str
+ size_bytes: int
+
+
+def _upload_suffix(name: str) -> str:
+ suffix = pathlib.Path(name or "").suffix.lower()
+ if suffix not in ALLOWED_UPLOAD_SUFFIXES:
+ raise UploadValidationError("Uploaded files must be .pcap or .pcapng.")
+ return suffix
+
+
+def _read_chunks(uploaded: BinaryIO) -> Iterable[bytes]:
+ while True:
+ chunk = uploaded.read(CHUNK_SIZE)
+ if not chunk:
+ break
+ if isinstance(chunk, str):
+ chunk = chunk.encode("utf-8")
+ yield chunk
+
+
+def save_uploaded_pcaps(
+ uploaded_files: list[BinaryIO],
+ data_dir: pathlib.Path | str = C.DATA_DIR,
+ *,
+ timestamp: int | None = None,
+ run_id: str | None = None,
+ max_file_size: int = C.BATCH_MAX_FILE_SIZE_BYTES,
+ max_total_size: int = C.BATCH_MAX_TOTAL_SIZE_BYTES,
+) -> list[SavedUpload]:
+ """Stream uploaded PCAPs to disk with size and magic-byte validation.
+
+ Any validation failure removes files written by this call so the UI does not
+ leave half-accepted uploads behind.
+ """
+ if not uploaded_files:
+ return []
+
+ target_dir = pathlib.Path(data_dir).resolve()
+ ensure_dir(target_dir)
+ ts = int(time.time()) if timestamp is None else timestamp
+ token = run_id or uuid.uuid4().hex[:8]
+ saved: list[SavedUpload] = []
+ total_size = 0
+
+ try:
+ for index, uploaded in enumerate(uploaded_files):
+ original_name = getattr(uploaded, "name", f"upload_{index}.pcap") or f"upload_{index}.pcap"
+ suffix = _upload_suffix(original_name)
+ save_path = target_dir / f"upload_{ts}_{token}_{index}{suffix}"
+ size = 0
+ head = b""
+
+ with save_path.open("wb") as fh:
+ for chunk in _read_chunks(uploaded):
+ if len(head) < 8:
+ head += chunk[: 8 - len(head)]
+ size += len(chunk)
+ total_size += len(chunk)
+ if size > max_file_size:
+ raise UploadValidationError(f"{original_name} is too large ({size / (1024**2):.1f} MB).")
+ if total_size > max_total_size:
+ raise UploadValidationError(f"Uploaded batch is too large ({total_size / (1024**2):.1f} MB).")
+ fh.write(chunk)
+
+ if size == 0:
+ raise UploadValidationError(f"{original_name} is empty.")
+ if not is_valid_pcap_magic(head):
+ raise UploadValidationError(f"{original_name} is not a valid PCAP/PCAPNG file.")
+
+ saved.append(SavedUpload(path=str(save_path), original_name=original_name, size_bytes=size))
+ except Exception:
+ for item in saved:
+ pathlib.Path(item.path).unlink(missing_ok=True)
+ if "save_path" in locals():
+ save_path.unlink(missing_ok=True)
+ raise
+
+ return saved
diff --git a/app/utils/config_manager.py b/app/utils/config_manager.py
index aa0d6e3..0d74736 100644
--- a/app/utils/config_manager.py
+++ b/app/utils/config_manager.py
@@ -14,6 +14,8 @@
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
+from app import config as C
+
logger = logging.getLogger(__name__)
# Keys that should be encrypted
@@ -30,8 +32,8 @@
# Default configuration values
DEFAULT_CONFIG = {
- "cfg_llm_endpoint": "http://localhost:11434/v1",
- "cfg_llm_model": "llama3.1:8b",
+ "cfg_llm_endpoint": C.LM_BASE_URL,
+ "cfg_llm_model": C.LM_MODEL,
"cfg_llm_language": "US English",
"cfg_llm_provider": "lmstudio",
"cfg_openai_model": "gpt-4o",
diff --git a/app/utils/navigator_export.py b/app/utils/navigator_export.py
index bcfa39f..a408a5a 100644
--- a/app/utils/navigator_export.py
+++ b/app/utils/navigator_export.py
@@ -14,8 +14,12 @@
# Default version strings (update when Navigator releases new versions)
# Check https://github.com/mitre-attack/attack-navigator for latest versions
-DEFAULT_ATTACK_VERSION = "14"
-DEFAULT_NAVIGATOR_VERSION = "4.9.1"
+# Keep exports aligned with the current Enterprise ATT&CK release. The
+# mapping engine remains intentionally conservative (network evidence is a
+# hypothesis, not endpoint proof), but stale version metadata makes Navigator
+# imports misleading and can hide newer technique identifiers.
+DEFAULT_ATTACK_VERSION = "19.1"
+DEFAULT_NAVIGATOR_VERSION = "5.3.2"
DEFAULT_LAYER_VERSION = "4.5"
# Color gradient for confidence levels
@@ -61,7 +65,12 @@ def export_navigator_layer(
color = _confidence_to_color(tech.confidence)
# Build comment from evidence
- comment = "; ".join(tech.evidence) if tech.evidence else ""
+ comment_parts = list(tech.evidence)
+ if tech.analytic_id:
+ comment_parts.append(f"ATT&CK analytic: {tech.analytic_id}")
+ if tech.disposition != "unreviewed":
+ comment_parts.append(f"Analyst disposition: {tech.disposition}")
+ comment = "; ".join(comment_parts)
techniques.append(
{
@@ -84,7 +93,11 @@ def export_navigator_layer(
"layer": layer_version or DEFAULT_LAYER_VERSION,
},
"domain": "enterprise-attack",
- "description": description or f"Generated by PCAP Hunter on {datetime.now(tz=timezone.utc).isoformat()}",
+ "description": description
+ or (
+ "Network-evidence ATT&CK hypotheses generated by PCAP Hunter; "
+ f"validate with endpoint telemetry. Generated on {datetime.now(tz=timezone.utc).isoformat()}"
+ ),
"filters": {
"platforms": [
"Linux",
@@ -125,6 +138,7 @@ def export_navigator_layer(
"metadata": [
{"name": "generated_by", "value": "PCAP Hunter"},
{"name": "generated_at", "value": datetime.now(tz=timezone.utc).isoformat()},
+ {"name": "mapping_schema_version", "value": str(getattr(mapping, "mapping_schema_version", 1))},
{"name": "technique_count", "value": str(len(techniques))},
{"name": "overall_severity", "value": mapping.overall_severity},
],
diff --git a/app/utils/pcap_validation.py b/app/utils/pcap_validation.py
new file mode 100644
index 0000000..2a35135
--- /dev/null
+++ b/app/utils/pcap_validation.py
@@ -0,0 +1,20 @@
+"""Shared PCAP format validation helpers."""
+
+from __future__ import annotations
+
+# pcap classic, pcap classic byte-swapped, pcap nanosecond, pcapng
+PCAP_MAGICS: tuple[bytes, ...] = (
+ b"\xd4\xc3\xb2\xa1", # classic
+ b"\xa1\xb2\xc3\xd4", # classic byte-swapped
+ b"\x4d\x3c\xb2\xa1", # nanosecond
+ b"\xa1\xb2\x3c\x4d", # nanosecond byte-swapped
+ b"\x0a\x0d\x0d\x0a", # pcapng (block type for SHB)
+)
+
+
+def is_valid_pcap_magic(prefix: bytes) -> bool:
+ """Return True if prefix begins with a known pcap or pcapng magic."""
+ if len(prefix) < 4:
+ return False
+ head = prefix[:4]
+ return any(head == magic for magic in PCAP_MAGICS)
diff --git a/docker-compose.yml b/docker-compose.yml
index 8d1b2a9..34e173a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -13,10 +13,13 @@ services:
environment:
# LM Studio runs on the HOST — localhost inside the container is the container.
LM_BASE_URL: "${LM_BASE_URL:-http://host.docker.internal:1234/v1}"
+ # Allow the UI/API to translate a host LAN or loopback address to the
+ # Docker Desktop host bridge when LM Studio is running on this computer.
+ PCAP_HUNTER_DOCKER_HOST_FALLBACK: "1"
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
- - ./data:/app/data
+ - ${PCAP_HUNTER_DATA_BIND:-./data}:/app/data
- pcap-hunter-home:/home/runner
restart: unless-stopped
@@ -36,10 +39,11 @@ services:
PYTHONPATH: "."
# LM Studio runs on the HOST — localhost inside the container is the container.
LM_BASE_URL: "${LM_BASE_URL:-http://host.docker.internal:1234/v1}"
+ PCAP_HUNTER_DOCKER_HOST_FALLBACK: "1"
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
- - ./data:/app/data
+ - ${PCAP_HUNTER_DATA_BIND:-./data}:/app/data
- pcap-hunter-home:/home/runner
depends_on:
- pcap-hunter-ui
diff --git a/docs/API.md b/docs/API.md
index b01d6f0..b60004c 100644
--- a/docs/API.md
+++ b/docs/API.md
@@ -7,7 +7,7 @@ The Integrations API lets external platforms (SOAR, SIEM, log analysis tools, cu
| | |
|---|---|
-| **API version** | `1.0.0` |
+| **API version** | `2.0.0` |
| **Base URL** | `http://:8000` — all business endpoints live under `/api/v1`; health probes (`/healthz`, `/readyz`) are at the root |
| **Interactive docs** | Swagger UI at `/docs`, ReDoc at `/redoc`, OpenAPI 3.1 JSON at `/api/v1/openapi.json` (all unauthenticated) |
| **Auth scheme** | `Authorization: Bearer ` |
@@ -323,7 +323,7 @@ curl http://localhost:8000/readyz
#### `POST /api/v1/pcaps`
-Submit a PCAP file for background analysis. The upload is streamed to disk in 1 MiB chunks (under `PCAP_HUNTER_API_UPLOADS_DIR`, default `data/api_uploads/.pcap`), size-checked during the stream and magic-checked afterwards — oversized or non-PCAP uploads are deleted immediately and rejected. On acceptance the endpoint **creates a new case** (visible in the Streamlit Cases tab with `source = api`) and **enqueues a job** on the analysis queue, then returns `202 Accepted` with polling links. Requires `full` scope. **Not idempotent** — every successful call creates a fresh case and job, even for a byte-identical file. Note one edge: if the queue is full, the 503 is raised *after* the case row and upload file were created; the orphaned upload ages out via the PCAP TTL, and the empty case remains until deleted.
+Submit a PCAP file for background analysis. The upload is streamed to disk in 1 MiB chunks (under `PCAP_HUNTER_API_UPLOADS_DIR`, default `data/api_uploads/.pcap`), size-checked during the stream and magic-checked afterwards — oversized or non-PCAP uploads are deleted immediately and rejected. On acceptance the endpoint **creates a new case** (visible in the Streamlit Cases tab with `source = api`) and **enqueues a job** on the analysis queue, then returns `202 Accepted` with polling links. Requires `full` scope. **Not idempotent** — every successful call creates a fresh case and job, even for a byte-identical file. If the queue is full, the upload file and provisional case are removed before returning `503 queue_full`.
**Auth:** `full` scope required
@@ -338,13 +338,10 @@ Submit a PCAP file for background analysis. The upload is streamed to disk in 1
| `tags` | string | No | `[]` | JSON array (e.g. `["soar","edr"]`) or comma-separated list (`soar,edr`) |
| `severity_hint` | string | No | `medium` | `low`, `medium`, `high`, or `critical`; unrecognized values fall back to `medium` |
| `osint_enabled` | boolean | No | `true` | Run OSINT enrichment after analysis (see below) |
-| `llm_enabled` | boolean | No | `true` | Accepted for forward compatibility — LLM reports are **not yet supported headless** (see below) |
| `pyshark_packet_limit` | integer | No | server default (200,000) | Cap on packets to deep-parse |
**OSINT enrichment (`osint_enabled`):** when enabled, the worker enriches the top public IPs after analysis using provider keys from the saved Streamlit config (`cfg_*_key` values) or, as a fallback, the environment (`OTX_KEY`, `VT_KEY`, `ABUSEIPDB_KEY`, `GREYNOISE_KEY`, `SHODAN_KEY`). If no provider keys are configured, the job still completes — with the warning code `osint_not_configured` in the result. Note: the API path always queries providers fresh; the OSINT response cache is not used headless.
-**LLM reports (`llm_enabled`):** LLM report generation is not yet supported on the API path. The field is accepted so existing clients keep working, but jobs complete with the warning code `llm_unsupported_on_api_path` in the result — including default submissions, since the field defaults to `true`.
-
**Sample request:**
```bash
@@ -559,9 +556,19 @@ curl http://localhost:8000/api/v1/jobs/j_7d4e9f21/result \
"packet_count": 4821,
"duration_seconds": 12.4,
"stages_run": ["pcap_count", "pyshark_pass", "zeek", "dns_analysis", "tls_certs", "beacon", "carve", "yara_scan", "osint"],
- "warnings": ["llm_unsupported_on_api_path"],
+ "warnings": [],
"summary_narrative": null,
- "mitre_techniques": [],
+ "mitre_techniques": ["T1071.001", "T1568.002"],
+ "attack_mapping": {
+ "attack_version": "19.1",
+ "techniques": []
+ },
+ "capture_metrics": {
+ "packet_count": 4821,
+ "flow_count": 312,
+ "parse_ratio": 1.0,
+ "visibility_gaps": []
+ },
"dns_analysis": {},
"tls_analysis": {},
"beacon_df_records": []
@@ -578,7 +585,6 @@ curl http://localhost:8000/api/v1/jobs/j_7d4e9f21/result \
| `osint_not_configured` | `osint_enabled` was true but no OSINT provider keys are configured (saved config or env) |
| `osint_failed` | OSINT enrichment raised an error; analysis completed without enrichment |
| `yara_failed` | YARA scan over carved files raised an error |
-| `llm_unsupported_on_api_path` | `llm_enabled` was true; LLM report generation is not yet supported headless |
Stage-level pipeline warnings may also appear (`pcap_count_unavailable`, `pyshark_failed`, `pyshark_no_data`, `zeek_failed`, `zeek_no_logs`, `dns_analysis_failed`, `tls_certs_failed`, `beacon_failed`, `carve_failed`) — each marks a stage that failed or produced no data without aborting the run.
@@ -807,7 +813,7 @@ All feed endpoints require `feed` scope (a `full`-scope key also works), are rea
**Feed semantics:**
- **Scoring:** each indicator's `score` derives from the **worst** severity recorded across its sightings: `low` = 25, `medium` = 50, `high` = 75, `critical` = 100; `severity` is the matching label.
-- **Deduplication:** the same indicator appearing in multiple analyses collapses to a single row carrying that maximum severity/score; `first_seen`/`last_seen` span all sightings and `case_ids` lists every contributing case.
+- **Deduplication:** the same indicator appearing in multiple analyses collapses to a single row carrying that maximum severity/score; `first_seen`/`last_seen` span all sightings and `case_ids` lists every contributing case. `mitre_techniques` contains the distinct persisted ATT&CK technique IDs observed in those contributing analyses.
- **Filtering:** `min_score` is applied in SQL (not post-filtered), so it composes correctly with `limit`/`cursor` — pages are always full up to `limit` and no matching rows are dropped at page boundaries.
- **Ordering:** deterministic — `last_seen` descending, then indicator value ascending as a tie-breaker. Stable ordering makes cursor pagination reliable.
- **Pagination:** `next_cursor` is non-null exactly when the page came back full (`count == limit`); pass it as `cursor` for the next page. (CSV/STIX responses don't carry a cursor — page by incrementing `cursor` by `limit` while pages stay full.)
@@ -841,7 +847,7 @@ curl "http://localhost:8000/api/v1/iocs.json?min_score=50&type=ip,domain&limit=1
"first_seen": "2026-06-10T08:02:11.402199",
"last_seen": "2026-06-12T09:15:40.992103",
"case_ids": ["c4a1b2d9"],
- "mitre_techniques": []
+ "mitre_techniques": ["T1071.001", "T1571"]
},
{
"type": "domain",
@@ -852,7 +858,7 @@ curl "http://localhost:8000/api/v1/iocs.json?min_score=50&type=ip,domain&limit=1
"first_seen": "2026-06-11T17:44:03.215587",
"last_seen": "2026-06-11T17:44:03.215587",
"case_ids": ["b91e0f2c", "c4a1b2d9"],
- "mitre_techniques": []
+ "mitre_techniques": ["T1568.002"]
}
],
"count": 2,
diff --git a/docs/en/USER_MANUAL.md b/docs/en/USER_MANUAL.md
index 0cfb4ca..c0a68e3 100644
--- a/docs/en/USER_MANUAL.md
+++ b/docs/en/USER_MANUAL.md
@@ -14,14 +14,15 @@ This manual walks through the application the way a new SOC analyst would: insta
2. [Loading PCAPs](#loading-pcaps)
3. [The Analysis Pipeline & Progress Tab](#the-analysis-pipeline--progress-tab)
4. [Dashboard](#dashboard)
-5. [OSINT Enrichment](#osint-enrichment)
-6. [LLM Analysis (AI Threat Report)](#llm-analysis-ai-threat-report)
-7. [Raw Data](#raw-data)
-8. [Cases](#cases)
-9. [Exports & PDF Reports](#exports--pdf-reports)
-10. [Configuration](#configuration)
-11. [Data Retention](#data-retention)
-12. [Troubleshooting](#troubleshooting)
+5. [MITRE ATT&CK Analysis](#mitre-attck-analysis)
+6. [OSINT Enrichment](#osint-enrichment)
+7. [LLM Analysis (AI Threat Report)](#llm-analysis-ai-threat-report)
+8. [Raw Data](#raw-data)
+9. [Cases](#cases)
+10. [Exports & PDF Reports](#exports--pdf-reports)
+11. [Configuration](#configuration)
+12. [Data Retention](#data-retention)
+13. [Troubleshooting](#troubleshooting)
---
@@ -146,6 +147,12 @@ Treat ✅ as an answer and 📭 as a gap to close before signing off an investig
The flow table includes explicit **First Seen (UTC)** and **Last Seen (UTC)** columns — true flow start/end times are kept exact even when per-flow packet samples are capped (5,000 samples per flow).
+## MITRE ATT&CK Analysis
+
+The **MITRE ATT&CK Analysis** tab is a separate workspace from the Dashboard. It presents network-derived technique matches as **hypotheses**, with supporting evidence and a confidence band for each match. It also shows detector coverage and visibility gaps so an unavailable stage is not mistaken for a clean result.
+
+The page is deliberately scoped to the capture: PCAP evidence cannot establish process lineage, user identity, authorization, host persistence, or traffic outside the sensor. Validate the raw flows and supporting endpoint telemetry before treating a technique as confirmed. The **Coverage & Gaps** sub-tab records packet parse coverage, capture window, flow totals, sampling limits, stage warnings, and detector availability. The **Exports** sub-tab provides an ATT&CK Navigator layer using current ATT&CK version metadata.
+
---
## OSINT Enrichment
@@ -299,7 +306,7 @@ Programmatic keys for the Integrations API are managed in the separate **API Key
| Red banner: required binary missing (e.g. `tshark`) | Dependencies not installed | Follow the banner's OS-specific hint, or run `python3 scripts/install.py`; verify with `make doctor`. In Docker this never happens — binaries are baked in |
| YARA panel: "no rules configured" | No rules directory set and `data/yara_rules/` absent | Set Config → YARA Rules to your rules folder, or create `data/yara_rules/` (Docker: `./data/yara_rules`); check the live rule-count feedback |
| OSINT pill ⏳ *GreyNoise rate limited* | Free/community quota exhausted | Wait for the quota window to reset or upgrade the key; cached results (💾) remain usable |
-| LM Studio "Test Connection" fails from Docker | Container can't see `localhost` on the host | Use `http://host.docker.internal:1234/v1` (the compose default); confirm LM Studio's server is started |
+| LM Studio "Test Connection" fails from Docker | Container networking differs from the host; LAN/loopback addresses may not route directly | Use `http://host.docker.internal:1234/v1` (the compose default). The Docker runtime also adapts a host LAN address such as `192.168.2.114:1234` automatically; confirm LM Studio's server is started |
| OSINT pill ➖ *no data* | Provider has no records for these indicators | Nothing to fix — that's an honest negative, not an error |
| PDF generation error (standalone macOS/Linux) | WeasyPrint system libraries missing | macOS: `brew install pango glib cairo`; Linux: install the `libpango`/`libcairo` set (the installer does this). Docker images include them |
| Dashboard panel shows 📭 after loading a case | That artifact isn't persisted in cases | Re-run the original PCAP to regenerate it |
diff --git a/docs/images/01-upload.png b/docs/images/01-upload.png
index 5492ba0..01d80fa 100644
Binary files a/docs/images/01-upload.png and b/docs/images/01-upload.png differ
diff --git a/docs/images/02-progress.png b/docs/images/02-progress.png
index 4629790..d2b4e69 100644
Binary files a/docs/images/02-progress.png and b/docs/images/02-progress.png differ
diff --git a/docs/images/03-dashboard.png b/docs/images/03-dashboard.png
index 71b2405..cd52b24 100644
Binary files a/docs/images/03-dashboard.png and b/docs/images/03-dashboard.png differ
diff --git a/docs/images/04-llm-analysis.png b/docs/images/04-llm-analysis.png
index 6618fc6..7e2608b 100644
Binary files a/docs/images/04-llm-analysis.png and b/docs/images/04-llm-analysis.png differ
diff --git a/docs/images/05-osint.png b/docs/images/05-osint.png
index 63d314d..78e02fc 100644
Binary files a/docs/images/05-osint.png and b/docs/images/05-osint.png differ
diff --git a/docs/images/06-raw-data.png b/docs/images/06-raw-data.png
index dbf858d..1a9f129 100644
Binary files a/docs/images/06-raw-data.png and b/docs/images/06-raw-data.png differ
diff --git a/docs/images/07-cases.png b/docs/images/07-cases.png
index 774af88..6b000f4 100644
Binary files a/docs/images/07-cases.png and b/docs/images/07-cases.png differ
diff --git a/docs/images/08-config.png b/docs/images/08-config.png
index 559a282..705e981 100644
Binary files a/docs/images/08-config.png and b/docs/images/08-config.png differ
diff --git a/docs/images/09-llm-providers.png b/docs/images/09-llm-providers.png
index 89dde2b..ba5286b 100644
Binary files a/docs/images/09-llm-providers.png and b/docs/images/09-llm-providers.png differ
diff --git a/docs/images/10-mitre-analysis.png b/docs/images/10-mitre-analysis.png
new file mode 100644
index 0000000..6727629
Binary files /dev/null and b/docs/images/10-mitre-analysis.png differ
diff --git a/docs/images/11-api-keys.png b/docs/images/11-api-keys.png
new file mode 100644
index 0000000..c8d130e
Binary files /dev/null and b/docs/images/11-api-keys.png differ
diff --git a/docs/zh-TW/README.md b/docs/zh-TW/README.md
index ae984e4..de744ff 100644
--- a/docs/zh-TW/README.md
+++ b/docs/zh-TW/README.md
@@ -1,6 +1,7 @@
# PCAP Hunter
[](https://github.com/ninedter/pcap-hunter/actions/workflows/ci.yml)
+[](https://github.com/ninedter/pcap-hunter/releases/tag/v2.0.0)
[](https://www.python.org/downloads/)
[](../../LICENSE)
@@ -14,8 +15,20 @@
---
+## 版本 2 新功能
+
+- **獨立的 MITRE ATT&CK 工作區** — 以證據為本的技術假設、ATT&CK v19.1 中繼資料、分析師處置、擷取涵蓋範圍、可視性缺口與 Navigator 匯出。
+- **擷取品質遙測** — 封包/流量規模、解析比率、時間範圍、取樣上限、完成階段與警告會隨 UI/API 結果傳遞，並與案件一同保存。
+- **更安全的 PCAP 攝取** — Streamlit 上傳採有限區塊串流，保留 `.pcap`/`.pcapng`，驗證 Magic Bytes、執行批次限制，拒絕時清除不完整檔案。
+- **更強的整合 API** — 無介面工作會回傳 ATT&CK 對應與擷取指標，IOC 摘要包含相關技術 ID，失敗提交會清除暫存案件與檔案。
+- **不依賴 LLM 的證據檢視** — 即使跳過或無法產生 AI 敘事，確定性的封包、IOC、關聯、階段與警告證據仍然可見。
+- **執行與匯出可靠性** — Docker 會將本地 LM Studio 位址轉至主機橋接、HTTP 提取正確解碼 tshark 位元組、案件重存會取代過期 IOC，PDF 時間戳記統一使用 UTC。
+
+---
+
## 目錄
+- [版本 2 新功能](#版本-2-新功能)
- [視覺導覽](#視覺導覽)
- [主要功能](#主要功能)
- [整合 API](#整合-api)
@@ -32,6 +45,8 @@
## 視覺導覽
+以下畫面來自在 Docker 中執行的真實版本 2 Streamlit 應用程式，並以內建範例 PCAP 完成分析。提交前會在圖片像素中遮蔽 IPv4/IPv6 位址、API 機密、電子郵件與本機使用者路徑。
+
### 1. Upload — 載入一個或多個 PCAP
拖放 `.pcap` / `.pcapng` 檔案，或貼上容器路徑。上傳多個檔案會啟動批次模式並進行跨檔案關聯分析；可關閉的入門導覽面板會引導初次使用者熟悉整個工作流程。
@@ -50,35 +65,45 @@
-### 4. LLM Analysis — AI 產生的威脅報告
+### 4. MITRE Analysis — 行為、證據與涵蓋範圍
+
+獨立的 ATT&CK 工作區將對應視為分析師假設，而不是既定事實。它會把網路證據連結至技術與適用的偵測情境，讓分析師記錄處置與筆記，明確顯示偵測缺口，並匯出含版本中繼資料的 ATT&CK Navigator 圖層。
+
+
+
+### 5. LLM Analysis — AI 產生的威脅報告
九個章節的敘事報告——從執行摘要到建議行動，外加 **IOC 摘要表**與**以真正 Markdown 表格呈現的風險矩陣（Risk Matrix）**——包含信心度修飾語與 MITRE ATT&CK 對應。可透過 LM Studio 在本地逐節產生，或透過 OpenAI / Anthropic 以單次完整上下文呼叫產生。報告支援 9 種語言，包括繁體中文（zh-TW）。
-### 5. OSINT — 多供應商 IOC 情資豐富化
+即使報告被跳過或無法產生，此分頁仍會顯示解析封包、流量、IOC、關聯、完成階段與管道警告的確定性快照。
+
+### 6. OSINT — 多供應商 IOC 情資豐富化
優先排序的 IOC 表格，將 VirusTotal、AbuseIPDB、GreyNoise、Shodan、OTX 與 VT Domain 的訊號合併為單一檢視。**供應商狀態標籤**誠實回報每個供應商的狀態（正常 / 快取 / 速率受限 / 金鑰遭拒 / 無資料），明確的 **WHOIS 查詢**下拉選單 + 按鈕與點選資料列的對話框相輔相成，IOC 搜尋並提供顯示全部結果的切換開關。子分頁涵蓋網域、詳細資訊卡、地理地圖、基礎設施 ASN 分群、匯出、裝置與筆記。
-### 6. Raw Data — Zeek log、流量、提取酬載、YARA 比對
+### 7. Raw Data — Zeek log、流量、提取酬載、YARA 比對
所有底層資料來源一應俱全：流量表（附明確的 **First/Last Seen (UTC)** 時間戳記欄位）、DNS 與 TLS 分析、NXDOMAIN 分析、JA3/JA3S 指紋、Zeek `conn.log`/`dns.log`/`http.log`/`ssl.log`、提取的 HTTP 酬載與 YARA 掃描結果。任何檢視都能匯出為 CSV 或 JSON，內建 CSV 注入防護。
-### 7. Cases — 持續性的調查追蹤
+### 8. Cases — 持續性的調查追蹤
-將任何擷取與其發現升級為案件。案件包含 IOC、嚴重程度、標籤、調查筆記、狀態與搜尋功能——儲存在本地 SQLite 資料庫。
+將任何擷取與其發現升級為案件。案件包含 IOC、嚴重程度、標籤、調查筆記、ATT&CK 對應、擷取品質指標、狀態與搜尋功能——儲存在本地 SQLite 資料庫。
-### 8. API Keys — 管理程式化存取
+### 9. API Keys — 管理程式化存取
為整合 API 建立、撤銷與監控 API 金鑰。每把金鑰擁有自己的權限範圍（完整或僅限摘要）、選用的到期時間、逐金鑰速率限制與使用量趨勢圖。環境變數金鑰會顯示為唯讀的初始（bootstrap）項目。
-### 9. Config — 集中式設定
+
+
+### 10. Config — 集中式設定
**LLM Integration** 區塊提供三種供應商（LM Studio、OpenAI、Anthropic），**YARA Rules** 區塊提供可設定的規則目錄，OSINT 供應商金鑰搭配 **Test Providers** 即時檢測按鈕，加上世界地圖的自家位置、執行檔路徑與管道門檻值——全部集中一處，各區塊並有獨立的清除按鈕。API 金鑰以 PBKDF2 加密儲存。
@@ -100,8 +125,10 @@
- **OpenAI**（雲端）— 單次完整上下文呼叫，一次送入全部證據語料產生報告。
- **Anthropic**（雲端）— 透過官方 `anthropic` SDK 使用 Claude（`claude-opus-4-8`、`claude-sonnet-4-6`、`claude-haiku-4-5`），單次呼叫並支援串流。
- **以證據為本的報告** — SOC 就緒的報告，包含嚴重程度校準評估、誤報意識、信心度修飾語、以真正 Markdown 表格呈現的風險矩陣，以及 IOC 摘要表。
+- **LLM 選用的證據檢視** — 即使略過或無法使用模型，解析封包、流量、IOC、關聯、階段與警告證據仍然可見。
- **多語言報告** — 9 種語言與地區術語：英文、繁體中文（台灣）、簡體中文、日文、韓文、義大利文、西班牙文、法文、德文。
-- **MITRE ATT&CK 對應** — 自動將偵測到的行為與 IOC 對應至 ATT&CK 技術與攻擊鏈（Kill Chain）階段。
+- **MITRE ATT&CK 分析** — 獨立的行為與涵蓋範圍工作區，將網路證據對應至版本化 ATT&CK 假設、連結適用的 Detection Strategy/Data Component、保存分析師處置並匯出 Navigator 圖層。
+- **擷取品質遙測** — 封包/流量規模、解析涵蓋率、時間範圍、取樣上限、管道警告與偵測器可視性缺口會與發現一併保存。
- **攻擊敘事合成** — 將原始事件轉譯為連貫、可執行的資安事件故事。
### IOC 優先級評分
@@ -125,6 +152,7 @@
### 多 PCAP 批次處理
- **多檔案上傳** — 同時上傳並分析多個 PCAP 檔案。
+- **經驗證的串流攝取** — `.pcap` 與 `.pcapng` 以有限區塊寫入，檢查檔案/批次上限與 Magic Bytes，任何失敗都會回復整批暫存檔。
- **跨檔案關聯** — 偵測跨檔案共用的 IP、網域與 JA3 指紋。
- **合併儀表板** — 彙整結果，附逐檔案詳細資訊卡與批次摘要。
- **資源限制** — 可設定的限制：每檔案 1 GB、最多 50 個檔案、總計 5 GB。
@@ -209,7 +237,7 @@
## 整合 API
-PCAP Hunter 隨附以 FastAPI 打造的 REST API，與 Streamlit UI 並行運作，讓 SOAR 平台、SIEM 系統與自訂腳本能以程式方式提交 PCAP、輪詢工作進度、取得案件 / PDF 報告，並拉取 IOC 摘要（JSON / CSV / STIX 2.1）。它重複使用與 UI 相同的 10 階段管道、SQLite 案件資料庫與設定；資料庫支援的 API 金鑰可在 API Keys 分頁管理。
+PCAP Hunter 隨附以 FastAPI 打造的 REST API，與 Streamlit UI 並行運作，讓 SOAR 平台、SIEM 系統與自訂腳本能以程式方式提交 PCAP、輪詢工作進度、取得案件 / PDF 報告，並拉取 IOC 摘要（JSON / CSV / STIX 2.1）。它重複使用與 UI 相同的 10 階段管道、SQLite 案件資料庫與設定；資料庫支援的 API 金鑰可在 API Keys 分頁管理。無介面結果包含擷取品質指標與 ATT&CK 假設，IOC 摘要則包含相關技術 ID；若排程或保存失敗，暫存檔案與案件會一併移除。
```bash
make run-api # http://localhost:8000
@@ -231,7 +259,7 @@ app/
│ ├── key_auth.py # 資料庫 + 環境變數認證流程
│ ├── key_repository.py # SQLite API 金鑰儲存
│ ├── rate_limiter.py # 逐金鑰滑動視窗速率限制
-│ └── worker.py # 背景管道執行（ProcessPoolExecutor）
+│ └── queue.py # 背景管道執行（ProcessPoolExecutor）
├── database/ # 案件管理（SQLite）
├── llm/ # LLM 用戶端 + 多供應商分派（providers.py）
├── pipeline/ # 10 階段分析管道
@@ -250,7 +278,7 @@ app/
├── reports/ # PDF 報告產生（WeasyPrint + kaleido 圖表）
├── security/ # OPSEC 強化與資料清理
├── threat_intel/ # MITRE ATT&CK 對應
-├── ui/ # Streamlit 介面（9 個分頁、嚴重程度色彩系統）
+├── ui/ # Streamlit 介面（10 個分頁、上傳驗證、MITRE 工作區）
├── utils/ # 匯出、GeoIP、設定、執行檔探索、CEF
├── config.py # 應用程式預設值
└── main.py # Streamlit 進入點
@@ -289,7 +317,7 @@ make docker-down # 停止 compose 服務
Compose 注意事項：
-- `./data` 會掛載進容器，因此 PCAP、提取的檔案、Zeek log 與案件資料庫都存放在主機上。YARA 規則請放在 `./data/yara_rules`。
+- `./data` 會掛載進容器，因此 PCAP、提取的檔案、Zeek log 與案件資料庫都存放在主機上。YARA 規則請放在 `./data/yara_rules`；可設定 `PCAP_HUNTER_DATA_BIND` 改用其他主機資料目錄。
- 在 UI 中儲存的 API 金鑰會保存在 `pcap-hunter-home` volume；compose 檔固定了 `hostname:`，確保設定加密金鑰在容器重建後維持穩定。
- 主機上執行的 LM Studio 從容器內即可連線——`LM_BASE_URL` 預設為 `http://host.docker.internal:1234/v1`。
- 第二個 compose 服務（`pcap-hunter-api`）使用同一個映像檔，在 8000 連接埠提供整合 API。
@@ -370,7 +398,7 @@ make run # 獨立安裝（先執行 python3 scripts/install.py）
2. **設定** — 在 Config 分頁選擇 LLM 供應商（LM Studio / OpenAI / Anthropic）、設定自家位置（洲 > 國家 > 城市）、OSINT API 金鑰，並可選擇性指定 YARA 規則目錄。
3. **分析** — 點擊 **Extract & Analyze** 啟動管道。
4. **監控** — 在 Progress 分頁觀察各階段執行：封包計數 > 解析 + Zeek（平行）> DNS / TLS / 信標偵測 / 酬載提取（同時執行）> YARA > OSINT > LLM 報告。
-5. **審閱** — 在 Dashboard、LLM Analysis、OSINT、Raw Data、Cases 分頁瀏覽結果。
+5. **審閱** — 在 Dashboard、MITRE Analysis、LLM Analysis、OSINT、Raw Data、Cases 分頁瀏覽結果。
6. **匯出** — 下載 CSV/JSON 資料、PDF 報告、STIX 套件、ATT&CK Navigator 圖層或 CEF syslog 事件。
### 重新產生報告
@@ -388,7 +416,7 @@ make run # 獨立安裝（先執行 python3 scripts/install.py）
- 預設值位於 `app/config.py`（門檻值、路徑、URL）
- 持久化設定位於 `~/.pcap_hunter_config.json`（由 `ConfigManager` 管理）
- API 金鑰以機器衍生的 PBKDF2 金鑰加密儲存
-- 環境變數覆寫：`OTT_KEY`、`VT_KEY`、`SHODAN_KEY` 等
+- 環境變數覆寫：`OTX_KEY`、`VT_KEY`、`SHODAN_KEY` 等
- LLM 預設值：LM Studio 位於 `http://localhost:1234/v1`
- YARA 規則：目錄留空時，若 `data/yara_rules/` 存在則自動使用
@@ -440,7 +468,18 @@ make verify # 格式檢查 + lint + 完整測試套件
### 重新產生文件截圖
-`scripts/capture_screenshots.py` 使用 Playwright 無介面 Chromium（搭配 tesseract 進行 OCR）重新擷取所有 README / 使用手冊截圖，並在儲存前自動遮蔽 IP 位址。
+`scripts/capture_screenshots.py` 使用 Playwright 無介面 Chromium（搭配 tesseract 進行 OCR）從真實 Docker Streamlit UI 重新擷取所有 README / 使用手冊截圖，並在圖片像素中遮蔽 IPv4/IPv6 位址、API 機密、電子郵件與本機使用者路徑。最終 OCR 稽核若仍辨識出敏感值會使擷取失敗。
+
+```bash
+python3 -m pip install -r requirements-docs.txt
+python3 -m playwright install chromium
+DOCS_DATA="$(mktemp -d)"
+cp data/sample.pcap "$DOCS_DATA/sample.pcap"
+PCAP_HUNTER_DATA_BIND="$DOCS_DATA" make docker-up
+python3 scripts/capture_screenshots.py --seed-docs-key
+```
+
+隔離的資料掛載可避免本機案件、金鑰、快取或先前擷取進入文件。範例 API 金鑰會透過真實 UI 建立，且在截圖前重新載入以清除只顯示一次的完整機密。
### 測試紀律
diff --git a/docs/zh-TW/USER_MANUAL.md b/docs/zh-TW/USER_MANUAL.md
index ad2ccbe..6baa5e3 100644
--- a/docs/zh-TW/USER_MANUAL.md
+++ b/docs/zh-TW/USER_MANUAL.md
@@ -14,14 +14,15 @@
2. [載入 PCAP](#載入-pcap)
3. [分析管道與 Progress 分頁](#分析管道與-progress-分頁)
4. [儀表板](#儀表板)
-5. [OSINT 情資豐富化](#osint-情資豐富化)
-6. [LLM 分析與 AI 威脅報告](#llm-分析與-ai-威脅報告)
-7. [Raw Data 分頁](#raw-data-分頁)
-8. [案件管理](#案件管理)
-9. [匯出與 PDF 報告](#匯出與-pdf-報告)
-10. [設定](#設定)
-11. [資料保留](#資料保留)
-12. [疑難排解](#疑難排解)
+5. [MITRE ATT&CK 分析](#mitre-attck-分析)
+6. [OSINT 情資豐富化](#osint-情資豐富化)
+7. [LLM 分析與 AI 威脅報告](#llm-分析與-ai-威脅報告)
+8. [Raw Data 分頁](#raw-data-分頁)
+9. [案件管理](#案件管理)
+10. [匯出與 PDF 報告](#匯出與-pdf-報告)
+11. [設定](#設定)
+12. [資料保留](#資料保留)
+13. [疑難排解](#疑難排解)
---
@@ -134,6 +135,12 @@ PCAP Hunter 執行 10 階段管道：
把 ✅ 視為一個答案，把 📭 視為結案前必須補齊的缺口。
+## MITRE ATT&CK 分析
+
+**MITRE ATT&CK Analysis** 是獨立於 Dashboard 的工作區。它會把網路證據對應成**分析假設**，並為每項技術顯示支援證據與信心等級；同時呈現偵測器涵蓋範圍與可見性缺口，避免把未執行的階段誤認為乾淨結果。
+
+此頁面刻意限定在擷取檔範圍內：單靠 PCAP 無法證明程序來源、使用者身分、授權狀態、主機持久化或感測器以外的流量。確認技術前，請回看原始流量並補充端點遙測。**Coverage & Gaps** 子分頁會記錄封包解析涵蓋率、擷取時間窗、流量總量、取樣限制、階段警告與偵測器可見性；**Exports** 子分頁可下載含目前 ATT&CK 版本資訊的 Navigator 圖層。
+
### 篩選器與圖表
- **世界地圖** — 框選 / 套索選取會交叉篩選整個儀表板；在 Config 設定自家位置可讓連線弧線準確呈現。
@@ -299,7 +306,7 @@ PCAP Hunter 執行 10 階段管道：
| 紅色橫幅：缺少必要執行檔（例如 `tshark`） | 相依套件未安裝 | 依照橫幅上的作業系統專屬提示操作，或執行 `python3 scripts/install.py`；以 `make doctor` 驗證。Docker 中不會發生——執行檔已內建 |
| YARA 面板顯示「no rules configured」 | 未設定規則目錄，且 `data/yara_rules/` 不存在 | 將 Config → YARA Rules 指向你的規則資料夾，或建立 `data/yara_rules/`（Docker：`./data/yara_rules`）；留意即時規則數量回饋 |
| OSINT 標籤 ⏳ *GreyNoise rate limited* | 免費 / 社群配額用罄 | 等待配額視窗重置或升級金鑰；快取結果（💾）仍可使用 |
-| Docker 環境中 LM Studio「Test Connection」失敗 | 容器看不到主機的 `localhost` | 使用 `http://host.docker.internal:1234/v1`（compose 預設值）；確認 LM Studio 的伺服器已啟動 |
+| Docker 環境中 LM Studio「Test Connection」失敗 | 容器網路與主機不同，區域網路或迴路位址可能無法直接路由 | 使用 `http://host.docker.internal:1234/v1`（compose 預設值）。Docker 執行環境也會自動將 `192.168.2.114:1234` 這類主機區域網路位址轉換；確認 LM Studio 伺服器已啟動 |
| OSINT 標籤 ➖ *no data* | 供應商沒有這些指標的紀錄 | 不需處理——這是誠實的陰性結果，不是錯誤 |
| PDF 產生錯誤（獨立安裝的 macOS/Linux） | 缺少 WeasyPrint 系統函式庫 | macOS：`brew install pango glib cairo`；Linux：安裝 `libpango`/`libcairo` 系列（安裝程式會處理）。Docker 映像檔已內含 |
| 載入案件後儀表板面板顯示 📭 | 該產物未保存在案件中 | 重新分析原始 PCAP 即可重新產生 |
diff --git a/requirements-docs.txt b/requirements-docs.txt
new file mode 100644
index 0000000..7a2ec28
--- /dev/null
+++ b/requirements-docs.txt
@@ -0,0 +1,7 @@
+# Tooling for capturing privacy-safe screenshots from the real Streamlit UI.
+# The tesseract system binary is also required (brew install tesseract or the
+# equivalent package for the host OS).
+numpy>=1.26,<3.0
+pillow>=10.0,<13.0
+playwright>=1.50,<2.0
+pytesseract>=0.3.13,<1.0
diff --git a/scripts/api_smoke_test.sh b/scripts/api_smoke_test.sh
index 5c6c68e..88f5443 100755
--- a/scripts/api_smoke_test.sh
+++ b/scripts/api_smoke_test.sh
@@ -27,8 +27,7 @@ RESP=$(curl -fsS -X POST "$API/api/v1/pcaps" \
-H "Authorization: Bearer $KEY" \
-F "pcap=@$FIXTURE" \
-F "name=smoke" \
- -F "osint_enabled=false" \
- -F "llm_enabled=false")
+ -F "osint_enabled=false")
echo "$RESP"
JOB_ID=$(echo "$RESP" | python3 -c "import json,sys; print(json.load(sys.stdin)['job_id'])")
CASE_ID=$(echo "$RESP" | python3 -c "import json,sys; print(json.load(sys.stdin)['case_id'])")
diff --git a/scripts/capture_screenshots.py b/scripts/capture_screenshots.py
index 887c4f8..18d86d1 100644
--- a/scripts/capture_screenshots.py
+++ b/scripts/capture_screenshots.py
@@ -1,4 +1,4 @@
-"""Capture README screenshots of the PCAP Hunter UI and redact all IPs.
+"""Capture real README screenshots and redact sensitive values.
Drives a headless Chromium via Playwright against a running Streamlit
instance (default http://localhost:8501), uploads ``data/sample.pcap``
@@ -6,21 +6,24 @@
clicks Extract & Analyze, waits for the pipeline to finish, then
snapshots each tab at 1440×900.
-After capture, every PNG is post-processed with Pillow to redact IPv4
-addresses. We do this at the pixel level using OCR-free regex scanning
-of the DOM rather than image OCR: the script extracts the bounding
-boxes of every text node that matches an IPv4 pattern, converts them
-into pixel coordinates, and draws solid black rectangles in Pillow.
+After capture, every PNG is post-processed with Pillow to redact IPv4/IPv6
+addresses, email addresses, full PCAP Hunter API keys, and user-home paths.
+The primary pass extracts the exact DOM bounding boxes and draws solid pixel
+rectangles. A multi-pass OCR fallback catches IPs rendered into Streamlit's
+canvas-based data tables, and a final OCR audit fails the capture if a
+recognizable sensitive value remains.
Usage:
python3 scripts/capture_screenshots.py
python3 scripts/capture_screenshots.py --base-url http://localhost:8501
+ python3 scripts/capture_screenshots.py --seed-docs-key # isolated data bind only
python3 scripts/capture_screenshots.py --keep-ips # skip redaction
"""
from __future__ import annotations
import argparse
+import ipaddress
import re
import sys
import time
@@ -37,6 +40,11 @@
# IPv4 pattern — matches anything that looks like a.b.c.d with 0-255 octets.
# We redact any IP, including RFC1918/loopback — the goal is zero IPs visible.
IPV4_RE = re.compile(r"\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})\b")
+# Broad candidate matcher; candidates are validated with ipaddress.ip_address.
+IPV6_CANDIDATE_RE = re.compile(r"(?]+", re.IGNORECASE)
# ---------------------------------------------------------------------------
@@ -126,6 +134,35 @@ def upload_sample_pcap(page: Page, pcap_path: str) -> None:
page.get_by_text("Active Source", exact=False).first.wait_for(timeout=15_000)
+def configure_local_llm(page: Page) -> None:
+ """Select a model exposed by the real LM Studio endpoint, when available."""
+ print("→ configuring LM Studio from the real Config screen")
+ click_tab(page, "Config")
+ try:
+ page.get_by_role("button", name="Fetch Models", exact=True).first.click()
+ wait_for_streamlit_idle(page, timeout_ms=45_000)
+ page.get_by_text(re.compile(r"Found \d+ models?\.", re.IGNORECASE)).first.wait_for(timeout=45_000)
+ selected = page.get_by_label("Model name").input_value()
+ print(f" model selected: {selected}")
+ except Exception as exc: # noqa: BLE001 — screenshot setup degrades to configured default
+ print(f" WARNING: live model discovery failed; using configured default ({exc})")
+ click_tab(page, "Upload")
+
+
+def prepare_documentation_api_key(page: Page) -> None:
+ """Create a disposable example key through the real UI when the database is empty."""
+ click_tab(page, "API Keys")
+ if "README documentation key" not in page.inner_text("body"):
+ page.get_by_label("Key Name").fill("README documentation key")
+ page.get_by_role("button", name="Create Key", exact=True).click()
+ wait_for_streamlit_idle(page)
+ # A newly created secret is intentionally stored in session state and shown
+ # once. Reload to clear that one-time value and refresh the repository list.
+ page.reload(wait_until="networkidle")
+ page.wait_for_timeout(2_000)
+ click_tab(page, "API Keys")
+
+
def run_extract_analyze(page: Page, wait_for_llm: bool = False, timeout_s: int = 300) -> None:
"""Click the Extract & Analyze button and wait for the pipeline.
@@ -141,6 +178,11 @@ def run_extract_analyze(page: Page, wait_for_llm: bool = False, timeout_s: int =
"the server rejected the pcap path — pass a path the SERVER can see "
"(container-relative like data/sample.pcap when using Docker)"
)
+ # Record the real in-flight phase tracker while it is present. Once a run
+ # completes, the Progress tab intentionally returns to its idle guidance.
+ page.get_by_role("tab", name=re.compile("Progress", re.IGNORECASE)).click()
+ page.wait_for_timeout(3_000)
+ capture_and_redact(page, "02-progress.png", redact=True)
print(f" waiting for pipeline (up to {timeout_s}s)...")
deadline = time.time() + timeout_s
last_status = ""
@@ -159,8 +201,7 @@ def run_extract_analyze(page: Page, wait_for_llm: bool = False, timeout_s: int =
// Data-stages complete: Beacon / YARA / OSINT sections appear in UI,
// or we see the LLM phase's "Generating AI report" caption, or LLM done.
const dataStageDone =
- bodyText.match(/Generating AI report/i) ||
- bodyText.match(/Generating LLM report/i) ||
+ bodyText.match(/Generating (?:AI |LLM )?report/i) ||
bodyText.match(/OSINT enrichment complete/i) ||
bodyText.match(/Completed: LLM report/i) ||
bodyText.match(/Beacon candidates/i) ||
@@ -199,8 +240,8 @@ def run_extract_analyze(page: Page, wait_for_llm: bool = False, timeout_s: int =
# ---------------------------------------------------------------------------
-def collect_ip_boxes(page: Page) -> list[dict]:
- """Return [{x,y,w,h}, ...] for every visible IPv4 text on the page.
+def collect_sensitive_boxes(page: Page) -> list[dict]:
+ """Return pixel boxes for visible sensitive text on the page.
We walk text nodes, match IPv4 patterns, then use Range.getClientRects()
to get exact pixel coordinates in the page (which equal viewport pixels
@@ -208,29 +249,47 @@ def collect_ip_boxes(page: Page) -> list[dict]:
"""
return page.evaluate(
r"""() => {
- const ipRe = /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})\b/g;
+ const sensitiveRes = [
+ /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d{1,2})\b/g,
+ /[0-9A-Fa-f:]*:[0-9A-Fa-f:]+/g,
+ /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/gi,
+ /\bphk_[0-9a-f]{16,}\b/gi,
+ /(?:\/(?:Users|home)\/|[A-Z]:\\Users\\)[^\s\"'<>]+/gi,
+ ];
const boxes = [];
const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT);
let node;
while ((node = walker.nextNode())) {
const text = node.nodeValue;
if (!text) continue;
- const matches = [...text.matchAll(ipRe)];
- if (!matches.length) continue;
- for (const m of matches) {
- const range = document.createRange();
- range.setStart(node, m.index);
- range.setEnd(node, m.index + m[0].length);
- const rects = range.getClientRects();
- for (const r of rects) {
- if (r.width > 0 && r.height > 0) {
- boxes.push({
- x: Math.floor(r.left + window.scrollX),
- y: Math.floor(r.top + window.scrollY),
- w: Math.ceil(r.width),
- h: Math.ceil(r.height),
- text: m[0],
- });
+ const element = node.parentElement;
+ if (!element || !element.checkVisibility({checkOpacity: true, checkVisibilityCSS: true})) continue;
+ const panel = element.closest('[role="tabpanel"]');
+ if (panel && panel.getAttribute('aria-hidden') === 'true') continue;
+ for (const sensitiveRe of sensitiveRes) {
+ sensitiveRe.lastIndex = 0;
+ const matches = [...text.matchAll(sensitiveRe)];
+ for (const m of matches) {
+ // The broad IPv6 candidate matcher also sees timestamps.
+ // Require compressed notation or at least four colons.
+ if (sensitiveRe === sensitiveRes[1]) {
+ const colonCount = (m[0].match(/:/g) || []).length;
+ if (!m[0].includes('::') && colonCount < 4) continue;
+ }
+ const range = document.createRange();
+ range.setStart(node, m.index);
+ range.setEnd(node, m.index + m[0].length);
+ const rects = range.getClientRects();
+ for (const r of rects) {
+ if (r.width > 0 && r.height > 0) {
+ boxes.push({
+ x: Math.floor(r.left + window.scrollX),
+ y: Math.floor(r.top + window.scrollY),
+ w: Math.ceil(r.width),
+ h: Math.ceil(r.height),
+ text: m[0],
+ });
+ }
}
}
}
@@ -248,6 +307,19 @@ def _valid_ip_octet(s: str) -> bool:
return False
+def _valid_ipv6_values(text: str) -> list[str]:
+ """Return syntactically valid IPv6 values found in OCR text."""
+ values: list[str] = []
+ for match in IPV6_CANDIDATE_RE.finditer(text):
+ candidate = match.group(0)
+ try:
+ if ipaddress.ip_address(candidate).version == 6:
+ values.append(candidate)
+ except ValueError:
+ continue
+ return values
+
+
def _extract_ip_boxes_from_data(data: dict, scale: float = 1.0) -> list[dict]:
"""Three match strategies on one tesseract image_to_data result."""
boxes: list[dict] = []
@@ -263,9 +335,9 @@ def _box(i: int, text: str) -> dict:
"text": text,
}
- # Strategy 1: per-token strict IPv4
+ # Strategy 1: per-token strict IPv4/IPv6
for i, text in enumerate(data["text"]):
- if text and IPV4_RE.search(text):
+ if text and (IPV4_RE.search(text) or _valid_ipv6_values(text)):
boxes.append(_box(i, text))
# Strategy 2: 4-token sliding window on the same line
@@ -319,8 +391,10 @@ def ocr_ip_boxes(png_path: Path) -> list[dict]:
"""
try:
import pytesseract
- except ImportError:
- return []
+ except ImportError as exc:
+ raise RuntimeError(
+ "pytesseract is required for privacy-safe screenshot capture; install requirements-docs.txt"
+ ) from exc
img = Image.open(png_path)
img_2x = img.resize((img.width * 2, img.height * 2), Image.LANCZOS)
@@ -352,7 +426,7 @@ def redact_png(
dpr: float = 1.0,
run_ocr_fallback: bool = True,
) -> int:
- """Overlay each IP box with a solid black rectangle. Returns total redactions.
+ """Overlay each sensitive box with a solid rectangle. Returns total redactions.
First pass: uses the DOM-collected boxes (exact, fast).
Second pass: OCR the saved PNG to catch IPs in canvas-rendered grids
@@ -400,6 +474,23 @@ def _draw_box(x: int, y: int, w: int, h: int, pad_x: int = 2, pad_y: int = 1) ->
return total
+def audit_redacted_png(png_path: Path) -> list[str]:
+ """Return recognizable sensitive values that remain after redaction."""
+ try:
+ import pytesseract
+ except ImportError as exc:
+ raise RuntimeError(
+ "pytesseract is required for the final screenshot privacy audit; install requirements-docs.txt"
+ ) from exc
+
+ text = pytesseract.image_to_string(Image.open(png_path), config="--psm 11")
+ findings: list[str] = []
+ for pattern in (IPV4_RE, EMAIL_RE, API_KEY_RE, USER_PATH_RE):
+ findings.extend(match.group(0) for match in pattern.finditer(text))
+ findings.extend(_valid_ipv6_values(text))
+ return sorted(set(findings))
+
+
def crop_trailing_blank(
png_path: Path,
min_blank_run: int = 300,
@@ -447,13 +538,17 @@ def capture_and_redact(
dpr: float = 1.0,
) -> None:
"""Take a screenshot and immediately redact IPs in place."""
- # Collect IP bounding boxes BEFORE screenshot so the DOM is the same state
- boxes = collect_ip_boxes(page) if redact else []
+ # Collect sensitive bounding boxes BEFORE screenshot so the DOM is the same state.
+ boxes = collect_sensitive_boxes(page) if redact else []
path = save_screenshot(page, filename)
if redact:
n = redact_png(path, boxes, dpr=dpr, run_ocr_fallback=True)
- print(f" ✂ redacted {n} IP region(s) in {filename}")
+ print(f" ✂ redacted {n} sensitive region(s) in {filename}")
crop_trailing_blank(path)
+ if redact:
+ findings = audit_redacted_png(path)
+ if findings:
+ raise RuntimeError(f"sensitive text remains in {filename}: {', '.join(findings)}")
# ---------------------------------------------------------------------------
@@ -466,6 +561,11 @@ def main() -> int:
parser.add_argument("--base-url", default="http://localhost:8501")
parser.add_argument("--keep-ips", action="store_true", help="skip IP redaction")
parser.add_argument("--skip-analysis", action="store_true", help="don't upload/analyze (just snap empty tabs)")
+ parser.add_argument(
+ "--seed-docs-key",
+ action="store_true",
+ help="create a disposable README API key; use only with an isolated data bind",
+ )
parser.add_argument(
"--redact-only",
action="store_true",
@@ -516,6 +616,12 @@ def main() -> int:
# Streamlit's first render after networkidle still sometimes paints late
page.wait_for_timeout(2_000)
+ # Discover the model through the actual Config UI. This keeps the LLM
+ # screenshot tied to a functioning local provider instead of a mock or
+ # hard-coded model name.
+ if not args.skip_analysis:
+ configure_local_llm(page)
+
# 1. Upload tab (pre-analysis)
print("→ tab: Upload (pre-analysis)")
click_tab(page, "Upload")
@@ -526,6 +632,7 @@ def main() -> int:
for label, filename in (
("Config", "08-config.png"),
("Cases", "07-cases.png"),
+ ("API Keys", "11-api-keys.png"),
):
print(f"→ tab: {label} (empty state)")
click_tab(page, label)
@@ -572,8 +679,8 @@ def main() -> int:
# 3. Capture each tab
for label, filename in (
- ("Progress", "02-progress.png"),
("Dashboard", "03-dashboard.png"),
+ ("MITRE Analysis", "10-mitre-analysis.png"),
("LLM Analysis", "04-llm-analysis.png"),
("OSINT", "05-osint.png"),
("Raw Data", "06-raw-data.png"),
@@ -584,10 +691,20 @@ def main() -> int:
click_tab(page, label)
capture_and_redact(page, filename, redact=redact)
- # 4. LLM Integration close-up with the Anthropic provider selected.
+ # 4. Create an example key in the isolated documentation database via
+ # the UI, reload away the one-time secret, then capture management state.
+ print("→ tab: API Keys")
+ if args.seed_docs_key:
+ prepare_documentation_api_key(page)
+ else:
+ click_tab(page, "API Keys")
+ capture_and_redact(page, "11-api-keys.png", redact=redact)
+
+ # 5. LLM Integration close-up with the Anthropic provider selected.
# No IPs appear in this section, so no redaction pass is needed.
print("→ provider close-up: 09-llm-providers.png")
try:
+ click_tab(page, "Config")
page.get_by_text("Anthropic", exact=True).first.click()
page.wait_for_timeout(2_000)
heading = page.get_by_text("LLM Integration", exact=True).first
diff --git a/tests/api/test_feed_query.py b/tests/api/test_feed_query.py
index 3f0ed9b..f857042 100644
--- a/tests/api/test_feed_query.py
+++ b/tests/api/test_feed_query.py
@@ -58,6 +58,22 @@ def test_query_iocs_includes_case_ids_and_first_seen(tmp_path):
assert r["first_seen"] is not None
+def test_query_iocs_includes_persisted_mitre_techniques(tmp_path):
+ repo = CaseRepository(db_path=str(tmp_path / "t.db"))
+ repo.create_case(Case(id="mitrecase", title="t"))
+ analysis = Analysis(
+ case_id="mitrecase",
+ pcap_path="x.pcap",
+ attack_mapping={"techniques": [{"technique_id": "T1071.004"}, {"technique_id": "T1568.002"}]},
+ iocs=[IOC(ioc_type=IOCType.DOMAIN, value="dns.example")],
+ )
+ repo.save_analysis(analysis)
+
+ rows = query_iocs(repo, IOCFilter())
+
+ assert rows[0]["mitre_techniques"] == ["T1071.004", "T1568.002"]
+
+
def test_query_iocs_score_from_severity(tmp_path):
repo = _seed(tmp_path)
rows = query_iocs(repo, IOCFilter())
diff --git a/tests/api/test_health.py b/tests/api/test_health.py
index 195967f..83173fc 100644
--- a/tests/api/test_health.py
+++ b/tests/api/test_health.py
@@ -39,11 +39,41 @@ def test_readyz_returns_200_when_ready(client):
assert r.status_code == 200
+def test_readyz_does_not_initialize_job_queue(client):
+ from app.api.deps import get_queue
+
+ assert get_queue.cache_info().currsize == 0
+ r = client.get("/readyz")
+ assert r.status_code == 200
+ assert get_queue.cache_info().currsize == 0
+
+
def test_healthz_no_auth_required(client):
r = client.get("/healthz", headers={"Authorization": "Bearer wrong"})
assert r.status_code == 200
+def test_openapi_reports_package_version(client):
+ from app import __version__
+
+ response = client.get("/api/v1/openapi.json")
+
+ assert response.status_code == 200
+ assert response.json()["info"]["version"] == __version__ == "2.0.0"
+
+
+def test_openapi_operation_ids_are_unique(client):
+ schema = client.get("/api/v1/openapi.json").json()
+ operation_ids = [
+ operation["operationId"]
+ for path in schema["paths"].values()
+ for operation in path.values()
+ if isinstance(operation, dict) and "operationId" in operation
+ ]
+
+ assert len(operation_ids) == len(set(operation_ids))
+
+
def test_request_id_header_echoed(client):
r = client.get("/healthz", headers={"X-Request-ID": "abc-123"})
assert r.headers.get("X-Request-ID") == "abc-123"
diff --git a/tests/api/test_key_repository.py b/tests/api/test_key_repository.py
index 39b029c..0d29e8b 100644
--- a/tests/api/test_key_repository.py
+++ b/tests/api/test_key_repository.py
@@ -100,11 +100,12 @@ def test_increment_usage_upsert(tmp_path):
repo = _repo(tmp_path)
_, key = _make_key()
kid = repo.create_key(key)
- repo.increment_usage(kid, "2026-05-23", 5)
- repo.increment_usage(kid, "2026-05-23", 3)
+ usage_date = (datetime.now() - timedelta(days=1)).strftime("%Y-%m-%d")
+ repo.increment_usage(kid, usage_date, 5)
+ repo.increment_usage(kid, usage_date, 3)
usage = repo.get_usage(kid, days=30)
# Should have one entry with 8 total
- day_data = [u for u in usage if u["date"] == "2026-05-23"]
+ day_data = [u for u in usage if u["date"] == usage_date]
assert len(day_data) == 1
assert day_data[0]["requests"] == 8
# total_requests on key should also be 8
@@ -116,13 +117,15 @@ def test_get_usage_returns_daily_data(tmp_path):
repo = _repo(tmp_path)
_, key = _make_key()
kid = repo.create_key(key)
- repo.increment_usage(kid, "2026-05-22", 10)
- repo.increment_usage(kid, "2026-05-23", 20)
+ yesterday = (datetime.now() - timedelta(days=2)).strftime("%Y-%m-%d")
+ recent = (datetime.now() - timedelta(days=1)).strftime("%Y-%m-%d")
+ repo.increment_usage(kid, yesterday, 10)
+ repo.increment_usage(kid, recent, 20)
usage = repo.get_usage(kid, days=30)
assert len(usage) == 2
dates = [u["date"] for u in usage]
- assert "2026-05-22" in dates
- assert "2026-05-23" in dates
+ assert yesterday in dates
+ assert recent in dates
def test_get_usage_summary(tmp_path):
@@ -131,10 +134,11 @@ def test_get_usage_summary(tmp_path):
_, k2 = _make_key(name="k2")
id1 = repo.create_key(k1)
id2 = repo.create_key(k2)
- repo.increment_usage(id1, "2026-05-23", 10)
- repo.increment_usage(id2, "2026-05-23", 5)
+ usage_date = (datetime.now() - timedelta(days=1)).strftime("%Y-%m-%d")
+ repo.increment_usage(id1, usage_date, 10)
+ repo.increment_usage(id2, usage_date, 5)
summary = repo.get_usage_summary(days=30)
- day_data = [s for s in summary if s["date"] == "2026-05-23"]
+ day_data = [s for s in summary if s["date"] == usage_date]
assert len(day_data) == 1
assert day_data[0]["requests"] == 15
diff --git a/tests/api/test_pcaps.py b/tests/api/test_pcaps.py
index 23c8d21..cf325f7 100644
--- a/tests/api/test_pcaps.py
+++ b/tests/api/test_pcaps.py
@@ -3,6 +3,7 @@
from __future__ import annotations
+import os
import pathlib
import pytest
@@ -40,7 +41,7 @@ def test_post_pcap_returns_202_and_ids(client):
"/api/v1/pcaps",
headers={"Authorization": "Bearer MAIN"},
files={"pcap": ("tiny.pcap", f, "application/vnd.tcpdump.pcap")},
- data={"name": "smoke", "osint_enabled": "false", "llm_enabled": "false"},
+ data={"name": "smoke", "osint_enabled": "false"},
)
assert r.status_code == 202, r.text
body = r.json()
@@ -77,6 +78,19 @@ def test_post_with_invalid_magic_returns_415(client):
assert r.status_code == 415
+def test_invalid_magic_does_not_initialize_queue(client, monkeypatch):
+ def fail_if_called():
+ raise AssertionError("queue should not be initialized before upload validation")
+
+ monkeypatch.setattr("app.api.routers.pcaps.get_queue", fail_if_called)
+ r = client.post(
+ "/api/v1/pcaps",
+ headers={"Authorization": "Bearer MAIN"},
+ files={"pcap": ("not.pcap", b"PK\x03\x04" + b"\x00" * 200)},
+ )
+ assert r.status_code == 415
+
+
@pytest.mark.skipif(not FIXTURE.exists(), reason="fixture missing")
def test_post_returns_503_when_queue_full(client, monkeypatch):
monkeypatch.setenv("PCAP_HUNTER_API_QUEUE_DEPTH", "1")
@@ -96,7 +110,58 @@ def test_post_returns_503_when_queue_full(client, monkeypatch):
"/api/v1/pcaps",
headers={"Authorization": "Bearer MAIN"},
files={"pcap": ("tiny.pcap", f, "application/vnd.tcpdump.pcap")},
- data={"osint_enabled": "false", "llm_enabled": "false"},
+ data={"osint_enabled": "false"},
)
assert r.status_code == 503
assert r.headers.get("Retry-After") == "60"
+ assert len(repo.list_cases()) == 1
+ uploads_dir = pathlib.Path(os.environ["PCAP_HUNTER_API_UPLOADS_DIR"])
+ assert not list(uploads_dir.glob("*.pcap"))
+
+
+@pytest.mark.skipif(not FIXTURE.exists(), reason="fixture missing")
+def test_queue_initialization_failure_cleans_upload_and_case(client, monkeypatch):
+ def boom():
+ raise RuntimeError("queue init failed")
+
+ monkeypatch.setattr("app.api.routers.pcaps.get_queue", boom)
+
+ with FIXTURE.open("rb") as f:
+ with pytest.raises(RuntimeError, match="queue init failed"):
+ client.post(
+ "/api/v1/pcaps",
+ headers={"Authorization": "Bearer MAIN"},
+ files={"pcap": ("tiny.pcap", f, "application/vnd.tcpdump.pcap")},
+ data={"osint_enabled": "false"},
+ )
+
+ from app.api.deps import get_repo
+
+ assert get_repo().list_cases() == []
+ uploads_dir = pathlib.Path(os.environ["PCAP_HUNTER_API_UPLOADS_DIR"])
+ assert not list(uploads_dir.glob("*.pcap"))
+
+
+@pytest.mark.skipif(not FIXTURE.exists(), reason="fixture missing")
+def test_case_persistence_failure_cleans_upload(client, monkeypatch):
+ from app.database.repository import CaseRepository
+
+ def boom(self, case):
+ raise RuntimeError("case save failed")
+
+ monkeypatch.setattr(CaseRepository, "create_case", boom)
+
+ with FIXTURE.open("rb") as f:
+ with pytest.raises(RuntimeError, match="case save failed"):
+ client.post(
+ "/api/v1/pcaps",
+ headers={"Authorization": "Bearer MAIN"},
+ files={"pcap": ("tiny.pcap", f, "application/vnd.tcpdump.pcap")},
+ data={"osint_enabled": "false"},
+ )
+
+ from app.api.deps import get_repo
+
+ assert get_repo().list_cases() == []
+ uploads_dir = pathlib.Path(os.environ["PCAP_HUNTER_API_UPLOADS_DIR"])
+ assert not list(uploads_dir.glob("*.pcap"))
diff --git a/tests/api/test_queue.py b/tests/api/test_queue.py
index cc9985b..75de3da 100644
--- a/tests/api/test_queue.py
+++ b/tests/api/test_queue.py
@@ -12,7 +12,6 @@
import app.api.queue as queue_mod
from app.api.queue import (
- WARNING_LLM_UNSUPPORTED,
WARNING_OSINT_NOT_CONFIGURED,
WARNING_PERSISTENCE_FAILED,
WARNING_YARA_FAILED,
@@ -60,7 +59,6 @@ def test_inprocess_queue_runs_a_job(tmp_path):
pcap_path=str(FIXTURE_PCAP),
options={
"osint_enabled": False,
- "llm_enabled": False,
"do_yara": False,
"do_carve": False,
"pyshark_packet_limit": 50,
@@ -164,7 +162,7 @@ def test_worker_persists_analysis_and_iocs(tmp_path):
repo.create_case(Case(id="cafe0001", title="persist-test", status=CaseStatus.IN_PROGRESS, severity=Severity.LOW))
job_id = repo.create_job(Job(case_id="cafe0001", pcap_path=str(FIXTURE_PCAP), options_json="{}"))
- _worker_run(job_id, db, str(FIXTURE_PCAP), {"osint_enabled": False, "llm_enabled": False})
+ _worker_run(job_id, db, str(FIXTURE_PCAP), {"osint_enabled": False})
job = repo.get_job(job_id)
assert job.status.value == "done"
@@ -203,7 +201,7 @@ def fake_enrich(arts, keys, phase=None):
repo.create_case(Case(id="cafe0002", title="osint-test", status=CaseStatus.IN_PROGRESS, severity=Severity.LOW))
job_id = repo.create_job(Job(case_id="cafe0002", pcap_path=str(FIXTURE_PCAP), options_json="{}"))
- _worker_run(job_id, db, str(FIXTURE_PCAP), {"osint_enabled": True, "llm_enabled": False})
+ _worker_run(job_id, db, str(FIXTURE_PCAP), {"osint_enabled": True})
job = repo.get_job(job_id)
result = json.loads(job.result_json)
@@ -222,26 +220,12 @@ def test_worker_warns_when_osint_enabled_but_unconfigured(tmp_path, monkeypatch)
repo.create_case(Case(id="cafe0003", title="t", status=CaseStatus.IN_PROGRESS, severity=Severity.LOW))
job_id = repo.create_job(Job(case_id="cafe0003", pcap_path=str(FIXTURE_PCAP), options_json="{}"))
- _worker_run(job_id, db, str(FIXTURE_PCAP), {"osint_enabled": True, "llm_enabled": False})
+ _worker_run(job_id, db, str(FIXTURE_PCAP), {"osint_enabled": True})
result = json.loads(repo.get_job(job_id).result_json)
assert WARNING_OSINT_NOT_CONFIGURED in result["warnings"]
-@pytest.mark.skipif(not FIXTURE_PCAP.exists(), reason="tiny.pcap fixture missing")
-def test_worker_flags_llm_as_unsupported(tmp_path):
- db = str(tmp_path / "t.db")
- repo = CaseRepository(db_path=db)
- repo.create_case(Case(id="cafe0004", title="t", status=CaseStatus.IN_PROGRESS, severity=Severity.LOW))
- job_id = repo.create_job(Job(case_id="cafe0004", pcap_path=str(FIXTURE_PCAP), options_json="{}"))
-
- _worker_run(job_id, db, str(FIXTURE_PCAP), {"osint_enabled": False, "llm_enabled": True})
-
- result = json.loads(repo.get_job(job_id).result_json)
- assert WARNING_LLM_UNSUPPORTED in result["warnings"]
- assert result["summary_narrative"] is None
-
-
@pytest.mark.skipif(not FIXTURE_PCAP.exists(), reason="tiny.pcap fixture missing")
def test_worker_keeps_job_done_when_persistence_fails(tmp_path, monkeypatch):
"""save_analysis raising must not fail the job: warn + analysis_id stays None."""
@@ -256,7 +240,7 @@ def boom(self, analysis):
repo.create_case(Case(id="cafe0005", title="t", status=CaseStatus.IN_PROGRESS, severity=Severity.LOW))
job_id = repo.create_job(Job(case_id="cafe0005", pcap_path=str(FIXTURE_PCAP), options_json="{}"))
- _worker_run(job_id, db, str(FIXTURE_PCAP), {"osint_enabled": False, "llm_enabled": False})
+ _worker_run(job_id, db, str(FIXTURE_PCAP), {"osint_enabled": False})
job = repo.get_job(job_id)
assert job.status == JobStatus.DONE
@@ -281,6 +265,7 @@ def fake_run_pipeline(pcap_path, case_id, options, progress, heartbeat=None):
"artifacts": {"ips": ["10.0.0.1", "8.8.8.8"], "domains": [], "urls": [], "hashes": [], "ja3": []},
},
beacon_df_records=list(records),
+ zeek_tables={"conn": [{"uid": "x"}]},
)
# _worker_run imports run_pipeline from app.pipeline.runner at call time,
@@ -295,10 +280,11 @@ def fake_run_pipeline(pcap_path, case_id, options, progress, heartbeat=None):
repo.create_case(Case(id="cafe0006", title="t", status=CaseStatus.IN_PROGRESS, severity=Severity.LOW))
job_id = repo.create_job(Job(case_id="cafe0006", pcap_path=str(fake_pcap), options_json="{}"))
- _worker_run(job_id, db, str(fake_pcap), {"osint_enabled": False, "llm_enabled": False})
+ _worker_run(job_id, db, str(fake_pcap), {"osint_enabled": False})
result = json.loads(repo.get_job(job_id).result_json)
assert result["analysis_id"], "persistence must succeed with the faked pipeline result"
+ assert result["capture_metrics"]["detectors"]["zeek"] == "available"
persisted = repo.get_analysis(result["analysis_id"])
assert persisted.features["beacon_records"] == records
@@ -315,7 +301,7 @@ def test_done_job_reports_complete_progress(tmp_path):
repo.create_case(Case(id="cafe0005", title="t", status=CaseStatus.IN_PROGRESS, severity=Severity.LOW))
job_id = repo.create_job(Job(case_id="cafe0005", pcap_path=str(FIXTURE_PCAP), options_json="{}"))
- _worker_run(job_id, db, str(FIXTURE_PCAP), {"osint_enabled": False, "llm_enabled": False})
+ _worker_run(job_id, db, str(FIXTURE_PCAP), {"osint_enabled": False})
job = repo.get_job(job_id)
assert job.status.value == "done"
@@ -362,7 +348,7 @@ def fake_scan(items, rules_dirs=None):
repo.create_case(Case(id="cafe0010", title="t", status=CaseStatus.IN_PROGRESS, severity=Severity.LOW))
job_id = repo.create_job(Job(case_id="cafe0010", pcap_path=str(fake_pcap), options_json="{}"))
- _worker_run(job_id, db, str(fake_pcap), {"osint_enabled": False, "llm_enabled": False, "do_yara": True})
+ _worker_run(job_id, db, str(fake_pcap), {"osint_enabled": False, "do_yara": True})
result = json.loads(repo.get_job(job_id).result_json)
assert "yara_scan" in result["stages_run"], "yara_scan must be recorded in stages_run"
@@ -403,7 +389,7 @@ def fake_scan_boom(items, rules_dirs=None):
repo.create_case(Case(id="cafe0011", title="t", status=CaseStatus.IN_PROGRESS, severity=Severity.LOW))
job_id = repo.create_job(Job(case_id="cafe0011", pcap_path=str(fake_pcap), options_json="{}"))
- _worker_run(job_id, db, str(fake_pcap), {"osint_enabled": False, "llm_enabled": False, "do_yara": True})
+ _worker_run(job_id, db, str(fake_pcap), {"osint_enabled": False, "do_yara": True})
job = repo.get_job(job_id)
assert job.status.value == "done", "job must remain done despite YARA failure"
@@ -452,7 +438,7 @@ def test_worker_skips_cancelled_job(tmp_path, monkeypatch):
job_id = repo.create_job(Job(case_id="cafe0020", pcap_path=str(fake_pcap), options_json="{}"))
assert cancel_queued_job(repo, job_id) is True
- _worker_run(job_id, db, str(fake_pcap), {"osint_enabled": False, "llm_enabled": False})
+ _worker_run(job_id, db, str(fake_pcap), {"osint_enabled": False})
job = repo.get_job(job_id)
assert job.status == JobStatus.CANCELLED, f"cancelled job must stay cancelled, got {job.status.value}"
@@ -477,7 +463,7 @@ def test_worker_skips_deleted_job(tmp_path, monkeypatch):
job_id = repo.create_job(Job(case_id="cafe0021", pcap_path=str(fake_pcap), options_json="{}"))
assert repo.delete_case("cafe0021") is True
- _worker_run(job_id, db, str(fake_pcap), {"osint_enabled": False, "llm_enabled": False})
+ _worker_run(job_id, db, str(fake_pcap), {"osint_enabled": False})
assert repo.get_job(job_id) is None, "the explicitly-deleted job row must stay gone"
assert "ran" not in calls, "pipeline must not run for a deleted job"
diff --git a/tests/test_carve_streaming.py b/tests/test_carve_streaming.py
index 2491ad8..e5b31a9 100644
--- a/tests/test_carve_streaming.py
+++ b/tests/test_carve_streaming.py
@@ -159,6 +159,36 @@ def test_carve_malformed_lines_skipped(self, tmp_path):
assert len(results) == 1
assert results[0]["sha256"] == hashlib.sha256(b"hello").hexdigest()
+ def test_carve_decodes_compact_hex_file_data(self, tmp_path):
+ payload = b"\x00\xffABC"
+ line = f"1700000000.0\t3\tapplication/octet-stream\t5\t{payload.hex()}\n"
+ fake = FakePopen([line])
+ with (
+ patch("app.utils.common.find_bin", return_value="/usr/bin/tshark"),
+ patch("subprocess.Popen", return_value=fake),
+ ):
+ results = carve_http_payloads("dummy.pcap", str(tmp_path))
+
+ expected_sha = hashlib.sha256(payload).hexdigest()
+ carved = tmp_path / f"stream3_{expected_sha[:10]}.bin"
+ assert results[0]["sha256"] == expected_sha
+ assert carved.read_bytes() == payload
+
+ def test_carve_decodes_colon_separated_hex_file_data(self, tmp_path):
+ payload = b"hello"
+ line = "1700000000.0\t3\ttext/plain\t5\t68:65:6c:6c:6f\n"
+ fake = FakePopen([line])
+ with (
+ patch("app.utils.common.find_bin", return_value="/usr/bin/tshark"),
+ patch("subprocess.Popen", return_value=fake),
+ ):
+ results = carve_http_payloads("dummy.pcap", str(tmp_path))
+
+ expected_sha = hashlib.sha256(payload).hexdigest()
+ carved = tmp_path / f"stream3_{expected_sha[:10]}.bin"
+ assert results[0]["sha256"] == expected_sha
+ assert carved.read_bytes() == payload
+
def test_carve_timeout_raises_carve_error(self, tmp_path, monkeypatch):
monkeypatch.setattr("app.config.CARVE_TIMEOUT_SECONDS", 0.05)
fake = FakePopen([CARVE_LINE, CARVE_LINE], line_delay=0.2)
diff --git a/tests/test_case_management.py b/tests/test_case_management.py
index 9c9c117..7ceed55 100644
--- a/tests/test_case_management.py
+++ b/tests/test_case_management.py
@@ -165,6 +165,27 @@ def test_to_dict(self):
assert d["packet_count"] == 1000
assert len(d["iocs"]) == 1
+ def test_to_dict_and_repository_round_trip_new_analytic_fields(self, tmp_path):
+ analysis = Analysis(
+ case_id="CASE-001",
+ pcap_path="/tmp/test.pcap",
+ attack_mapping={"attack_version": "19.1", "techniques": [{"technique_id": "T1571"}]},
+ capture_metrics={"flow_count": 4, "visibility_gaps": ["zeek"]},
+ )
+ payload = analysis.to_dict()
+ restored = Analysis.from_dict(payload)
+
+ assert restored.attack_mapping["attack_version"] == "19.1"
+ assert restored.capture_metrics["flow_count"] == 4
+
+ repo = CaseRepository(db_path=str(tmp_path / "cases.db"))
+ analysis_id = repo.save_analysis(analysis)
+ persisted = repo.get_analysis(analysis_id)
+
+ assert persisted is not None
+ assert persisted.attack_mapping["techniques"][0]["technique_id"] == "T1571"
+ assert persisted.capture_metrics["visibility_gaps"] == ["zeek"]
+
def test_from_dict(self):
data = {
"id": "ANL-002",
@@ -420,6 +441,27 @@ def test_save_analysis_with_iocs(self, repo):
retrieved = repo.get_analysis(analysis_id)
assert len(retrieved.iocs) == 2
+ def test_save_analysis_replaces_existing_iocs(self, repo):
+ """Saving an existing analysis ID must not leave stale IOC rows."""
+ case_id = repo.create_case(Case(title="Replace IOCs"))
+ analysis = Analysis(
+ id="analysis-1",
+ case_id=case_id,
+ pcap_path="/test.pcap",
+ iocs=[
+ IOC(ioc_type=IOCType.IP, value="1.2.3.4"),
+ IOC(ioc_type=IOCType.DOMAIN, value="old.example"),
+ ],
+ )
+ repo.save_analysis(analysis)
+
+ analysis.iocs = [IOC(ioc_type=IOCType.HASH, value="ab" * 32)]
+ repo.save_analysis(analysis)
+
+ retrieved = repo.get_analysis("analysis-1")
+ assert [ioc.value for ioc in retrieved.iocs] == ["ab" * 32]
+ assert retrieved.iocs[0].ioc_type == IOCType.HASH
+
def test_search_iocs(self, repo):
"""Test IOC search."""
case_id = repo.create_case(Case(title="IOC Search Test"))
diff --git a/tests/test_config_manager.py b/tests/test_config_manager.py
index cd624a7..9a0163f 100644
--- a/tests/test_config_manager.py
+++ b/tests/test_config_manager.py
@@ -2,6 +2,7 @@
import pytest
+from app import config as C
from app.utils.config_manager import DEFAULT_CONFIG, SENSITIVE_KEYS, ConfigManager
@@ -18,6 +19,11 @@ def config_manager(temp_config_file):
class TestConfigManagerBasic:
+ def test_llm_defaults_match_runtime_configuration(self):
+ """Fresh installs use the same LM Studio endpoint and model as the runtime."""
+ assert DEFAULT_CONFIG["cfg_llm_endpoint"] == C.LM_BASE_URL
+ assert DEFAULT_CONFIG["cfg_llm_model"] == C.LM_MODEL
+
def test_load_missing_file(self, config_manager):
"""Loading non-existent file returns defaults."""
config = config_manager.load()
diff --git a/tests/test_llm_client.py b/tests/test_llm_client.py
index cd0bbc1..1041bf9 100644
--- a/tests/test_llm_client.py
+++ b/tests/test_llm_client.py
@@ -410,6 +410,20 @@ def test_existing_custom_path_unchanged(self):
assert _normalize_base_url("http://h:1234/api/v0") == "http://h:1234/api/v0"
+ def test_docker_local_compatible_rewrites_host_lan_ip(self):
+ from app.llm.client import _normalize_base_url
+
+ with patch.dict("os.environ", {"PCAP_HUNTER_DOCKER_HOST_FALLBACK": "1"}):
+ assert _normalize_base_url("http://192.168.2.114:1234", local_compatible=True) == (
+ "http://host.docker.internal:1234/v1"
+ )
+
+ def test_cloud_endpoint_is_not_rewritten_by_local_fallback(self):
+ from app.llm.client import _normalize_base_url
+
+ with patch.dict("os.environ", {"PCAP_HUNTER_DOCKER_HOST_FALLBACK": "1"}):
+ assert _normalize_base_url("https://10.0.0.9:9000/v1") == "https://10.0.0.9:9000/v1"
+
def test_empty_string_passes_through(self):
from app.llm.client import _normalize_base_url
diff --git a/tests/test_llm_providers.py b/tests/test_llm_providers.py
index 90f7b1d..05262f2 100644
--- a/tests/test_llm_providers.py
+++ b/tests/test_llm_providers.py
@@ -287,6 +287,7 @@ def test_probe_lmstudio_delegates_to_test_connection(self):
P.PROVIDER_LMSTUDIO, base_url="http://localhost:1234", api_key="lm", model="local"
)
tc.assert_called_once()
+ self.assertTrue(tc.call_args.kwargs["local_compatible"])
self.assertTrue(ok)
def test_probe_openai_blank_url_uses_default_endpoint(self):
@@ -294,6 +295,7 @@ def test_probe_openai_blank_url_uses_default_endpoint(self):
ok, _ = P.probe_provider(P.PROVIDER_OPENAI, base_url="", api_key="sk", model="gpt-4o")
# Probe still hits the real default endpoint instead of erroring on no URL
self.assertEqual(tc.call_args.args[0], "https://api.openai.com/v1")
+ self.assertFalse(tc.call_args.kwargs["local_compatible"])
self.assertTrue(ok)
def test_probe_openai_error_returns_false(self):
diff --git a/tests/test_mitre_page.py b/tests/test_mitre_page.py
new file mode 100644
index 0000000..113e700
--- /dev/null
+++ b/tests/test_mitre_page.py
@@ -0,0 +1,75 @@
+"""Tests for the dedicated MITRE ATT&CK workspace helpers."""
+
+from __future__ import annotations
+
+import pandas as pd
+
+from app.threat_intel.attack_mapping import AttackMapping, TechniqueMatch
+from app.ui.mitre_page import _coerce_mapping, build_attack_mapping, build_visibility_rows
+
+
+def test_build_attack_mapping_uses_production_shaped_session_state():
+ state = {
+ "features": {
+ "flows": [{"src": "10.0.0.1", "dst": "203.0.113.5", "count": 20, "bytes": 12_000_000}],
+ "artifacts": {"ips": ["10.0.0.1", "203.0.113.5"], "ja3": []},
+ },
+ "beacon_df": pd.DataFrame([{"src": "10.0.0.1", "dst": "203.0.113.5", "score": 0.85}]),
+ "dns_analysis": {"alerts": {"dga_count": 0, "tunneling_count": 0, "fast_flux_count": 0}},
+ "tls_analysis": {"certificates": [], "alerts": {}},
+ "yara_results": {"by_severity": {}, "results": []},
+ "osint": {},
+ }
+
+ mapping = build_attack_mapping(state)
+
+ assert isinstance(mapping, AttackMapping)
+ assert mapping.techniques
+ assert any(tech.technique_id == "T1071.001" for tech in mapping.techniques)
+
+
+def test_build_attack_mapping_returns_none_before_analysis():
+ assert build_attack_mapping({"features": None}) is None
+
+
+def test_attack_mapping_round_trips_for_persisted_state():
+ original = AttackMapping(
+ techniques=[TechniqueMatch("T1071.001", "Web Protocols", "command-and-control", 0.8, ["HTTP"])],
+ tactics_summary={"command-and-control": 1},
+ kill_chain_phase="command-and-control",
+ overall_severity="high",
+ )
+
+ restored = AttackMapping.from_dict(original.to_dict())
+
+ assert restored.to_dict() == original.to_dict()
+
+
+def test_coerce_mapping_restores_persisted_dictionary():
+ restored = _coerce_mapping(
+ {
+ "techniques": [
+ {
+ "technique_id": "T1571",
+ "technique_name": "Non-Standard Port",
+ "tactic": "command-and-control",
+ "confidence": 0.7,
+ "disposition": "confirmed",
+ "analyst_note": "Validated against flow evidence.",
+ }
+ ]
+ }
+ )
+
+ assert isinstance(restored, AttackMapping)
+ assert restored.techniques[0].disposition == "confirmed"
+ assert restored.techniques[0].analyst_note == "Validated against flow evidence."
+
+
+def test_visibility_rows_explicitly_mark_unavailable_detectors():
+ rows = build_visibility_rows({"features": None, "zeek_tables": {}, "correlations": None})
+
+ statuses = {row["Detector"]: row["Status"] for row in rows}
+ assert statuses["Packet / flow telemetry"] == "unavailable"
+ assert statuses["Zeek protocol telemetry"] == "unavailable"
+ assert statuses["Cross-indicator correlation"] == "unavailable"
diff --git a/tests/test_phase4.py b/tests/test_phase4.py
index f84316d..f1ad479 100644
--- a/tests/test_phase4.py
+++ b/tests/test_phase4.py
@@ -74,6 +74,26 @@ def test_to_dict(self):
assert d["confidence"] == 0.7
assert len(d["evidence"]) == 2
+ def test_to_dict_preserves_analytic_context_and_disposition(self):
+ tech = TechniqueMatch(
+ "T1571",
+ "Non-Standard Port",
+ "command-and-control",
+ 0.6,
+ ["port 31337"],
+ analytic_id="DET0227",
+ data_components=["Network Traffic Flow"],
+ limitations=["Needs protocol context"],
+ disposition="confirmed",
+ analyst_note="Validated against the flow table.",
+ )
+
+ payload = tech.to_dict()
+
+ assert payload["analytic_id"] == "DET0227"
+ assert payload["disposition"] == "confirmed"
+ assert payload["analyst_note"] == "Validated against the flow table."
+
class TestAttackMapping:
"""Test AttackMapping dataclass."""
@@ -150,6 +170,35 @@ def test_self_signed_cert_detection(self):
mapping = mapper.map_analysis(tls_analysis=tls_analysis)
technique_ids = [t.technique_id for t in mapping.techniques]
assert "T1573.002" in technique_ids
+ assert "T1587.003" not in technique_ids
+
+ def test_mapping_attaches_detection_context(self):
+ mapping = ATTACKMapper().map_analysis(
+ dns_analysis={"alerts": {"dga_count": 1}, "dga_detections": [{"domain": "x.test"}]},
+ beacon_results=[{"dst": "203.0.113.5", "score": 0.9, "proto": "https"}],
+ )
+
+ web = next(tech for tech in mapping.techniques if tech.technique_id == "T1071.001")
+ dga = next(tech for tech in mapping.techniques if tech.technique_id == "T1568.002")
+ assert web.analytic_id == "DET0027"
+ assert "Network Traffic: Web Protocols" in web.data_components
+ assert dga.limitations
+
+ def test_generic_beacon_does_not_claim_web_analytic_coverage(self):
+ mapping = ATTACKMapper().map_analysis(beacon_results=[{"dst": "203.0.113.5", "score": 0.9}])
+
+ web = next(tech for tech in mapping.techniques if tech.technique_id == "T1071.001")
+
+ assert web.analytic_id is None
+
+ def test_configured_c2_suspect_port_is_mapped_with_flow_evidence(self):
+ mapping = ATTACKMapper().map_analysis(
+ features={"flows": [{"dst": "203.0.113.5", "dport": "31337", "count": 5, "bytes": 5000}]}
+ )
+
+ port_match = next(tech for tech in mapping.techniques if tech.technique_id == "T1571")
+ assert "31337" in port_match.evidence[0]
+ assert port_match.analytic_id == "DET0227"
def test_yara_match_detection(self):
mapper = ATTACKMapper()
@@ -493,6 +542,8 @@ def test_export_layer(self, sample_mapping):
layer = export_navigator_layer(sample_mapping)
assert layer["name"] == "PCAP Analysis"
assert layer["domain"] == "enterprise-attack"
+ assert layer["versions"]["attack"] == "19.1"
+ assert layer["versions"]["navigator"] == "5.3.2"
assert len(layer["techniques"]) == 2
def test_export_layer_technique_format(self, sample_mapping):
diff --git a/tests/test_pipeline_runner.py b/tests/test_pipeline_runner.py
index 8f349cf..c842c06 100644
--- a/tests/test_pipeline_runner.py
+++ b/tests/test_pipeline_runner.py
@@ -53,6 +53,8 @@ def test_pipeline_result_to_dict_is_json_serializable():
restored = json.loads(serialized)
assert restored["analysis_id"] == "def67890"
assert restored["mitre_techniques"] == ["T1071.001", "T1568.002"]
+ assert "attack_mapping" in restored
+ assert "capture_metrics" in restored
assert restored["summary_narrative"] == "A short narrative."
assert restored["dns_analysis"]["dga_count"] == 3
assert restored["tls_analysis"]["certs"][0]["subject"] == "evil.example"
diff --git a/tests/test_upload.py b/tests/test_upload.py
new file mode 100644
index 0000000..fd6db97
--- /dev/null
+++ b/tests/test_upload.py
@@ -0,0 +1,52 @@
+"""Tests for Streamlit upload helpers."""
+
+from __future__ import annotations
+
+import io
+
+import pytest
+
+from app.ui.upload import CHUNK_SIZE, UploadValidationError, save_uploaded_pcaps
+
+
+class FakeUpload(io.BytesIO):
+ def __init__(self, name: str, payload: bytes):
+ super().__init__(payload)
+ self.name = name
+ self.read_sizes: list[int] = []
+
+ def read(self, size: int = -1) -> bytes:
+ self.read_sizes.append(size)
+ return super().read(size)
+
+
+def test_save_uploaded_pcaps_streams_chunks_and_preserves_suffix(tmp_path):
+ payload = b"\x0a\x0d\x0d\x0a" + b"x" * (CHUNK_SIZE + 10)
+ upload = FakeUpload("sample.pcapng", payload)
+
+ saved = save_uploaded_pcaps([upload], tmp_path, timestamp=123, run_id="abc12345")
+
+ assert saved[0].original_name == "sample.pcapng"
+ assert saved[0].path.endswith("upload_123_abc12345_0.pcapng")
+ assert saved[0].size_bytes == len(payload)
+ assert upload.read_sizes[0] == CHUNK_SIZE
+ assert (tmp_path / "upload_123_abc12345_0.pcapng").read_bytes() == payload
+
+
+def test_save_uploaded_pcaps_rejects_invalid_magic_and_removes_file(tmp_path):
+ upload = FakeUpload("bad.pcap", b"PK\x03\x04" + b"x" * 20)
+
+ with pytest.raises(UploadValidationError, match="not a valid PCAP"):
+ save_uploaded_pcaps([upload], tmp_path, timestamp=123)
+
+ assert not list(tmp_path.iterdir())
+
+
+def test_save_uploaded_pcaps_cleans_earlier_files_on_batch_failure(tmp_path):
+ good = FakeUpload("good.pcap", b"\xd4\xc3\xb2\xa1" + b"x" * 20)
+ bad = FakeUpload("bad.pcap", b"PK\x03\x04" + b"x" * 20)
+
+ with pytest.raises(UploadValidationError):
+ save_uploaded_pcaps([good, bad], tmp_path, timestamp=123)
+
+ assert not list(tmp_path.iterdir())
diff --git a/tests/test_visibility.py b/tests/test_visibility.py
new file mode 100644
index 0000000..f2f631e
--- /dev/null
+++ b/tests/test_visibility.py
@@ -0,0 +1,74 @@
+"""Tests for capture-quality and detector-visibility metrics."""
+
+from app.analysis.visibility import build_capture_metrics
+
+
+def test_capture_metrics_preserve_true_flow_totals_and_sample_cap():
+ metrics = build_capture_metrics(
+ {
+ "__total_pkts": 12,
+ "features": {
+ "flows": [
+ {
+ "src": "10.0.0.1",
+ "dst": "203.0.113.4",
+ "proto": "tcp",
+ "count": 10,
+ "bytes": 9000,
+ "first_ts": 100.0,
+ "last_ts": 110.0,
+ "pkt_times": [100.0, 110.0],
+ }
+ ],
+ "artifacts": {"ips": ["10.0.0.1", "203.0.113.4"], "domains": ["example.test"]},
+ },
+ "zeek_tables": {},
+ "pipeline_warnings": ["zeek_no_logs"],
+ }
+ )
+
+ assert metrics["packet_count"] == 12
+ assert metrics["parsed_packet_count"] == 10
+ assert metrics["parse_ratio"] == round(10 / 12, 4)
+ assert metrics["total_bytes"] == 9000
+ assert metrics["sampled_flow_count"] == 1
+ assert metrics["duration_seconds"] == 10.0
+ assert "zeek" in metrics["visibility_gaps"]
+ assert metrics["limitations"]
+
+
+def test_capture_metrics_mark_clean_empty_detectors_as_partial_not_absent():
+ metrics = build_capture_metrics(
+ {
+ "features": {"flows": [], "artifacts": {"ips": [], "domains": []}},
+ "zeek_tables": {"conn": []},
+ "dns_analysis": {"total_records": 0},
+ "tls_analysis": {"total_certificates": 0},
+ "yara_results": {},
+ "osint": {},
+ "correlations": [],
+ }
+ )
+
+ assert metrics["detectors"]["packet_flow"] == "available"
+ assert metrics["detectors"]["zeek"] == "available"
+ assert metrics["detectors"]["dns"] == "available"
+ assert metrics["detectors"]["correlation"] == "available"
+
+
+def test_capture_metrics_ignore_missing_or_malformed_timestamps():
+ metrics = build_capture_metrics(
+ {
+ "features": {
+ "flows": [
+ {"count": 1, "bytes": 20, "first_ts": None, "last_ts": None, "pkt_times": [None, "bad"]},
+ {"count": 2, "bytes": 40, "first_ts": 100.0, "last_ts": 105.0, "pkt_times": []},
+ ],
+ "artifacts": {},
+ }
+ }
+ )
+
+ assert metrics["first_seen"] == "1970-01-01T00:01:40+00:00"
+ assert metrics["last_seen"] == "1970-01-01T00:01:45+00:00"
+ assert metrics["duration_seconds"] == 5.0