Action attribution is limited to the newest safe processing window. Use narrower filters in the JSON API.
{% for row in overview.action_origin_counts %}
diff --git a/forensics/tests.py b/forensics/tests.py
index d3ee21c..f89ff91 100644
--- a/forensics/tests.py
+++ b/forensics/tests.py
@@ -95,6 +95,36 @@
DIGEST_A = "aa" * 32
DIGEST_B = "bb" * 32
+HEAVY_EVENT_SELECT_COLUMNS = {
+ field: f'"forensics_auditevent"."{field}"'
+ for field in (
+ "raw_line",
+ "raw_event",
+ "raw_kind",
+ "raw_context",
+ "context_human_action",
+ "context_transport",
+ "context_engine",
+ "context_group",
+ "context_convergence",
+ "context_source",
+ )
+}
+HEAVY_BULK_SELECT_COLUMNS = (
+ '"forensics_auditfile"."raw_text"',
+ *HEAVY_EVENT_SELECT_COLUMNS.values(),
+)
+
+
+def heavy_bulk_selects(captured_queries, *, allowed_columns=()):
+ prohibited_columns = set(HEAVY_BULK_SELECT_COLUMNS) - set(allowed_columns)
+ return [
+ query["sql"]
+ for query in captured_queries
+ if query["sql"].lstrip().upper().startswith("SELECT")
+ and any(column in query["sql"] for column in prohibited_columns)
+ ]
+
def audit_event(
seq,
@@ -1617,16 +1647,27 @@ def test_v2_upload_builds_audit_projections(self):
User.objects.create_user(username="analyst", password="correct horse battery staple")
self.client.login(username="analyst", password="correct horse battery staple")
- api_response = self.client.get(
- reverse("api-group-projections", kwargs={"slug": group.slug}),
- {"engine_id": ENGINE_ALICE},
- )
+ with CaptureQueriesContext(connection) as projection_api_queries:
+ api_response = self.client.get(
+ reverse("api-group-projections", kwargs={"slug": group.slug}),
+ {"engine_id": ENGINE_ALICE},
+ )
self.assertEqual(api_response.status_code, 200)
payload = api_response.json()
+ self.assertEqual(
+ heavy_bulk_selects(
+ projection_api_queries.captured_queries,
+ allowed_columns=(
+ HEAVY_EVENT_SELECT_COLUMNS["raw_kind"],
+ HEAVY_EVENT_SELECT_COLUMNS["context_source"],
+ ),
+ ),
+ [],
+ )
self.assertEqual(payload["schema_version"], "goggles-audit-projections/v1")
self.assertEqual(payload["filters"]["engine_id"], ENGINE_ALICE)
- self.assertEqual(payload["pagination"]["delivery_artifacts"]["limit"], 500)
+ self.assertEqual(payload["pagination"]["delivery_artifacts"]["limit"], 100)
self.assertEqual(
payload["delivery_artifacts"][0]["decoded_payload"]["text"], "hello from Alice"
)
@@ -2492,6 +2533,62 @@ def test_invalid_jsonl_returns_400_and_saves_quarantined_upload(self):
self.assertEqual(bad_event.raw_line, "{not-json}")
self.assertIn("JSON", bad_event.validation_error)
+ @override_settings(GOGGLES_MAX_DUMP_RECORDS=1)
+ def test_record_limit_quarantines_raw_upload_before_object_expansion(self):
+ raw_token, _token = UploadToken.issue("bounded parser")
+ body = representative_audit_log()
+
+ response = self.client.post(
+ reverse("api-audit-log-upload"),
+ data=body,
+ content_type="application/x-ndjson",
+ HTTP_AUTHORIZATION=f"Bearer {raw_token}",
+ )
+
+ self.assertEqual(response.status_code, 400)
+ self.assertIn("record count exceeds maximum of 1", response.json()["error"])
+ audit_file = AuditFile.objects.get()
+ self.assertEqual(audit_file.raw_text, body)
+ self.assertEqual(audit_file.events.count(), 1)
+ self.assertEqual(audit_file.events.get().raw_line, body)
+
+ def test_line_byte_limit_quarantines_raw_upload_before_json_loads(self):
+ raw_token, _token = UploadToken.issue("bounded parser")
+ body = jsonl(audit_event_v2(0, kind={"type": "recorder_started", "recorder": "mdk"}))
+ byte_limit = len(body.rstrip("\n").encode("utf-8")) - 1
+
+ with self.settings(GOGGLES_MAX_JSONL_LINE_BYTES=byte_limit):
+ response = self.client.post(
+ reverse("api-audit-log-upload"),
+ data=body,
+ content_type="application/x-ndjson",
+ HTTP_AUTHORIZATION=f"Bearer {raw_token}",
+ )
+
+ self.assertEqual(response.status_code, 400)
+ self.assertIn(f"exceeds maximum of {byte_limit} UTF-8 bytes", response.json()["error"])
+ audit_file = AuditFile.objects.get()
+ self.assertEqual(audit_file.raw_text, body)
+ self.assertEqual(audit_file.events.count(), 1)
+
+ @override_settings(GOGGLES_MAX_JSONL_LINE_BYTES=32)
+ def test_whitespace_only_line_still_obeys_line_byte_limit(self):
+ raw_token, _token = UploadToken.issue("bounded parser")
+ body = (" " * 33) + "\n"
+
+ response = self.client.post(
+ reverse("api-audit-log-upload"),
+ data=body,
+ content_type="application/x-ndjson",
+ HTTP_AUTHORIZATION=f"Bearer {raw_token}",
+ )
+
+ self.assertEqual(response.status_code, 400)
+ self.assertIn("line 1 exceeds maximum of 32 UTF-8 bytes", response.json()["error"])
+ audit_file = AuditFile.objects.get()
+ self.assertEqual(audit_file.raw_text, body)
+ self.assertEqual(audit_file.events.get().raw_line, body)
+
def test_invalid_utf8_group_upload_links_file_to_fallback_group(self):
raw_token, _token = UploadToken.issue("ios test client")
dump_bytes = b"\xff\xfeinvalid marmot audit bytes"
@@ -4482,6 +4579,24 @@ def _upload_message_event(self, seq, msg_id):
source_name=f"append-{seq}.jsonl",
)
+ def test_projection_queries_never_select_verbatim_evidence_columns(self):
+ with CaptureQueriesContext(connection) as captured:
+ result = self._upload_message_event(0, MSG_ID)
+
+ self.assertTrue(result.created)
+ self.assertEqual(
+ heavy_bulk_selects(
+ captured.captured_queries,
+ allowed_columns=(
+ HEAVY_EVENT_SELECT_COLUMNS["raw_kind"],
+ HEAVY_EVENT_SELECT_COLUMNS["context_transport"],
+ HEAVY_EVENT_SELECT_COLUMNS["context_convergence"],
+ ),
+ ),
+ [],
+ "projection ingestion must not hydrate raw upload or raw event bodies",
+ )
+
def test_small_append_does_not_clear_and_reproject_whole_group(self):
# Seed a group with several prior messages -> several projection rows.
prior_msg_ids = [f"{byte:02x}" * 32 for byte in range(0xA0, 0xA5)]
@@ -5015,6 +5130,30 @@ def action(target_count):
# The genuine first-seen 0 is preserved, not dropped or overwritten.
self.assertEqual(action_groups[0]["target_count"], 0)
+ @override_settings(GOGGLES_MAX_ACTION_EVENTS_PER_REQUEST=1)
+ def test_action_api_reports_when_history_scan_is_safely_truncated(self):
+ ingest_body(
+ jsonl(
+ audit_event(0, wall_time_ms=T0, context={"human_action": self._human_action()}),
+ audit_event(
+ 1,
+ wall_time_ms=T0 + 1000,
+ context={"human_action": self._human_action("rename_group")},
+ ),
+ )
+ )
+ group = AuditGroup.objects.get(slug=GROUP_REF)
+ User.objects.create_user(username="analyst", password="correct horse battery staple")
+ self.client.login(username="analyst", password="correct horse battery staple")
+
+ with CaptureQueriesContext(connection) as captured:
+ response = self.client.get(reverse("api-group-actions", kwargs={"slug": group.slug}))
+
+ self.assertEqual(response.status_code, 200)
+ self.assertTrue(response.json()["pagination"]["scan_truncated"])
+ self.assertEqual(response.json()["pagination"]["scan_limit"], 1)
+ self.assertEqual(heavy_bulk_selects(captured.captured_queries), [])
+
class UploadTokenLifecycleTests(TestCase):
"""Lock in the documented upload-token lifecycle: reusable by default,
@@ -5500,7 +5639,7 @@ def test_manifest_advertises_sections_and_eof_counts_are_accurate(self):
def test_streams_every_row_past_the_projection_page_cap(self):
# The paginated projections API caps a section at GROUP_PROJECTION_API_DEFAULT_LIMIT
- # (500). The export must stream every row — this is its reason to exist.
+ # (100). The export must stream every row — this is its reason to exist.
over_cap = GROUP_PROJECTION_API_DEFAULT_LIMIT + 1
evidence_event = AuditEvent.objects.filter(
group=self.group, parse_status=AuditEvent.STATUS_VALID
@@ -6088,33 +6227,41 @@ def test_group_detail_is_login_required_and_shows_audit_workflows(self):
self.assertNotContains(response, 'id="timeline-data"')
self.assertNotContains(response, MSG_ID)
- delivery_response = self.client.get(
- reverse("group-tab", kwargs={"slug": group.slug, "tab": "delivery"})
- )
+ with CaptureQueriesContext(connection) as tab_queries:
+ delivery_response = self.client.get(
+ reverse("group-tab", kwargs={"slug": group.slug, "tab": "delivery"})
+ )
+ network_response = self.client.get(
+ reverse("group-tab", kwargs={"slug": group.slug, "tab": "network"})
+ )
+ convergence_response = self.client.get(
+ reverse("group-tab", kwargs={"slug": group.slug, "tab": "convergence"})
+ )
+ state_response = self.client.get(
+ reverse("group-tab", kwargs={"slug": group.slug, "tab": "state"})
+ )
self.assertContains(delivery_response, "Message artifacts")
self.assertContains(delivery_response, MSG_ID[:16])
self.assertContains(delivery_response, "hello from Alice")
- network_response = self.client.get(
- reverse("group-tab", kwargs={"slug": group.slug, "tab": "network"})
- )
self.assertContains(network_response, "Transport observations")
self.assertContains(network_response, "wss://relay.example")
- convergence_response = self.client.get(
- reverse("group-tab", kwargs={"slug": group.slug, "tab": "convergence"})
- )
self.assertContains(convergence_response, "Convergence runs")
self.assertContains(convergence_response, "branch-a")
self.assertContains(convergence_response, "highest_weight")
- state_response = self.client.get(
- reverse("group-tab", kwargs={"slug": group.slug, "tab": "state"})
- )
self.assertContains(state_response, "Group state deltas")
self.assertContains(state_response, "text value")
self.assertNotContains(state_response, "Launch room")
self.assertContains(state_response, "Epoch state transitions")
+ self.assertEqual(
+ heavy_bulk_selects(
+ tab_queries.captured_queries,
+ allowed_columns=(HEAVY_EVENT_SELECT_COLUMNS["context_source"],),
+ ),
+ [],
+ )
evidence_response = self.client.get(
reverse("group-tab", kwargs={"slug": group.slug, "tab": "evidence"})
@@ -6122,6 +6269,28 @@ def test_group_detail_is_login_required_and_shows_audit_workflows(self):
self.assertContains(evidence_response, "Audit files")
self.assertContains(evidence_response, "Recent evidence rows")
+ with CaptureQueriesContext(connection) as evidence_api_queries:
+ evidence_api_response = self.client.get(
+ reverse("api-group-evidence", kwargs={"slug": group.slug})
+ )
+ self.assertEqual(evidence_api_response.status_code, 200)
+ self.assertTrue(evidence_api_response.json()["evidence"])
+ self.assertEqual(heavy_bulk_selects(evidence_api_queries.captured_queries), [])
+
+ event_id = evidence_api_response.json()["evidence"][0]["evidence_ref"]["event_id"]
+ with CaptureQueriesContext(connection) as event_api_queries:
+ event_api_response = self.client.get(
+ reverse("api-event-evidence", kwargs={"event_id": event_id})
+ )
+ self.assertEqual(event_api_response.status_code, 200)
+ self.assertIn("raw_line", event_api_response.json()["event"])
+ self.assertFalse(
+ any(
+ '"forensics_auditfile"."raw_text"' in query["sql"]
+ for query in event_api_queries.captured_queries
+ )
+ )
+
def test_group_detail_tabs_cap_large_group_rows(self):
tab_limit = GROUP_DETAIL_TAB_EVENT_LIMIT
row_count = tab_limit + 5
@@ -6162,12 +6331,14 @@ def test_group_detail_tabs_cap_large_group_rows(self):
artifact_kind="application_message",
first_seen_ms=1_700_000_000_000 + i,
)
- DeliveryObservation.objects.create(
+ artifact.evidence_events.add(evidence_event)
+ observation = DeliveryObservation.objects.create(
artifact=artifact,
engine_id=ENGINE_ALICE,
latest_state=f"delivery-marker-{i:03d}",
first_seen_ms=1_700_000_000_000 + i,
)
+ observation.evidence_events.add(evidence_event)
NetworkObservation.objects.create(
group=group,
artifact=artifact,
@@ -6209,9 +6380,10 @@ def test_group_detail_tabs_cap_large_group_rows(self):
User.objects.create_user(username="analyst", password="correct horse battery staple")
self.client.login(username="analyst", password="correct horse battery staple")
- delivery_response = self.client.get(
- reverse("group-tab", kwargs={"slug": group.slug, "tab": "delivery"})
- )
+ with CaptureQueriesContext(connection) as delivery_queries:
+ delivery_response = self.client.get(
+ reverse("group-tab", kwargs={"slug": group.slug, "tab": "delivery"})
+ )
network_response = self.client.get(
reverse("group-tab", kwargs={"slug": group.slug, "tab": "network"})
)
@@ -6223,6 +6395,13 @@ def test_group_detail_tabs_cap_large_group_rows(self):
)
self.assertEqual(delivery_response.status_code, 200)
+ self.assertEqual(
+ heavy_bulk_selects(
+ delivery_queries.captured_queries,
+ allowed_columns=(HEAVY_EVENT_SELECT_COLUMNS["context_source"],),
+ ),
+ [],
+ )
self.assertEqual(len(delivery_response.context["artifacts"]), tab_limit)
self.assertContains(delivery_response, f"Showing first {tab_limit} message artifacts")
self.assertContains(delivery_response, "delivery-marker-000")
@@ -6388,23 +6567,36 @@ def test_group_detail_shell_size_stays_bounded_for_large_groups(self):
User.objects.create_user(username="analyst", password="correct horse battery staple")
self.client.login(username="analyst", password="correct horse battery staple")
- response = self.client.get(reverse("group-detail", kwargs={"slug": group.slug}))
+ with CaptureQueriesContext(connection) as shell_queries:
+ response = self.client.get(reverse("group-detail", kwargs={"slug": group.slug}))
self.assertEqual(response.status_code, 200)
self.assertLess(len(response.content), 250_000)
self.assertNotContains(response, "RAW-LINE-MARKER-2999")
self.assertNotContains(response, 'id="timeline-data"')
-
- evidence_response = self.client.get(
- reverse("group-tab", kwargs={"slug": group.slug, "tab": "evidence"})
+ self.assertEqual(
+ heavy_bulk_selects(
+ shell_queries.captured_queries,
+ allowed_columns=(
+ HEAVY_EVENT_SELECT_COLUMNS["raw_kind"],
+ HEAVY_EVENT_SELECT_COLUMNS["context_source"],
+ ),
+ ),
+ [],
)
+ with CaptureQueriesContext(connection) as evidence_queries:
+ evidence_response = self.client.get(
+ reverse("group-tab", kwargs={"slug": group.slug, "tab": "evidence"})
+ )
+
self.assertEqual(evidence_response.status_code, 200)
self.assertLess(len(evidence_response.content), 250_000)
self.assertLessEqual(
len(evidence_response.context["recent_events"]),
GROUP_DETAIL_TAB_EVENT_LIMIT,
)
+ self.assertEqual(heavy_bulk_selects(evidence_queries.captured_queries), [])
class AuditFileDetailViewTests(TestCase):
@@ -8083,6 +8275,18 @@ def test_group_agent_export_returns_agent_readable_json(self):
self.assertNotIn("raw_line", payload["events"][0])
self.assertIn("raw_upload_bodies", payload["sensitivity"]["omits"])
+ @override_settings(GOGGLES_AGENT_EXPORT_MAX_EVENTS=1)
+ def test_group_agent_export_rejects_oversized_synchronous_build(self):
+ ingest_body(representative_audit_log())
+ User.objects.create_user(username="analyst", password="correct horse battery staple")
+ self.client.login(username="analyst", password="correct horse battery staple")
+
+ response = self.client.get(reverse("group-agent-export", kwargs={"slug": GROUP_REF}))
+
+ self.assertEqual(response.status_code, 413)
+ self.assertEqual(response.json()["event_count"], 2)
+ self.assertEqual(response.json()["max_events"], 1)
+
def test_group_detail_fetches_events_with_bounded_queries(self):
ingest_body(representative_audit_log(engine_id=ENGINE_ALICE))
ingest_body(representative_audit_log(engine_id=ENGINE_BOB))
@@ -8419,6 +8623,18 @@ def test_admin_module_defines_no_event_inline(self):
"The unbounded AuditEventInline must not exist (goggles#34).",
)
+ def test_admin_changelists_do_not_select_verbatim_evidence_columns(self):
+ audit_file = self.make_audit_file(3, source_name="large")
+ AuditFile.objects.filter(pk=audit_file.pk).update(raw_text="x" * 5_000_000)
+
+ with CaptureQueriesContext(connection) as captured:
+ file_response = self.client.get(reverse("admin:forensics_auditfile_changelist"))
+ event_response = self.client.get(reverse("admin:forensics_auditevent_changelist"))
+
+ self.assertEqual(file_response.status_code, 200)
+ self.assertEqual(event_response.status_code, 200)
+ self.assertEqual(heavy_bulk_selects(captured.captured_queries), [])
+
def test_change_page_query_count_is_bounded_regardless_of_events(self):
small = self.make_audit_file(2)
large = self.make_audit_file(200, source_name="big")
@@ -8531,6 +8747,35 @@ def test_audit_event_admin_keeps_audit_file_deep_link_without_sidebar_filter(sel
self.assertIn("audit_file", AuditEventAdmin.autocomplete_fields)
self.assertIn("group", AuditEventAdmin.autocomplete_fields)
+ def test_audit_event_change_page_bounds_raw_evidence(self):
+ from .admin import AuditEventAdmin
+
+ audit_file = self.make_audit_file(1)
+ event = audit_file.events.get()
+ trailing_marker = "TRAILING-RAW-LINE-MARKER"
+ AuditEvent.objects.filter(pk=event.pk).update(
+ raw_line="FIRST-RAW-LINE-MARKER" + ("x" * 2_000_000) + trailing_marker,
+ raw_event={"oversized": "y" * 2_000_000},
+ raw_kind={"oversized": "z" * 500_000},
+ raw_context={"oversized": "q" * 500_000},
+ )
+
+ response = self.client.get(reverse("admin:forensics_auditevent_change", args=[event.pk]))
+
+ self.assertEqual(response.status_code, 200)
+ self.assertLess(len(response.content), 200_000)
+ self.assertContains(response, "Raw line (preview)")
+ self.assertContains(response, "FIRST-RAW-LINE-MARKER")
+ self.assertNotContains(response, trailing_marker)
+ self.assertContains(response, "Open JSON evidence")
+ self.assertContains(
+ response,
+ reverse("api-event-evidence", kwargs={"event_id": event.pk}),
+ )
+ for field in HEAVY_EVENT_SELECT_COLUMNS:
+ self.assertIn(field, AuditEventAdmin.exclude)
+ self.assertNotContains(response, f'name="{field}"')
+
# --- raw_text bounding (goggles#34, adversarial review follow-up) -------
#
# Removing the event inline alone did not make the change page bounded:
diff --git a/forensics/views.py b/forensics/views.py
index 1af0983..0006030 100644
--- a/forensics/views.py
+++ b/forensics/views.py
@@ -67,8 +67,8 @@
RAW_TEXT_PREVIEW_CHARS = 32 * 1024
GROUP_ENGINE_PREVIEW_LIMIT = 12
GROUP_DETAIL_TAB_EVENT_LIMIT = 100
-GROUP_PROJECTION_API_DEFAULT_LIMIT = 500
-GROUP_PROJECTION_API_MAX_LIMIT = 5_000
+GROUP_PROJECTION_API_DEFAULT_LIMIT = 100
+GROUP_PROJECTION_API_MAX_LIMIT = 500
GROUP_EXPORT_SCHEMA_VERSION = "goggles-group-export/v1"
FULL_DATA_AUDIT_MODE = "full_data"
ERROR_SEVERITY_TOKENS = (
@@ -105,6 +105,43 @@
"exports": "forensics/partials/group_exports.html",
}
+# Event list/projection responses render normalized summaries and evidence
+# references, never the verbatim line or parsed source object. Keep the heavy
+# evidence columns out of every bulk queryset; the single-event evidence API is
+# the intentionally narrow exception.
+HEAVY_EVENT_FIELDS = {
+ "raw_line",
+ "raw_event",
+ "raw_kind",
+ "raw_context",
+ "context_human_action",
+ "context_transport",
+ "context_engine",
+ "context_group",
+ "context_convergence",
+ "context_source",
+}
+EVENT_ROW_FIELDS = tuple(
+ field.name
+ for field in AuditEvent._meta.local_concrete_fields
+ if field.name not in HEAVY_EVENT_FIELDS
+)
+EVIDENCE_REF_EVENT_FIELDS = (
+ "id",
+ "audit_file_id",
+ "line_number",
+ "line_hash",
+ "schema_version",
+ "audit_data_mode",
+ "event_type",
+ "wall_time_ms",
+)
+
+
+def evidence_ref_event_queryset(*extra_fields: str):
+ fields = tuple(dict.fromkeys((*EVIDENCE_REF_EVENT_FIELDS, *extra_fields)))
+ return AuditEvent.objects.only(*fields)
+
def healthz(_request: HttpRequest):
return JsonResponse({"status": "ok"})
@@ -376,7 +413,7 @@ def engine_source_values(group: AuditGroup) -> dict[str, dict[str, list[str]]]:
"audit_file__source_account_npub",
)
)
- for event in events:
+ for event in events.iterator(chunk_size=2_000):
engine_values = values_by_engine.setdefault(
event.engine_id,
{key: set() for key, _file_field, _context_key in ENGINE_SOURCE_FIELD_MAP}
@@ -504,6 +541,7 @@ def group_overview_context(group: AuditGroup) -> dict:
"other",
action_filters,
),
+ "action_events_limited": action_groups.truncated,
}
@@ -528,10 +566,11 @@ def group_tab_context(group: AuditGroup, tab: str) -> dict:
if tab == "overview":
return {"group": group, "overview": group_overview_context(group)}
if tab == "evidence":
- audit_files = list(audit_files_for_group(group))
+ audit_files, audit_files_limited = limited_tab_events(audit_files_for_group(group))
return {
"group": group,
"audit_files": file_rows_for_group(audit_files, group),
+ "audit_files_limited": audit_files_limited,
"recent_events": recent_evidence_rows(group),
"tab_event_limit": GROUP_DETAIL_TAB_EVENT_LIMIT,
}
@@ -552,8 +591,8 @@ def group_tab_context(group: AuditGroup, tab: str) -> dict:
}
if tab == "network":
observations, has_more = limited_tab_events(
- NetworkObservation.objects.filter(group=group)
- .select_related("artifact", "audit_event")
+ network_observation_queryset()
+ .filter(group=group)
.order_by("wall_time_ms", "engine_id", "id")
)
return {
@@ -564,8 +603,8 @@ def group_tab_context(group: AuditGroup, tab: str) -> dict:
}
if tab == "convergence":
runs, has_more = limited_tab_events(
- ConvergenceRun.objects.filter(group=group)
- .prefetch_related("candidates", "rule_evaluations", "evidence_events")
+ convergence_run_queryset()
+ .filter(group=group)
.order_by("started_at_ms", "engine_id", "run_id")
)
attach_decisive_rules(runs)
@@ -576,11 +615,9 @@ def group_tab_context(group: AuditGroup, tab: str) -> dict:
"tab_event_limit": GROUP_DETAIL_TAB_EVENT_LIMIT,
}
if tab == "state":
- deltas, deltas_has_more = limited_tab_events(
- StateDelta.objects.filter(group=group).select_related("audit_event")
- )
+ deltas, deltas_has_more = limited_tab_events(state_delta_queryset().filter(group=group))
transitions, transitions_has_more = limited_tab_events(
- EpochStateTransition.objects.filter(group=group).select_related("audit_event")
+ epoch_transition_queryset().filter(group=group)
)
return {
"group": group,
@@ -595,7 +632,9 @@ def group_tab_context(group: AuditGroup, tab: str) -> dict:
"group": group,
"summary": group_summary_header_context(group)["summary"],
"classification": group_classification(group),
- "saved_reports": list(group.analysis_runs.select_related("created_by")[:20]),
+ "saved_reports": list(
+ group.analysis_runs.select_related("created_by").defer("report_json")[:20]
+ ),
}
raise Http404("unknown group detail tab")
@@ -668,7 +707,7 @@ def attach_decisive_rules(runs: list[ConvergenceRun]) -> None:
def recent_evidence_rows(group: AuditGroup) -> list[dict]:
events = list(
AuditEvent.objects.filter(group=group)
- .select_related("audit_file")
+ .only(*EVENT_ROW_FIELDS)
.order_by("-wall_time_ms", "-id")[:GROUP_DETAIL_TAB_EVENT_LIMIT]
)
rows = []
@@ -683,6 +722,16 @@ def recent_evidence_rows(group: AuditGroup) -> list[dict]:
@login_required
def group_agent_export(request: HttpRequest, slug: str):
group = get_object_or_404(AuditGroup, slug=slug)
+ event_count = valid_events_for_group(group).count()
+ if event_count > settings.GOGGLES_AGENT_EXPORT_MAX_EVENTS:
+ return JsonResponse(
+ {
+ "error": "group too large for synchronous export",
+ "event_count": event_count,
+ "max_events": settings.GOGGLES_AGENT_EXPORT_MAX_EVENTS,
+ },
+ status=413,
+ )
audit_files = list(audit_files_for_group(group))
events = list(valid_events_for_group(group, include_export_fields=True))
pretty = request.GET.get("pretty", "").lower() in {"1", "true", "yes"}
@@ -1202,7 +1251,10 @@ def api_engine_groups(request: HttpRequest, engine_id: str):
@login_required
def api_event_evidence(request: HttpRequest, event_id: int):
event = get_object_or_404(
- AuditEvent.objects.select_related("audit_file", "group"),
+ AuditEvent.objects.select_related("audit_file", "group").defer(
+ "audit_file__raw_text",
+ "audit_file__user_agent",
+ ),
pk=event_id,
)
return JsonResponse(
@@ -1724,15 +1776,27 @@ def audit_data_mode_change_payload_severity(payload: dict) -> str:
def delivery_artifact_queryset():
+ observation_evidence = evidence_ref_event_queryset(
+ "context_source",
+ "audit_file__source_account_pubkey_hex",
+ ).select_related("audit_file")
+ expectation_evidence = evidence_ref_event_queryset(
+ "engine_id",
+ "account_ref",
+ "context_source",
+ )
return DeliveryArtifact.objects.prefetch_related(
- Prefetch("evidence_events", queryset=AuditEvent.objects.select_related("audit_file")),
+ Prefetch("evidence_events", queryset=evidence_ref_event_queryset()),
"engine_observations",
Prefetch(
"engine_observations__evidence_events",
- queryset=AuditEvent.objects.select_related("audit_file"),
+ queryset=observation_evidence,
),
"recipient_expectations",
- "recipient_expectations__evidence_event",
+ Prefetch(
+ "recipient_expectations__evidence_event",
+ queryset=expectation_evidence,
+ ),
)
@@ -1768,7 +1832,12 @@ def apply_delivery_artifact_filters(artifacts, filters: dict):
def network_observation_queryset():
- return NetworkObservation.objects.select_related("artifact", "audit_event", "group")
+ return NetworkObservation.objects.prefetch_related(
+ Prefetch(
+ "audit_event",
+ queryset=evidence_ref_event_queryset(),
+ )
+ )
def filtered_network_observations(group: AuditGroup, filters: dict):
@@ -1792,7 +1861,7 @@ def filtered_network_observations(group: AuditGroup, filters: dict):
def convergence_run_queryset():
return ConvergenceRun.objects.prefetch_related(
- "evidence_events",
+ Prefetch("evidence_events", queryset=evidence_ref_event_queryset()),
"candidates",
"rule_evaluations",
)
@@ -1849,7 +1918,9 @@ def convergence_payload_matches_filters(payload: dict, filters: dict) -> bool:
def state_delta_queryset():
- return StateDelta.objects.select_related("audit_event", "group")
+ return StateDelta.objects.prefetch_related(
+ Prefetch("audit_event", queryset=evidence_ref_event_queryset())
+ )
def filtered_state_deltas(group: AuditGroup, filters: dict):
@@ -1879,7 +1950,9 @@ def filtered_state_deltas(group: AuditGroup, filters: dict):
def epoch_transition_queryset():
- return EpochStateTransition.objects.select_related("audit_event", "group")
+ return EpochStateTransition.objects.prefetch_related(
+ Prefetch("audit_event", queryset=evidence_ref_event_queryset())
+ )
def filtered_epoch_transitions(group: AuditGroup, filters: dict):
@@ -1908,7 +1981,19 @@ def filtered_epoch_transitions(group: AuditGroup, filters: dict):
def audit_data_mode_change_queryset(group: AuditGroup):
- return valid_group_event_queryset(group).filter(event_type="audit_data_mode_changed")
+ return (
+ valid_group_event_queryset(group)
+ .filter(event_type="audit_data_mode_changed")
+ .only(
+ *EVIDENCE_REF_EVENT_FIELDS,
+ "engine_id",
+ "account_ref",
+ "recorder_session_id",
+ "outcome",
+ "reason",
+ "raw_kind",
+ )
+ )
def filtered_audit_data_mode_changes(group: AuditGroup, filters: dict):
@@ -1925,11 +2010,28 @@ def filtered_audit_data_mode_changes(group: AuditGroup, filters: dict):
events = events.filter(wall_time_ms__gte=filters["from_ms"])
if filters["to_ms"] is not None:
events = events.filter(wall_time_ms__lte=filters["to_ms"])
- return events.select_related("audit_file", "group")
+ return events.only(
+ *EVIDENCE_REF_EVENT_FIELDS,
+ "engine_id",
+ "account_ref",
+ "recorder_session_id",
+ "outcome",
+ "reason",
+ "raw_kind",
+ )
def evidence_event_queryset(group: AuditGroup):
- return AuditEvent.objects.filter(group=group).select_related("audit_file", "group")
+ return (
+ AuditEvent.objects.filter(group=group)
+ .select_related("audit_file")
+ .only(
+ *EVENT_ROW_FIELDS,
+ "audit_file__source_name",
+ "audit_file__validation_status",
+ "audit_file__file_sha256",
+ )
+ )
def filtered_evidence_events(group: AuditGroup, filters: dict):
@@ -1966,7 +2068,7 @@ def action_event_queryset(group: AuditGroup):
return (
valid_group_event_queryset(group)
.exclude(human_action_action="")
- .select_related("audit_file", "group")
+ .only(*EVENT_ROW_FIELDS)
.order_by("wall_time_ms", "engine_id", "line_number", "id")
)
@@ -2001,13 +2103,34 @@ def filtered_action_events(group: AuditGroup, filters: dict):
}
-def action_groups_for_api(group: AuditGroup, filters: dict) -> list[dict]:
- events = list(filtered_action_events(group, filters))
+class ActionGroupCollection(list[dict]):
+ def __init__(self, values=(), *, truncated: bool = False):
+ super().__init__(values)
+ self.truncated = truncated
+
+
+def action_groups_for_api(group: AuditGroup, filters: dict) -> ActionGroupCollection:
+ max_events = settings.GOGGLES_MAX_ACTION_EVENTS_PER_REQUEST
+ # Pull the newest bounded window, then restore chronological order for the
+ # existing grouping semantics. This prevents a single group overview/API
+ # request from materializing an unbounded action history.
+ events = list(
+ filtered_action_events(group, filters).order_by(
+ "-wall_time_ms", "-engine_id", "-line_number", "-id"
+ )[: max_events + 1]
+ )
+ truncated = len(events) > max_events
+ if truncated:
+ events = events[:max_events]
+ events.reverse()
if filters["message_id"]:
events = [
event for event in events if filters["message_id"] in action_event_message_ids(event)
]
event_by_id = {event.id: event for event in events}
+ # A truncated window can bisect one operation_id group. In that case the
+ # request-level scan_truncated flag is authoritative and each affected
+ # group's event_count describes only its in-window evidence.
groups = [
action_group_payload(action_group, event_by_id)
for action_group in human_action_groups_for_group(events)
@@ -2018,7 +2141,7 @@ def action_groups_for_api(group: AuditGroup, filters: dict) -> list[dict]:
for action_group in groups
if action_attribution_payload_severity(action_group) == filters["severity"]
]
- return groups
+ return ActionGroupCollection(groups, truncated=truncated)
def action_origin_counts(group: AuditGroup) -> list[dict]:
@@ -2056,7 +2179,14 @@ def attribution_pagination_payload(action_groups: list[dict], filters: dict) ->
for kind, page_key in ACTION_ATTRIBUTION_PAGE_KEYS.items():
total = len([row for row in action_groups if row["attribution_kind"] == kind])
returned = min(max(total - offset, 0), limit)
- payload[page_key] = pagination_payload(limit, offset, returned, total > offset + limit)
+ payload[page_key] = pagination_payload(
+ limit,
+ offset,
+ returned,
+ total > offset + limit or getattr(action_groups, "truncated", False),
+ )
+ payload["scan_truncated"] = getattr(action_groups, "truncated", False)
+ payload["scan_limit"] = settings.GOGGLES_MAX_ACTION_EVENTS_PER_REQUEST
return payload
@@ -3178,14 +3308,13 @@ def read_upload_bytes(upload) -> bytes:
if upload_size is not None and upload_size > max_dump_bytes:
raise RequestDataTooBig(UPLOAD_TOO_LARGE_ERROR)
- chunks = []
- total_bytes = 0
- for chunk in upload.chunks():
- total_bytes += len(chunk)
- if total_bytes > max_dump_bytes:
- raise RequestDataTooBig(UPLOAD_TOO_LARGE_ERROR)
- chunks.append(chunk)
- return b"".join(chunks)
+ # Multipart uploads larger than FILE_UPLOAD_MAX_MEMORY_SIZE have already
+ # spooled to disk. Read at most one byte beyond the accepted limit instead
+ # of retaining a chunk list and then allocating a second full-size join.
+ data = upload.read(max_dump_bytes + 1)
+ if len(data) > max_dump_bytes:
+ raise RequestDataTooBig(UPLOAD_TOO_LARGE_ERROR)
+ return data
def source_metadata_from_request(request: HttpRequest) -> dict[str, str]: