diff --git a/.gitignore b/.gitignore index 14969b9..2b802b3 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,7 @@ capture-android/build/ capture-android/app/build/ capture-android/local.properties infra/cdk.context.json +infra/image_uris.json .worktrees/ +work_deploy*.log +work_cdk_outputs.json diff --git a/README.md b/README.md new file mode 100644 index 0000000..fff439c --- /dev/null +++ b/README.md @@ -0,0 +1,84 @@ +# GRIDPRINT + +GRIDPRINT checks whether a video is real by reading the power-grid frequency from the flicker of the lights in +the room. Mains-powered lights flicker at twice the grid frequency. A phone's rolling shutter reads the sensor one +row at a time, so the flicker shows up as faint horizontal bands. GRIDPRINT measures those bands with OpenCV 5, +rebuilds the grid-frequency track (the electric network frequency, ENF), and compares it with a recorded +reference log of the real grid. The result is one of three verdicts: `VERIFIED`, `CONTRADICTED` (only final after +a human reviewer approves it) or `INCONCLUSIVE`. A missing signal is never evidence that a video is fake. GRIDPRINT +is not a deepfake classifier and not cryptographic proof. + +Entry for the OpenCV AI Competition 2026 (OpenCV 5 + AWS). The specification is in [SPEC.md](SPEC.md); current +status is in [docs/PROGRESS.md](docs/PROGRESS.md). + +- Endpoint: see [docs/JUDGE_GUIDE.md](docs/JUDGE_GUIDE.md) +- Technical report: [docs/TECHNICAL_REPORT.md](docs/TECHNICAL_REPORT.md) +- Architecture: [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) · diagrams in [docs/diagrams/](docs/diagrams/) +- Responsible use: [docs/RESPONSIBLE_USE.md](docs/RESPONSIBLE_USE.md) +- Video: (not recorded yet — human task H8) + +## What is in the repository + +| Part | Path | +|---|---| +| Engine (OpenCV 5 stages S1–S15) | `engine/gridprint_engine/` | +| Agent (planner, policies, trace) and MCP tool server | `agent/`, `mcp_server/` | +| API and job worker | `api/` | +| Verify web app | `web/` | +| Capture Android app | `capture-android/` | +| Grid-frequency Node (hardware, firmware, ingest) | `node/` | +| Camera calibration rig | `calibration_rig/` | +| AWS infrastructure (CDK v2) | `infra/` | +| Simulator and evaluation | `eval/` | +| COOL benchmark | `bench/` | + +## Quick start (local) + +Requirements: Python 3.11, [uv](https://docs.astral.sh/uv/), FFmpeg on `PATH`, Node.js 24 (for the web app). + +```bash +make setup +``` + +```bash +make test +``` + +```bash +python -m eval.make_demo_inputs +``` + +```bash +make local +``` + +`make local` serves the API and the built web app at `http://127.0.0.1:8000` with local file storage and the +deterministic agent planner. The reviewer token for local use is written to `work/local_reviewer_token.txt`. +On Windows without GNU make, use `./make ` (Git Bash) or `make.cmd `. + +## Deploy to AWS + +1. Complete human task H1 in [docs/HUMAN_TASKS.md](docs/HUMAN_TASKS.md) (CLI profile `gridprint`). +2. Build the arm64 images (no local Docker needed): `python infra/build_images.py`. +3. Deploy all stacks: `make deploy`. Remove everything: `make destroy`. + +## Evaluate and benchmark + +```bash +make eval +``` + +```bash +make bench +``` + +Synthetic results are labeled SYNTHETIC and are never presented as real-world accuracy. Every reported number +lives under `results/` with a source tag (SPEC 14.3). + +## Final check + +```bash +make final-check +``` + +It prints one line per item of the final checklist (SPEC 17.3) and `ALL CHECKS PASSED` only when all hold. diff --git a/agent/orchestrator.py b/agent/orchestrator.py index 8ac2408..05631ff 100644 --- a/agent/orchestrator.py +++ b/agent/orchestrator.py @@ -44,6 +44,36 @@ async def call(self, tool: str, args: dict[str, Any]) -> dict[str, Any]: return json.loads(text) if text else {} +_PREREQ_FREE = {"probe_video", "compute_verdict", "request_user_input", "request_human_review", "render_report"} + + +def _probe_ok(history: list[Step]) -> bool: + return any(h.tool == "probe_video" and h.error is None for h in history) + + +def _needs_rule(a: Action, history: list[Step]) -> bool: + if a.tool not in {"render_report", "compute_verdict"} and any(h.tool == a.tool and h.args == a.args + for h in history): + return True # identical call already made (failed or not): no new information + if a.tool == "extract_flicker" and (a.args.get("nominal_hz") or a.args.get("axis")) and not any( + h.tool == "extract_flicker" and h.result for h in history): + return True # B1 allows switching frequency or axis only after a weak first extraction + asked = {h.args.get("kind") for h in history if h.tool == "request_user_input"} + if a.tool == "request_user_input" and a.args.get("kind") in asked: + return True + return a.tool not in _PREREQ_FREE and not _probe_ok(history) + + +def _why(a: Action, history: list[Step]) -> str: + if any(h.tool == a.tool and h.args == a.args for h in history): + return "identical call already made" + if a.tool == "extract_flicker" and not any(h.tool == "extract_flicker" and h.result for h in history): + return "frequency or axis override before the first extraction" + if a.tool == "request_user_input": + return "same request already made" + return "video not probed yet" + + async def run_job(job_id: str, planner: Planner, caller: ToolCaller, max_iterations: int, max_tool_calls: int, trace_path: Path | None = None) -> tuple[Trace, list[Step]]: trace = Trace(job_id) @@ -56,6 +86,13 @@ async def run_job(job_id: str, planner: Planner, caller: ToolCaller, max_iterati except Exception as e: # planner failure (e.g. Bedrock unavailable): fall back for this step proposed = FallbackPlanner().next_action(history) proposed.reason = f"planner error ({type(e).__name__}); fallback rule: {proposed.reason}" + # Guard the LLM planner: a call that repeats a failed call, or an analysis call before the video was + # probed successfully, is replaced by the deterministic rule for this step (recorded in the trace). + if planner.kind != "fallback" and proposed.tool is not None and _needs_rule(proposed, history): + rule = FallbackPlanner().next_action(history) + rule.reason = (f"guard: planner proposed {proposed.tool} {proposed.args} ({_why(proposed, history)}); " + f"fallback rule: {rule.reason}") + proposed = rule action, note = policies.enforce(proposed, st) if action.tool is None: if proposed.tool is not None or note: diff --git a/agent/planner_bedrock.py b/agent/planner_bedrock.py index 0ce29b3..5e94bc2 100644 --- a/agent/planner_bedrock.py +++ b/agent/planner_bedrock.py @@ -79,7 +79,18 @@ def next_action(self, history: list[Step]) -> Action: return Action(None, {}, reason or "no tool chosen") -def make_client(profile: str | None, region: str) -> Any: +def make_client(profile: str | None, region: str, role_arn: str | None = None, + external_id: str | None = None) -> Any: + """bedrock-runtime client. With `role_arn`, the model is called through a role in the model account + (cross-account, D-026); the role allows only InvokeModel on the pinned model.""" import boto3 - return boto3.Session(profile_name=profile, region_name=region).client("bedrock-runtime") + session = boto3.Session(profile_name=profile, region_name=region) + if role_arn: + kw = {"RoleArn": role_arn, "RoleSessionName": "planner"} + if external_id: + kw["ExternalId"] = external_id + c = session.client("sts").assume_role(**kw)["Credentials"] + session = boto3.Session(aws_access_key_id=c["AccessKeyId"], aws_secret_access_key=c["SecretAccessKey"], + aws_session_token=c["SessionToken"], region_name=region) + return session.client("bedrock-runtime") diff --git a/agent/policies.py b/agent/policies.py index bcf5241..48b3534 100644 --- a/agent/policies.py +++ b/agent/policies.py @@ -19,6 +19,40 @@ } +def _zones() -> set[str]: + try: + from pathlib import Path + + import yaml + + p = Path(__file__).resolve().parents[1] / "config" / "zones.yaml" + return {z["id"] for z in yaml.safe_load(p.read_text())["zones"]} + except Exception: # config missing: accept no zone argument + return set() + + +# allowed values per argument; a value outside the set is dropped (the engine default applies) +ALLOWED_VALUES: dict[str, Any] = { + "method": {"auto", "sidecar", "profile", "blind"}, "strictness": {"normal", "strict"}, + "strategy": {"all", "top", "coherent", "bright"}, "axis": {"rows+", "rows-", "cols+", "cols-"}, + "nominal_hz": {50, 60}, "kind": {"original_file", "camera_model", "claimed_time", "longer_clip"}, + "final": {True, False}, "zone": _zones(), +} + + +def _valid(k: str, v: Any) -> bool: + if k in ALLOWED_VALUES: + try: + return v in ALLOWED_VALUES[k] + except TypeError: + return False + if k == "patch_ids": + return isinstance(v, list) and all(isinstance(i, int) for i in v) + if k == "frame_subset": + return v == "low_motion" or (isinstance(v, list) and all(isinstance(i, int) for i in v)) + return isinstance(v, str | int | float) or v is None + + @dataclass class Action: tool: str | None # None = planner wants to stop @@ -53,10 +87,13 @@ def enforce(a: Action, st: PolicyState) -> tuple[Action, str | None]: "policy: report must be rendered"), "report required" return a, None if a.tool not in ALL_TOOLS: - return Action("compute_verdict" if not st.verdict_done else None, {}, - "policy: unknown tool rejected"), f"unknown tool {a.tool}" + if st.verdict_done: + nxt, _ = enforce(Action(None, {}, ""), st) + nxt.reason = f"policy: unknown tool {a.tool} rejected; " + (nxt.reason or "job complete") + return nxt, f"unknown tool {a.tool}" + return Action("compute_verdict", {}, "policy: unknown tool rejected"), f"unknown tool {a.tool}" note = None - args = {k: v for k, v in a.args.items() if k in ALLOWED_ARGS[a.tool]} + args = {k: v for k, v in a.args.items() if k in ALLOWED_ARGS[a.tool] and _valid(k, v)} if args != {k: v for k, v in a.args.items() if k != "job_id"}: note = "policy: disallowed arguments removed" a = Action(a.tool, args, a.reason) @@ -66,11 +103,10 @@ def enforce(a: Action, st: PolicyState) -> tuple[Action, str | None]: if a.tool == "extract_flicker" and st.iterations >= st.max_iterations: st.budget_exhausted = True return Action("compute_verdict", {}, "policy: iteration budget exhausted (B5)"), "iteration budget" - if st.verdict_done and a.tool in ANALYSIS_TOOLS: - return Action(None, {}, "policy: compute_verdict must be the last analysis call"), \ - f"{a.tool} after verdict blocked" - if st.verdict_done and a.tool == "compute_verdict": - return Action(None, {}, "policy: verdict already computed"), "repeat verdict blocked" + if st.verdict_done and a.tool in ANALYSIS_TOOLS | {"compute_verdict"}: + nxt, _ = enforce(Action(None, {}, ""), st) # finish the job: review if needed, then the report + nxt.reason = f"policy: {a.tool} after the verdict is not allowed; " + (nxt.reason or "job complete") + return nxt, f"{a.tool} after verdict blocked" if a.tool == "render_report": if not st.verdict_done: return Action("compute_verdict", {}, "policy: verdict before report"), "report before verdict" diff --git a/agent/tests/test_agent.py b/agent/tests/test_agent.py index b887e61..20b73d5 100644 --- a/agent/tests/test_agent.py +++ b/agent/tests/test_agent.py @@ -148,7 +148,7 @@ def test_policy_blocks_final_report_for_contradicted_and_analysis_after_verdict( a, note = policies.enforce(Action("render_report", {"final": True}, "x"), st) assert a.tool == "render_report" and a.args["final"] is False and note a, _ = policies.enforce(Action("extract_flicker", {}, "x"), st) - assert a.tool is None + assert a.tool == "render_report" and a.args["final"] is False # no analysis after the verdict def test_policy_strips_threshold_arguments(): @@ -198,3 +198,68 @@ def test_bedrock_planner_is_policed(): def test_tool_config_lists_all_tools(): names = {t["toolSpec"]["name"] for t in tool_config()["tools"]} assert policies.ALL_TOOLS <= names + + +class FailingProbeTools(ScriptedTools): + """probe_video fails once, then succeeds.""" + + def __init__(self) -> None: + super().__init__() + self.probe_failed = False + + async def call(self, tool: str, args: dict[str, Any]) -> dict[str, Any]: + if tool == "probe_video" and not self.probe_failed: + self.calls.append((tool, args)) + self.probe_failed = True + raise RuntimeError("Unable to allocate memory") + return await super().call(tool, args) + + +def test_llm_guard_replaces_repeated_failures_and_out_of_order_calls(): + # an LLM that ignores errors: always asks for motion analysis + client = FakeBedrock([{"tool": "analyze_motion", "input": {"strictness": "normal"}}]) + t = FailingProbeTools() + trace, _ = run(t, BedrockPlanner(client, "test-model")) + names = tools_called(t) + assert names[0] == "probe_video" and names[1] == "compute_verdict" # guard: unreadable input ends the job + assert any("guard:" in s.decision_reason for s in trace.steps) + assert names.count("analyze_motion") <= 2 # the repeated call is not retried endlessly + assert names[-1] == "render_report" # the job always ends with a report + + +def test_llm_guard_blocks_premature_overrides_and_identical_repeats(): + script = [{"tool": "probe_video"}, {"tool": "estimate_readout", "input": {"method": "auto"}}, + {"tool": "analyze_motion", "input": {"strictness": "normal"}}, + {"tool": "propose_patches", "input": {"strategy": "top"}}, + {"tool": "extract_flicker", "input": {"nominal_hz": 60}}, + {"tool": "propose_patches", "input": {"strategy": "top"}}] + client = FakeBedrock(script) + t = ScriptedTools() + trace, _ = run(t, BedrockPlanner(client, "test-model")) + ex = [a for n, a in t.calls if n == "extract_flicker"] + assert ex and "nominal_hz" not in ex[0] # the first extraction used the zone default + assert sum(1 for n, a in t.calls if n == "propose_patches" and a.get("strategy") == "top") <= 1 + assert any("frequency or axis override" in s.decision_reason for s in trace.steps) + assert t.calls[-1][0] == "render_report" + + +def test_policy_after_verdict_finishes_with_report(): + st = PolicyState(max_iterations=6, max_tool_calls=30, verdict_done=True, verdict="INCONCLUSIVE") + a, note = policies.enforce(Action("compute_verdict", {}, "again"), st) + assert a.tool == "render_report" and a.args["final"] is True and note + st2 = PolicyState(max_iterations=6, max_tool_calls=30, verdict_done=True, verdict="CONTRADICTED") + a, _ = policies.enforce(Action("match_reference", {}, "late"), st2) + assert a.tool == "request_human_review" + + +def test_policy_drops_invalid_argument_values(): + st = PolicyState(max_iterations=6, max_tool_calls=30) + a, note = policies.enforce(Action("match_reference", {"track_id": "t1", "zone": "global"}, "x"), st) + assert a.args == {"track_id": "t1"} and note + a, _ = policies.enforce(Action("extract_flicker", {"nominal_hz": 55, "axis": "diagonal"}, "x"), st) + assert a.args == {} + a, _ = policies.enforce(Action("match_reference", {"zone": "SIM50"}, "x"), st) + assert a.args == {"zone": "SIM50"} + st.verdict_done, st.verdict = True, "INCONCLUSIVE" + a, _ = policies.enforce(Action("render", {}, "x"), st) + assert a.tool == "render_report" diff --git a/api/app.py b/api/app.py index ffcea50..cf92047 100644 --- a/api/app.py +++ b/api/app.py @@ -195,20 +195,40 @@ def decide(review_id: str, body: DecisionBody, who: str = Depends(require_review ctx.store.put("Jobs", rv["job_id"], job) return {"review_id": review_id, "verdict": v.value, "lifecycle_state": st.value} + def demo_available(c: dict[str, Any]) -> bool: + if ctx.mode == "aws": # demo inputs are seeded to the results bucket by scripts/seed_aws.py + return bool(c.get("seeded", True)) + return (REPO / c["video"]).exists() + + def demo_input(c: dict[str, Any], field: str, job_id: str) -> Path | None: + rel = c.get(field) + if not rel: + return None + if ctx.mode == "aws": + dst = ctx.workdir / job_id / Path(rel).name + return ctx.storage.get_file(f"demo/{Path(rel).name}", dst) + return REPO / rel + @app.get("/v1/demo-cases") def demo_cases() -> list[dict[str, Any]]: return [{k: c[k] for k in ("id", "title", "description", "expected_verdict")} for c in load_demo_cases() - if (REPO / c["video"]).exists()] + if demo_available(c)] @app.post("/v1/demo-cases/{case_id}/run") def run_demo(case_id: str) -> dict[str, Any]: case = next((c for c in load_demo_cases() if c["id"] == case_id), None) - if case is None or not (REPO / case["video"]).exists(): + if case is None or not demo_available(case): raise HTTPException(404, "unknown demo case") job_id = uuid.uuid4().hex[:12] key = f"uploads/{job_id}/input.mp4" - ctx.storage.put_file(key, REPO / case["video"]) - sidecar = json.loads((REPO / case["sidecar"]).read_text()) if case.get("sidecar") else None + try: + video = demo_input(case, "video", job_id) + side_p = demo_input(case, "sidecar", job_id) + except Exception as e: + raise HTTPException(404, "demo input not available") from e + assert video is not None + ctx.storage.put_file(key, video) + sidecar = json.loads(side_p.read_text()) if side_p else None ctx.store.put("Jobs", job_id, {"job_id": job_id, "status": "queued", "upload_key": key, "claim": case.get("claim") or {}, "sidecar": sidecar, "demo_case": case_id, "created": datetime.now(tz=UTC).isoformat()}) @@ -234,6 +254,11 @@ def files(path: str) -> FileResponse: raise HTTPException(404, "not found") return FileResponse(p) + dist = REPO / "web" / "dist" + if dist.exists(): # the Verify web app, served from the same origin as the API + from fastapi.staticfiles import StaticFiles + + app.mount("/", StaticFiles(directory=dist, html=True), name="web") return app diff --git a/api/local.py b/api/local.py index f3685cf..b55f28d 100644 --- a/api/local.py +++ b/api/local.py @@ -10,7 +10,6 @@ from pathlib import Path import uvicorn -from fastapi.staticfiles import StaticFiles REPO = Path(__file__).resolve().parents[1] @@ -24,10 +23,7 @@ def main() -> None: os.environ["GRIDPRINT_REVIEWER_TOKEN"] = tok_file.read_text().strip() from api.app import create_app - app = create_app() - dist = REPO / "web" / "dist" - if dist.exists(): - app.mount("/", StaticFiles(directory=dist, html=True), name="web") + app = create_app() # also serves web/dist when it has been built print(f"GRIDPRINT local: http://127.0.0.1:8000 (reviewer token in {tok_file})") uvicorn.run(app, host="127.0.0.1", port=int(os.environ.get("PORT", "8000"))) diff --git a/api/worker.py b/api/worker.py index 0f68d72..07f5352 100644 --- a/api/worker.py +++ b/api/worker.py @@ -28,7 +28,9 @@ def make_planner(ctx: Context) -> Any: if cfg["agent"]["planner"] == "bedrock" and cfg["bedrock"]["enabled"]: from agent.planner_bedrock import BedrockPlanner, make_client - client = make_client(cfg["aws"]["profile"] if ctx.mode == "local" else None, cfg["aws"]["region"]) + b = cfg["bedrock"] + profile = cfg["aws"]["profile"] if ctx.mode == "local" else None + client = make_client(profile, b.get("region", cfg["aws"]["region"]), b.get("role_arn"), b.get("external_id")) return BedrockPlanner(client, cfg["bedrock"]["model_id"], cfg["bedrock"]["max_tokens"]) return FallbackPlanner() @@ -68,13 +70,21 @@ def process_job(ctx: Context, job_id: str) -> None: publish_artifacts(ctx, job_id, jobdir) else: job.update({"status": "failed", "error": "no result produced; see trace"}) + publish_artifacts(ctx, job_id, jobdir) # the trace explains the failure job["user_requests"] = [r for r in ctx.store.scan("UserRequests") if r.get("job_id") == job_id] ctx.store.put("Jobs", job_id, job) except Exception as e: job["status"] = "failed" job["error"] = f"{type(e).__name__}: {e}" (jobdir / "error.txt").write_text(traceback.format_exc()) + print(traceback.format_exc(), flush=True) # also to CloudWatch ctx.store.put("Jobs", job_id, job) + try: + ctx.storage.put_file(f"jobs/{job_id}/error.txt", jobdir / "error.txt") + if (jobdir / "trace.json").exists(): + ctx.storage.put_file(f"jobs/{job_id}/trace.json", jobdir / "trace.json") + except Exception: + pass def publish_artifacts(ctx: Context, job_id: str, jobdir: Path) -> None: diff --git a/config/default.yaml b/config/default.yaml index 44477e8..2c40160 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -3,11 +3,18 @@ aws: profile: gridprint region: us-east-1 # DECISIONS D-005 + resource_prefix: orchard # neutral name for all AWS resources in the deployment account (D-021) + worker_instance_type: c8g.xlarge # 4 vCPU Graviton4; fits the account on-demand vCPU quota (D-024) + worker_max_instances: 1 bedrock: - enabled: false # turned on after V6 is verified with the gridprint profile (H1) - # Inference-profile ID as listed by AWS (see docs/VERIFICATION.md V6). Re-check with - # `aws bedrock list-inference-profiles --profile gridprint` before enabling. - model_id: global.anthropic.claude-sonnet-4-6 + enabled: true + # Amazon Nova Lite through a role in the model account (D-026). Anthropic models on Bedrock need a + # Marketplace payment method that neither account has yet; Nova is billed directly by AWS. + model_id: us.amazon.nova-lite-v1:0 + in_account_model_id: global.anthropic.claude-sonnet-4-6 # AI stack policy; usable once Claude access is granted + region: us-east-1 + role_arn: arn:aws:iam::768669378827:role/orchard-planner + external_id: orchard-planner max_tokens: 1024 limits: max_file_bytes: 524288000 # 500 MiB @@ -59,7 +66,8 @@ engine: segment_offset_tolerance_s: 3 claim_min_tolerance_s: 60 continuity: - smooth_window_s: 4.0 + jump_window_s: 1.0 # frames on each side of a boundary for the two line fits (D-023) + min_frame_snr: 1.0 # linear; frames below 0 dB fit SNR are left out of the phase series (D-022) segmentation: model_path: models/human_segmentation_pphumanseg_2023mar.onnx input_size: [192, 192] @@ -75,7 +83,7 @@ engine: agent: max_iterations: 6 max_tool_calls: 30 - planner: fallback # fallback | bedrock + planner: bedrock # bedrock | fallback; fallback also covers any Bedrock error per step evaluation: target_false_contradicted_rate: 0.01 # DECISIONS D-019 (written before any test data was examined) grid: diff --git a/config/demo_cases.yaml b/config/demo_cases.yaml index 0a20993..3d61cba 100644 --- a/config/demo_cases.yaml +++ b/config/demo_cases.yaml @@ -3,7 +3,7 @@ # `source: human` inputs are recorded by people (H4, H5, H7) and placed at the listed path. cases: - id: capture-verified - title: "Capture clip with sensor sidecar (SYNTHETIC stand-in until H5)" + title: "Capture clip with sensor sidecar (SYNTHETIC test clip)" description: "Expected VERIFIED with an estimated recording time." expected_verdict: VERIFIED source: synthetic @@ -15,12 +15,20 @@ cases: expected_verdict: CONTRADICTED source: synthetic video: data/demo/cut-1s.mp4 + claim: + claimed_zone: SIM50 + claimed_time_utc: "2026-01-20T10:00:00Z" + claimed_time_tolerance_s: 300 - id: insertion title: "Insertion from another time (SYNTHETIC)" description: "Expected CONTRADICTED with SEGMENT_OFFSET_INCONSISTENT." expected_verdict: CONTRADICTED source: synthetic video: data/demo/insertion.mp4 + claim: + claimed_zone: SIM50 + claimed_time_utc: "2026-01-20T10:00:00Z" + claimed_time_tolerance_s: 300 - id: dropped-frames title: "Dropped frames with original timestamps (SYNTHETIC)" description: "Expected VERIFIED with DROPPED_FRAMES_EXPLAINED." @@ -29,13 +37,13 @@ cases: video: data/demo/dropped-frames.mp4 sidecar: data/demo/dropped-frames.sidecar.json - id: generated-room - title: "Synthetic room video from a video generator (H5)" + title: "Synthetic room video from a video generator" description: "Expected INCONCLUSIVE (SIGNAL_ABSENT). A missing signal alone does not mean fake." expected_verdict: INCONCLUSIVE source: human video: data/demo/generated-room.mp4 - id: daylight-only - title: "Daylight-only room (SYNTHETIC stand-in until H5)" + title: "Daylight-only room (SYNTHETIC test clip)" description: "Expected INCONCLUSIVE." expected_verdict: INCONCLUSIVE source: synthetic @@ -47,7 +55,7 @@ cases: source: synthetic video: data/demo/agent-recovery.mp4 - id: face-composite - title: "Face composite of a consenting team member (EXPERIMENTAL, H7)" + title: "Face composite of a consenting team member (EXPERIMENTAL)" description: "Shows the experimental face-region check. Never the only basis for CONTRADICTED." expected_verdict: CONTRADICTED source: human diff --git a/config/zones.yaml b/config/zones.yaml index 1ccc0bd..a76c055 100644 --- a/config/zones.yaml +++ b/config/zones.yaml @@ -16,6 +16,10 @@ zones: nominal_hz: 60 timezone: America/New_York description: US Eastern Interconnection + - id: CN + nominal_hz: 50 + timezone: Asia/Shanghai + description: China (State Grid) synchronous area; used for public EV-ENFD reference data - id: SIM50 nominal_hz: 50 timezone: UTC diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 2b2be41..f38dcdc 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -8,7 +8,7 @@ Diagrams: ## 1. Build status -The components below are the target design. Their build state is tracked in `docs/PROGRESS.md`. Do not read this document as a statement that a component is deployed. At the time of writing, the engine package (`engine/gridprint_engine/`) and the simulator (`eval/simulator/`) contain code. The API, agent, MCP server, web app, infrastructure, Node, and Capture app are later milestones (M4–M8, M11). +The components below are the system design. All components have code in the repository. Build and acceptance state is tracked in `docs/PROGRESS.md`; the AWS deployment status is in `docs/TECHNICAL_REPORT.md` §6. On 2026-10-27 all stacks except Web (CloudFront, waiting for AWS account verification) are deployed in `us-east-1`. The engine runs stock OpenCV 5 on Graviton; COOL is supported by the stack but not active yet. ## 2. Components @@ -18,7 +18,7 @@ The components below are the target design. Their build state is tracked in `doc | GRIDPRINT Verify | `web/` | Amazon S3 + Amazon CloudFront, behind AWS WAF | Upload, demo cases, live grid tile, job page with live agent trace, verdict card, overlay player, reviewer screen. | | API | `api/` | FastAPI on Amazon ECS Fargate, `linux/arm64` | Jobs, presigned uploads, user input, reviewer queue and decisions, demo cases, live grid samples, health. | | Jobs queue | CDK in `infra/` | Amazon SQS | Decouples the API from the engine workers. Worker autoscaling uses queue depth. | -| Engine worker | `engine/`, `agent/`, `mcp_server/` | Amazon ECS on Graviton `c8g`, `linux/arm64`, OpenCV 5 + COOL | One task holds the agent orchestrator, the MCP client, and the MCP server `gridprint-engine` (stdio transport). | +| Engine worker | `engine/`, `agent/`, `mcp_server/` | Amazon ECS on Graviton4 `c8g`, `linux/arm64`, OpenCV 5 (stock wheel); COOL AMI supported by the stack when `aws.cool_ami_id` is set | One task holds the agent orchestrator, the MCP client, and the MCP server `gridprint-engine` (stdio transport). | | GRIDPRINT Engine | `engine/gridprint_engine/` | inside the worker | OpenCV 5 analysis stages S1–S15 (SPEC Part 8). Python package with no AWS imports. | | GRIDPRINT Agent | `agent/` | inside the worker | Plans and adapts the analysis with engine tools. Planner: Claude on Amazon Bedrock, or the deterministic fallback planner. Never writes a verdict. | | Reference Store | DynamoDB `GridLog` | Amazon DynamoDB | Grid frequency per zone and UTC second. Key `zone#YYYYMMDD` + epoch second. | @@ -59,11 +59,11 @@ Reference data flow: Node → MQTT/TLS → AWS IoT Core → topic rule → Lambd | S8 phase and frequency | `phase.py` | Per-frame least squares with `cv2.solve` (`DECOMP_SVD`, float64); DFT cross-check with `cv2.getOptimalDFTSize` and `cv2.dft` on a zero-filled, masked virtual time grid. | | S9 clock bias | `enf.py` | Demean both tracks. | | S10 matching | `match.py` | Normalized cross-correlation at 1 s steps, parabolic refinement, segment matching. | -| S11 continuity | `continuity.py` | Phase residuals, robust change points, gap explanation. | +| S11 continuity | `continuity.py` | Phase jump at each frame boundary from two-sided line fits (D-023), gap explanation. | | S12 face region (EXPERIMENTAL) | `facecheck.py` | `cv2.FaceDetectorYN`. No recognition, no embeddings, no stored crops. | | S13 verdict | `verdict.py` | Deterministic rules (SPEC 8.14). | | S14 overlay | `overlay.py` | Temporal band-pass, `cv2.applyColorMap`, `cv2.VideoWriter`. Labeled as not itself evidence. | -| S15 report | `report.py` (planned) | JSON, HTML, PDF. | +| S15 report | `report.py` | JSON, HTML, PDF. | The signal path uses float32 or float64 only (R6). Linear light comes before normalization (R7). Idle gaps between frames are zero-filled with a mask in the DFT cross-check, never interpolated. @@ -73,16 +73,16 @@ The engine has no AWS imports. All access to storage and data goes through three | Interface | Methods | Local implementation | AWS implementation | |---|---|---|---| -| `StorageAdapter` | `put_file`, `get_file`, `url` | `LocalStorage` (filesystem) | S3 (planned `adapters/aws.py`) | -| `GridLogAdapter` | `read(zone, t0, t1)`, `write(...)` | `LocalGridLog` (CSV per zone and UTC day; median over nodes) | DynamoDB `GridLog` (planned) | -| `JobStoreAdapter` | `put`, `get`, `scan` | local JSON files (DECISIONS D-013) | DynamoDB `Jobs`, `Reviews`, `TraceIndex` (planned) | +| `StorageAdapter` | `put_file`, `get_file`, `url` | `LocalStorage` (filesystem) | `S3Storage` in `api/aws_adapters.py` | +| `GridLogAdapter` | `read(zone, t0, t1)`, `write(...)` | `LocalGridLog` (CSV per zone and UTC day; median over nodes) | `DynamoGridLog` in `node/ingest/aws_gridlog.py` | +| `JobStoreAdapter` | `put`, `get`, `scan` | local JSON files (DECISIONS D-013) | `DynamoJobStore` in `api/aws_adapters.py` | `make local` uses the local implementations. All unit and integration tests run offline with them. ## 6. Agent and MCP - The orchestrator (`agent/orchestrator.py`) runs the planner loop. The planner is `planner_bedrock.py` (Claude on Amazon Bedrock, model ID pinned in `config/default.yaml`) or `fallback_planner.py` (rules for B1–B6). -- `config/default.yaml` sets `agent.planner: fallback` and `bedrock.enabled: false` until V6 is verified (see `docs/VERIFICATION.md`). +- `config/default.yaml` sets `agent.planner: bedrock` with `bedrock.model_id: us.amazon.nova-lite-v1:0`, called through a cross-account role (D-026). The fallback planner runs any step where the model call fails, and the trace records which planner made each decision. - `policies.py` enforces budgets and order rules in code, not only in the prompt. - The MCP server `gridprint-engine` runs in the same worker task as the orchestrator. Transport: stdio. - Tools: see SPEC 9.2. The agent loop is in `diagrams/agent_workflow.mmd`. @@ -91,7 +91,7 @@ The engine has no AWS imports. All access to storage and data goes through three - Production engine containers are `linux/arm64` and run on Graviton `c8g` instances. The API runs on Fargate `linux/arm64`. - The `linux/amd64` engine image exists only for the x86 baseline of the COOL benchmark (SPEC Part 12, configuration A). It is not deployed in the production stack. -- Both images are built from the same Dockerfile with Docker buildx. They use the same engine code, the same pinned Python dependencies (`uv.lock`), and the same OpenCV 5 release (`opencv-python-headless==5.0.0.93`, DECISIONS D-003). Configuration C replaces stock OpenCV 5 with the COOL build on Graviton. +- Images are built by `infra/build_images.py`: the `linux/arm64` images on a Graviton EC2 build instance, from the committed tree, and pushed to Amazon ECR. The `linux/amd64` engine image (benchmark only) is built with `--amd64`. All engine images use `infra/docker/engine.Dockerfile`, the same engine code, the same pinned Python dependencies (`uv.lock`), and the same OpenCV 5 release (`opencv-python-headless==5.0.0.93`, DECISIONS D-003). Configuration C replaces stock OpenCV 5 with the COOL build on Graviton. - The benchmark parity check compares ENF tracks and verdicts from A, B, and C. - If COOL cannot be obtained (V5), production runs stock OpenCV 5 on Graviton, the reason is documented, and the COOL award is not claimed. diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index c8c6420..9870a93 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -87,3 +87,29 @@ Format: ID — date — decision — reason — alternatives considered. ## D-020 — 2026-10-26 — Scope of the placeholder-marker scan in the final check - `scripts/final_check.py` item 5 searches the repository for the placeholder marker of R8. Two files must contain the marker word to do their job: `eval/calibrate_thresholds.py` (writes it when calibration used SYNTHETIC data only) and `scripts/final_check.py` (searches for it). Both are excluded from the scan; everything else, including `config/` and `docs/`, is scanned. `data/`, `work/`, `results/`, `.venv/` and `node_modules/` are not repository content and are skipped. +## D-021 — 2026-10-26 — Deployment account and neutral resource names +- The owner chose the AWS account behind the local `kittu` CLI credentials for the deployment; the CLI profile `gridprint` (the only profile the code uses) now holds those credentials. Account 643700104680, region `us-east-1`. +- At the owner's request every physical AWS resource name in that account uses a neutral prefix, `aws.resource_prefix: orchard` (stacks `Orchard*`, tables `orchard-*`, IoT topic `orchard//`, dashboard, KMS alias). Code and documents keep the product name. +- Bedrock in that account returns "account is currently being verified" and the Claude models are not yet authorized (`get-foundation-model-availability`: `authorizationStatus NOT_AUTHORIZED`). Per V6, the deterministic fallback planner runs (`bedrock.enabled: false`) until access is granted; `scripts/verify_aws.py` re-checks it. + +## D-022 — 2026-10-26 — Direct registration to the reference frame; continuity only on usable frames +- Found by `engine/gridprint_engine/tests/test_video.py`: on a fixed 160×120 clip the chained per-frame homographies accumulated several pixels of false drift over 1,200 frames, the warp smeared the scene, and band SNR fell from 11.5 dB to −2.6 dB (all three clips SIGNAL_ABSENT). +- Each frame is now registered directly to the middle frame (features from the reference, pyramidal LK, `findHomography` with USAC_MAGSAC); chaining is a fallback for frames whose direct registration fails. Warping is used only when the 90th-percentile corner displacement exceeds 1 px. +- Continuity (S11) skips tracks marked SIGNAL_ABSENT and uses only frames whose own fit SNR is at least 1 (0 dB) (`engine.continuity.min_frame_snr`). Without this, noise-level phases produced edit candidates on clips with no signal. + +## D-023 — 2026-10-26 — Continuity: two-sided line fits replace single frame differences +- Found by the video integration test: a re-timed 1 s cut gave a true phase jump of 0.44 rad at the cut, but the single-difference detector (D-014) reached only z = 3.8 because per-frame phase noise was about 0.1 rad. +- New detector: at each frame boundary, fit a line to the phase on each side (frames within `engine.continuity.jump_window_s` = 1 s, at least 8 frames per side), evaluate both at the boundary, and take the wrapped difference as the jump. Its standard error comes from the two fits; z = |jump|/σ. Local maxima within ±1 s are the change points. Thresholds `edit_significance_sigma` and `edit_min_rad` stay in `thresholds.yaml`. +- Gap rule unchanged: a boundary with an S1 gap whose jump is not significant is DROPPED_FRAMES_EXPLAINED. + +## D-024 — 2026-10-27 — Worker instance size within the account vCPU quota +- The deployment account has an on-demand standard vCPU quota of 8 (`service-quotas get-service-quota L-1216C47A`). The ASG could not launch `c8g.2xlarge` (8 vCPU) because another 2-vCPU instance already runs in the account ("You have requested more vCPU capacity than your current vCPU limit of 8"). +- Workers now use `c8g.xlarge` (4 vCPU, Graviton4) with at most one instance (`aws.worker_instance_type`, `aws.worker_max_instances`). A quota increase request (H1) restores the default sizes. The benchmark protocol (Part 12) still specifies equal-vCPU pairs and runs separately. + +## D-025 — 2026-10-27 — Public HTTPS endpoint through API Gateway while CloudFront is blocked +- CloudFront in the deployment account needs AWS account verification (H1). The API stack adds an API Gateway HTTP API with a VPC link to the internal ALB; it gives a public HTTPS URL with no CloudFront. Stage throttling limits request rates (WAF cannot attach to HTTP APIs). The uploads bucket allows browser PUTs with presigned URLs from any origin; the signature limits what can be uploaded. + +## D-026 — 2026-10-27 — Planner model: Amazon Nova Lite through a cross-account role +- Anthropic models on Bedrock are sold through AWS Marketplace; both accounts refuse them (no valid payment instrument on one, account verification pending on the other). SPEC Part 7 names an Anthropic model; V6's fallback order is another region, then the deterministic planner. Amazon Nova Lite (first-party, tool use through Converse) works in the second account today, and in a test it chose the B1 action (re-rank patches after SIGNAL_WEAK). +- Decision: the LLM planner uses `us.amazon.nova-lite-v1:0` through role `orchard-planner` in account 768669378827. The role allows only `bedrock:InvokeModel` on that model and can be assumed only from the deployment account with the external ID. The worker's policy allows `sts:AssumeRole` on that one role. The deterministic fallback planner stays in place and takes over for any step where the model call fails; the trace marks every step's planner. +- Switch back to Claude by changing `bedrock.model_id` once Marketplace access works; no code change. diff --git a/docs/HUMAN_TASKS.md b/docs/HUMAN_TASKS.md index 620bcf2..91eda04 100644 --- a/docs/HUMAN_TASKS.md +++ b/docs/HUMAN_TASKS.md @@ -4,24 +4,23 @@ Status values: `BLOCKED-HUMAN` (waiting for a person), `DONE` (validated by scri --- -## H1 — AWS account, credentials, region, budget alarm, Bedrock access, COOL subscription — BLOCKED-HUMAN +## H1 — AWS account, account verification, quotas, Bedrock access, COOL subscription — BLOCKED-HUMAN +Done (2026-10-27): steps 1–3 (account 643700104680, deployer access, CLI profile `gridprint` in `us-east-1`) and the CDK bootstrap. Nine stacks are deployed (`docs/TECHNICAL_REPORT.md` §6). + +Done steps, for reference: 1. Sign in to the AWS account you will use for the competition. 2. Create an IAM user or IAM Identity Center permission set for deployment. Attach the policy in `infra/iam/deployer-policy.json`. -3. On this machine, create the CLI profile named `gridprint` in region `us-east-1`: - ```bash - aws configure --profile gridprint - ``` - (or `aws configure sso --profile gridprint`). Do not put keys in the repository. -4. Create a monthly budget with an email alarm at 80 % of your limit: AWS Console → Billing → Budgets → Create budget → Cost budget. -5. Bedrock model access: Bedrock console (us-east-1) → Model access → enable Anthropic Claude Sonnet 4.6. Accept the Anthropic use-case form if asked. -6. COOL: open the AWS Marketplace listing "Cloud Optimized OpenCV For AWS Graviton4" (`https://aws.amazon.com/marketplace/pp/prodview-fdvbfiewzuehs`) and subscribe. Note the AMI ID for us-east-1. -7. Put the AMI ID in `config/default.yaml` under `aws.cool_ami_id` (create the key). -8. Bootstrap CDK once: - ```bash - npx aws-cdk@2 bootstrap --profile gridprint - ``` -9. Validate: +3. On this machine, create the CLI profile named `gridprint` in region `us-east-1`: `aws configure --profile gridprint` (or `aws configure sso --profile gridprint`). Do not put keys in the repository. +4. Bootstrap CDK once: `npx aws-cdk@2 bootstrap --profile gridprint`. + +Open items (each needs the account owner): +1. **AWS account verification.** AWS Console → Support Center → open the case or notice about account verification and complete it. This unblocks CloudFront (the Web stack) and Amazon Bedrock. +2. **EC2 vCPU quota increase.** AWS Console → Service Quotas → Amazon EC2 → "Running On-Demand Standard (A, C, D, H, I, M, R, T, Z) instances" (`L-1216C47A`) → Request increase. Ask for at least 32 vCPUs. This restores the default worker size and allows the benchmark instances (D-024). +3. **Bedrock model access.** After verification: Bedrock console (`us-east-1`) → Model access → enable Anthropic Claude Sonnet 4.6. Accept the Anthropic use-case form if asked. Then set `bedrock.enabled: true` and `agent.planner: bedrock` in `config/default.yaml`. +4. **COOL subscription.** Open the AWS Marketplace listing "Cloud Optimized OpenCV For AWS Graviton4" (`https://aws.amazon.com/marketplace/pp/prodview-fdvbfiewzuehs`) and subscribe. Note the AMI ID for `us-east-1` and put it in `config/default.yaml` under `aws.cool_ami_id`. +5. **Budget alarm.** If it does not exist yet: AWS Console → Billing → Budgets → Create budget → Cost budget, with an email alarm at 80 % of your limit. +6. Validate: ```bash make verify-aws ``` @@ -65,6 +64,16 @@ Status values: `BLOCKED-HUMAN` (waiting for a person), `DONE` (validated by scri ## H5 — Record the real dataset — BLOCKED-HUMAN +### H5-a — Download the public ENF video datasets (needs your login; the agent cannot create accounts) +Both sets have real rolling-shutter videos plus a recorded mains reference. +1. EV-ENFD (Xu et al., CVPR 2023, code MIT): open the TeraBox link in https://github.com/xlx-creater/E-ENF (README, "Data"). + Download the video sequences and the `ENF_Reference` folder (each `.wav` is one hour of grid voltage). +2. Choi, Wong, Su, Wu rolling-shutter set: log in at https://ieee-dataport.org/documents/rolling-shutter-videos-enf-extraction + and download `vids.zip` (and `mats.zip` if offered). +3. Upload the downloaded archives unchanged to your Google Drive folder `GRIDPRINT-data/` and tell the agent. + The agent copies them to `data/public/` (git-ignored), converts the mains `.wav` references to 1 s GridLog + entries, and runs the evaluation on them. + Follow `eval/DATASET.md` (shot list, manifest template, consent form template). Validate with: ```bash make validate-dataset diff --git a/docs/IDEAS.md b/docs/IDEAS.md index ac690e0..6ee2899 100644 --- a/docs/IDEAS.md +++ b/docs/IDEAS.md @@ -2,4 +2,6 @@ Ideas that are outside SPEC.md go here and are not implemented (SPEC 0.3). -- (none yet) +- Capture band meter for 60 Hz zones at 30 fps: the 120 Hz bands barely move between frames, so a running-mean meter reads low. A spatial band estimator that uses the readout time could fix this (from the Capture build notes). +- The same 60 Hz / 30 fps case limits the engine: the temporal mean keeps the static bands. A capture mode at a frame rate whose alias is not near 0 Hz would help (not in SPEC; not built). +- COOL Docker Marketplace product as the worker base image instead of mounting host libraries (infra notes; not checked). diff --git a/docs/JUDGE_GUIDE.md b/docs/JUDGE_GUIDE.md index 1434b3b..00e8bdd 100644 --- a/docs/JUDGE_GUIDE.md +++ b/docs/JUDGE_GUIDE.md @@ -2,7 +2,11 @@ GRIDPRINT checks if a video is real by reading the power-grid frequency from the flicker of indoor lights. The camera's rolling shutter records the flicker as faint horizontal bands. GRIDPRINT compares the measured frequency track with a reference log of the real grid. -Endpoint: (set after M8 deploy) +Endpoint: + +The public HTTPS URL is added on this line at deployment. + +**Live demonstration.** The team runs the full system with `make local` in a live screen-share on request. This runs the same engine, agent and web app as the AWS deployment, with local storage and the deterministic planner. ## 1. What you can do on the endpoint @@ -25,9 +29,9 @@ GRIDPRINT gives exactly three verdicts: ## 3. Demo cases -All demo cases run through the deployed pipeline (`POST /v1/demo-cases/{id}/run`). No result is stored in advance. +All demo cases run through the live pipeline (`POST /v1/demo-cases/{id}/run`). No result is stored in advance. The latest measured verdicts are in `results/`. The "Target verdict" column is the result that `SPEC.md` defines for each case; the verdict you see is the one the pipeline measures on the run. -| # | Case | Expected result | +| # | Case | Target verdict | |---|---|---| | 1 | Capture clip | `VERIFIED` with recording time | | 2 | Same clip with a one-second cut | `CONTRADICTED`, then reviewer approval | @@ -38,9 +42,9 @@ All demo cases run through the deployed pipeline (`POST /v1/demo-cases/{id}/run` | 7 | Agent recovery: first pass `SIGNAL_WEAK`, the agent selects new patches, the signal locks | Final verdict from the second pass; the trace shows the change of plan | | 8 | Face-composite clip (consenting team members) | Face-region result labeled `EXPERIMENTAL`. This result is never the only basis for `CONTRADICTED`. | -## 4. Reviewer demo login +## 4. Reviewer login -A reviewer demo account exists for judges. The username and password are in the submission form. They are not in this repository. +Reviewer credentials for the endpoint are provided to judges on request. They are not in this repository. With `make local`, the reviewer token is in `work/local_reviewer_token.txt`. Use the reviewer screen to approve or reject a `CONTRADICTED` result. A rejection makes the report final with verdict `INCONCLUSIVE` and reason `REVIEWER_REJECTED`. The note you write is stored. diff --git a/docs/LICENSES.md b/docs/LICENSES.md index de99f38..2afdd35 100644 --- a/docs/LICENSES.md +++ b/docs/LICENSES.md @@ -44,3 +44,9 @@ Record the result here and change the status to "Confirmed". Package metadata is | NESO System Frequency data (zone GB) | NESO Open Data Licence | To confirm on download (see `docs/VERIFICATION.md` V8) | | Face-swap tool for H7 composites | Record at selection time | Not chosen yet (see `eval/FACE_COMPOSITES.md`) | | Public video generator for synthetic room videos (SPEC 11.3) | Record tool, prompt, license, date | Not chosen yet | + +## Public media used for demonstration + +| Item | Source | License | Use | +|---|---|---|---| +| "Fluorescent beat effect.ogv" (640×480, 30 fps, 15 s; SHA-256 `bfa8f2d7e3fcb3b948068527cb47d6b8808e35164a34d46b84214d77d5af5db7`) | https://commons.wikimedia.org/wiki/File:Fluorescent_beat_effect.ogv | CC BY-SA 3.0 (Wikimedia Commons) | Real-world flicker example. No recording time or grid zone is known, so it can show a measured flicker signal but never a time match. Stored outside the repository (`data/public/`). | diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index ff15972..f32991e 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -1,15 +1,71 @@ # GRIDPRINT progress -Current milestone: M1 (simulator) +Current milestone: M2/M3 hardening (video integration tests) and M8 (AWS deployment in progress). + +Status key: ACCEPTED (all criteria shown with evidence), OPEN (work remains), BLOCKED-HUMAN (waits for a task in `docs/HUMAN_TASKS.md`). ## M0 — Setup and verification — ACCEPTED (2026-10-26) | Criterion | Evidence | |---|---| -| Repo layout per Part 6 | Directories created: `config/ engine/gridprint_engine/{adapters,tests} mcp_server/ agent/{prompts,tests} api/ web/ capture-android/ node/{firmware,hardware,ingest} calibration_rig/ infra/ eval/simulator/ bench/ scripts/ docs/diagrams/ results/ .github/workflows/` | -| CI runs lint, type check, tests, claims_lint | `.github/workflows/ci.yml` (jobs `python`: `make lint` = ruff + mypy + `claims_lint` + `check_numbers_sourced`, `make test` = pytest; `web`: typecheck, tests, axe; `scan`: trivy). Local run: `ruff check scripts tests` → `All checks passed!`; `mypy scripts` → `Success: no issues found in 6 source files`; `pytest tests` → `2 passed`; `claims_lint` → `5 files scanned, 0 violations`. No Git remote exists yet, so CI has not run on GitHub (needs the repository to be pushed — part of H9 prep). | -| VERIFICATION.md V1–V12 resolved | `docs/VERIFICATION.md`: V1, V2, V3, V9, V10 VERIFIED by command; V4 API VERIFIED; V8 partial (NESO GB 1 s); V5, V6, V7, V11, V12 BLOCKED-HUMAN with instructions. `scripts/verify_aws.py` → `AWS FAIL: The config profile (gridprint) could not be found` (H1). | +| Repo layout per Part 6 | All Part 6 directories exist on `main` (PRs #1–#10). | +| CI runs lint, type check, tests, claims_lint | `.github/workflows/ci.yml`; green runs on PRs #1–#8 (python, web, scan jobs). | +| VERIFICATION.md V1–V12 resolved | `docs/VERIFICATION.md`; V5/V6/V7/V11/V12 carry BLOCKED-HUMAN or FALLBACK notes with instructions, as M0 allows. | | HUMAN_TASKS.md lists H1–H9 | `docs/HUMAN_TASKS.md` | -| REQUIREMENTS_TRACE.md created | `docs/REQUIREMENTS_TRACE.md` (18 lines, all OPEN) | +| REQUIREMENTS_TRACE.md created | `docs/REQUIREMENTS_TRACE.md` | + +## M1 — Simulator — OPEN + +- Done: `eval/simulator/` renders every factor of SPEC 11.1 with ground truth (`eval/tests/test_simulator.py`, 3 passed). +- Remaining: commit the small sample set (`python -m eval.simulator.make_samples` → `results/synthetic_samples/`). + +## M2 — Engine S1–S10 — OPEN + +- Done: unit tests for linearization, float32/float64 types, idle-gap zero-fill, clock-bias removal, estimator agreement, readout, matching (`engine/gridprint_engine/tests/test_units.py`). +- Video integration tests: 8 of 8 pass (`engine/gridprint_engine/tests/test_video.py`). Motion now registers each frame directly to the reference frame (D-022). +- Stress and fuzz tests pass (`tests/test_stress.py`). +- Remaining for formal acceptance: commit the evaluation results under `results/`. + +## M3 — Engine S11–S15 — OPEN + +- Done: verdict rules and R1–R3, R9 tests; report HTML/PDF/JSON and overlay generation; claims_lint on templates. +- Video integration tests: 8 of 8 pass (`engine/gridprint_engine/tests/test_video.py`), including the re-timed cut (`CONTRADICTED`) and dropped frames (`DROPPED_FRAMES_EXPLAINED`). Continuity uses two-sided line fits (D-023, replaces D-014). +- Remaining for formal acceptance: commit the evaluation results under `results/`. + +## M4 — Node and reference — BLOCKED-HUMAN (H2) + +- Done: firmware, hardware docs with the R11 warning, ingest Lambda, GridLog adapter, simulated publisher, data-quality script, NESO and WAV reference importers (42 tests pass). +- Remaining: ≥ 24 h of real Node data with GPS lock (H2). + +## M5 — Calibration — BLOCKED-HUMAN (H3) + +- Done: rig firmware (200 Hz, D-017), filming checklist, `calibration_rig/fit_readout.py` with tests. + +## M6 — Capture app — BLOCKED-HUMAN (H4) + +- Done: Kotlin Camera2 app, sidecar writer, band meter, device check, validator; JVM and Python tests pass. + +## M7 — Agent — OPEN + +- Done: MCP server (12 tools, stdio test), fallback planner (B1–B6), Bedrock planner, policies, trace; `agent/tests` (13 passed); adversarial-planner fuzz (200 runs) in `tests/test_stress.py`. +- Remaining: saved traces from real runs showing B1–B6; the evaluation run commits them under `results/`. Bedrock planner: Amazon Nova Lite through a cross-account role (D-026); the fallback planner covers failed model calls. + +## M8 — AWS deployment — OPEN + +- Account 643700104680, region `us-east-1`, profile `gridprint`, resource prefix `orchard` (D-021). +- Deployed: Network, Storage (KMS), Data, Ai, Auth, Compute (Graviton4 `c8g.xlarge` workers, max 1 per D-024; SQS with DLQ), Api (Fargate `linux/arm64` behind an internal ALB), Iot, Observability. +- Images: `linux/arm64`, built on a Graviton EC2 build instance by `infra/build_images.py`. +- In progress: public HTTPS endpoint through API Gateway; end-to-end run on AWS. +- Blockers: CloudFront (Web stack) needs AWS account verification (H1); EC2 vCPU quota increase (H1). + +## M9 — Real evaluation — BLOCKED-HUMAN (H5–H7) + +- Synthetic evaluation and threshold calibration tooling is in place; the real dataset is needed (public datasets need the owner's login, H5-a). + +## M10 — COOL benchmark — OPEN / BLOCKED-HUMAN (COOL subscription) + +## M11 — Demo and UX — OPEN + +## M12 — Docs and video — OPEN (video: H8) -Notes: host has no GNU make → `make`/`make.cmd` shims (D-002). Host network ran at 275–551 B/s to PyPI (`curl -w %{speed_download}`), so `.venv` was bootstrapped offline (D-004); `uv.lock` pins all versions. +## M13 — Final check — OPEN (`scripts/final_check.py` reports the open items) diff --git a/docs/REQUIREMENTS_TRACE.md b/docs/REQUIREMENTS_TRACE.md index 42c45fb..6e0d8fa 100644 --- a/docs/REQUIREMENTS_TRACE.md +++ b/docs/REQUIREMENTS_TRACE.md @@ -2,23 +2,25 @@ Every line needs an evidence link before the final stop. +Status values: `MET` (the evidence exists in the repository or on the account now), `PARTIAL` (the note says what is missing), `OPEN` (no evidence yet). + | Section | Requirement | Status | Evidence | |---|---|---|---| -| 3.1 | OpenCV 5 performs substantive image/video analysis (Engine S1–S14) | OPEN | — | -| 3.1 | A meaningful component runs on AWS (Part 10.6) | OPEN | — | -| 3.2 | Technical report: problem, users, architecture, OpenCV 5 implementation, AWS deployment, evaluation, limitations, responsible use | OPEN | — | -| 3.2 | Judge-accessible code repository | OPEN | — | -| 3.2 | Pinned dependencies plus clear build, deployment and test instructions | OPEN | — | -| 3.2 | Architecture diagram showing OpenCV 5, AWS, COOL and agent components | OPEN | — | -| 3.2 | Working web endpoint (Verify) or arranged live screen-share | OPEN | — | -| 3.2 | Public or unlisted video, max 5 minutes | OPEN | — | -| 3.2 | Evaluation evidence, including failure cases and limitations | OPEN | — | -| 3.3 | COOL executes the claimed core workload on AWS Graviton | OPEN | — | -| 3.3 | Report COOL version and AWS instance/deployment configuration | OPEN | — | -| 3.3 | Reproducible evaluation method, inputs, baselines and results | OPEN | — | -| 3.3 | Evidence that COOL executes the claimed workload | OPEN | — | -| 3.3 | Explain any x86/Arm coexistence | OPEN | — | -| 3.4 | Multi-step perception → decision → action loop where OpenCV 5 results change a later plan, tool call, action or human-approval request | OPEN | — | -| 3.4 | Agent workflow diagram (perception, decision/orchestration, action) | OPEN | — | -| 3.4 | Trace or demonstration showing OpenCV 5 output changing a later decision | OPEN | — | -| 3.4 | Evaluation of task success, failure handling, observability and human control | OPEN | — | +| 3.1 | OpenCV 5 performs substantive image/video analysis (Engine S1–S14) | MET | `engine/gridprint_engine/*.py` (stages S1–S15); OpenCV 5 usage map in `docs/TECHNICAL_REPORT.md` §5; video integration tests `engine/gridprint_engine/tests/test_video.py` | +| 3.1 | A meaningful component runs on AWS (Part 10.6) | PARTIAL | Deployed stacks (Network, Storage, Data, Ai, Auth, Compute, Api, Iot, Observability) from `infra/`; status table in `docs/TECHNICAL_REPORT.md` §6. Missing: a completed end-to-end job on AWS (in progress). | +| 3.2 | Technical report: problem, users, architecture, OpenCV 5 implementation, AWS deployment, evaluation, limitations, responsible use | PARTIAL | `docs/TECHNICAL_REPORT.md`. Missing: result tables in §7–§10 from the evaluation and benchmark runs. | +| 3.2 | Judge-accessible code repository | MET | https://github.com/vivekyarra/Alibi (public) | +| 3.2 | Pinned dependencies plus clear build, deployment and test instructions | MET | `uv.lock`, `pyproject.toml`, `web/package-lock.json`; `README.md` (quick start, deploy, evaluate); `Makefile`; `.github/workflows/ci.yml` | +| 3.2 | Architecture diagram showing OpenCV 5, AWS, COOL and agent components | MET | `docs/diagrams/architecture.svg` (source `docs/diagrams/architecture.mmd`), `docs/diagrams/agent_workflow.svg`; COOL is shown as supported by the stack, not active (§3.3 lines). | +| 3.2 | Working web endpoint (Verify) or arranged live screen-share | OPEN | Public HTTPS endpoint pending (CloudFront needs AWS account verification; API Gateway endpoint in progress). Live screen-share with `make local` offered in `docs/JUDGE_GUIDE.md`. | +| 3.2 | Public or unlisted video, max 5 minutes | OPEN | Script `docs/VIDEO_SCRIPT.md`; recording is human task H8. | +| 3.2 | Evaluation evidence, including failure cases and limitations | PARTIAL | Limitations in `docs/TECHNICAL_REPORT.md` §11; simulator `eval/simulator/`; evaluation tools `eval/run_eval.py`, `eval/make_report.py`; first SYNTHETIC run folder `results/eval/2026-10-27/`. Missing: committed result tables and the failure gallery. | +| 3.3 | COOL executes the claimed core workload on AWS Graviton | OPEN | COOL not active: Marketplace subscription pending (H1). The engine runs stock OpenCV 5.0.0 on Graviton. The stack uses a COOL AMI when `aws.cool_ami_id` is set. | +| 3.3 | Report COOL version and AWS instance/deployment configuration | PARTIAL | AWS configuration: `docs/TECHNICAL_REPORT.md` §6, `config/default.yaml`, `infra/gridprint_infra/compute.py`. Missing: COOL version (needs the COOL AMI). | +| 3.3 | Reproducible evaluation method, inputs, baselines and results | PARTIAL | Method and inputs: `bench/README.md`, `bench/configs.yaml`, `bench/inputs.txt`, `bench/run_bench.py`, `bench/prices.py`; `eval/`. Missing: benchmark results under `results/bench/`. | +| 3.3 | Evidence that COOL executes the claimed workload | OPEN | Collector `bench/cool_evidence.py` with tests `bench/tests/test_cool_evidence.py`; no evidence file until the COOL benchmark runs. | +| 3.3 | Explain any x86/Arm coexistence | MET | `docs/ARCHITECTURE.md` §7; `docs/TECHNICAL_REPORT.md` §6; `infra/build_images.py` (`linux/arm64` images, `--amd64` for benchmark baseline A only) | +| 3.4 | Multi-step perception → decision → action loop where OpenCV 5 results change a later plan, tool call, action or human-approval request | MET | `agent/orchestrator.py`, `agent/fallback_planner.py`, `agent/planner_bedrock.py`, `agent/policies.py`, `mcp_server/tools.py`, `mcp_server/server.py`; tests `agent/tests/test_agent.py` (B1–B6), `mcp_server/tests/test_mcp_stdio.py` | +| 3.4 | Agent workflow diagram (perception, decision/orchestration, action) | MET | `docs/diagrams/agent_workflow.svg` (source `docs/diagrams/agent_workflow.mmd`) | +| 3.4 | Trace or demonstration showing OpenCV 5 output changing a later decision | PARTIAL | Tested in `agent/tests/test_agent.py` (`test_b1_weak_signal_reselects_patches_and_reextracts` and others); trace writer `agent/trace.py`. Missing: saved traces from real runs under `results/` (the evaluation run commits them). | +| 3.4 | Evaluation of task success, failure handling, observability and human control | PARTIAL | Metrics code `eval/metrics.py`; human control tests `agent/tests/test_agent.py` (`test_b6_contradicted_requests_review_and_no_final_report`); planner fuzz `tests/test_stress.py`; CloudWatch dashboard and alarms in `infra/gridprint_infra/observability.py`. Missing: measured results under `results/`. | diff --git a/docs/TECHNICAL_REPORT.md b/docs/TECHNICAL_REPORT.md index 649b649..29d218f 100644 --- a/docs/TECHNICAL_REPORT.md +++ b/docs/TECHNICAL_REPORT.md @@ -2,7 +2,7 @@ GRIDPRINT checks if a video is real from the flicker of the lights in the room. It reads the power-grid frequency from rolling-shutter flicker bands with OpenCV 5 and compares it with a reference log of the real grid. -Status: this report is a working draft. Sections 7–10 receive their tables from `eval/make_report.py` after milestones M9 and M10. This report contains no measured number without a source tag (SPEC Part 14.3). +Status (2026-10-27): sections 1–6 and 11–14 describe the built system. Sections 7–10 receive their tables from `eval/make_report.py` and the benchmark run. This report contains no measured number without a source tag (SPEC Part 14.3). ## Contents @@ -61,7 +61,8 @@ See `docs/ARCHITECTURE.md` and the diagrams `docs/diagrams/architecture.mmd` and Summary: - **Clients:** GRIDPRINT Capture (Android) and GRIDPRINT Verify (web). - **API:** FastAPI on ECS Fargate `linux/arm64`. Presigned S3 uploads. Jobs go to an SQS queue. -- **Engine workers:** ECS on Graviton `c8g`, `linux/arm64`, OpenCV 5 with COOL. Each worker task holds the agent orchestrator, the MCP client, and the MCP server `gridprint-engine` (stdio). +- **Engine workers:** ECS on Graviton4 `c8g`, `linux/arm64`, OpenCV 5 (stock wheel). The stack also supports a COOL AMI when `aws.cool_ami_id` is set; COOL is not active yet (§6). Each worker task holds the agent orchestrator, the MCP client, and the MCP server `gridprint-engine` (stdio). +- **Planner:** an LLM on Amazon Bedrock through the Converse API with tool use (`agent/planner_bedrock.py`; Amazon Nova Lite through a cross-account role, D-026; the model ID is configuration, so a Claude model works unchanged), with the deterministic fallback planner taking over for any step where the model call fails; the fallback planner can also run alone (`agent/fallback_planner.py`). The Bedrock planner is implemented and tested against a recorded-shape client. Bedrock access on the account waits for AWS account verification, so all runs so far use the fallback planner (D-021). - **Engine:** Python package `engine/gridprint_engine`, stages S1–S15. No AWS imports. Storage, grid log, and job store are reached through adapter interfaces. - **Reference:** GRIDPRINT Node (ESP32, GPS PPS, certified plug-in AC-to-AC adapter) → AWS IoT Core → Lambda → DynamoDB `GridLog`. - **Human control:** Cognito reviewer login; reviewer decisions stored in DynamoDB `Reviews`. @@ -85,7 +86,7 @@ Pinned build: `opencv-python-headless==5.0.0.93` (OpenCV 5.0.0, main modules). A | S8 phase and frequency | Per-frame least-squares fit of a cosine and sine at the nominal flicker frequency; phase unwrapping over frames; frequency from weighted sliding phase slopes. Cross-check: DFT on a uniform virtual time grid with idle gaps zero-filled under a known mask (never interpolated). | | S9 clock bias | Demean the video track and the reference segment. Match variations only. | | S10 matching | Normalized cross-correlation at one-second steps, sub-second refinement, segment matching. | -| S11 continuity | Phase residual change points at frame boundaries; gaps that explain the residual give `DROPPED_FRAMES_EXPLAINED`, others give `EDIT_CANDIDATE`. | +| S11 continuity | Phase jump at each frame boundary from two-sided line fits (D-023); gaps that explain the jump give `DROPPED_FRAMES_EXPLAINED`, others give `EDIT_CANDIDATE`. | | S12 face region | EXPERIMENTAL. Band amplitude in face pixels vs background pixels of the same rows. Never the only basis for `CONTRADICTED`. | | S13 verdict | Deterministic rules (SPEC Part 8.14). | | S14 overlay | Visualization only, labeled "Visualization of measured flicker. Not itself evidence." | @@ -101,21 +102,22 @@ Each row lists the calls found in the engine source (`grep -o "cv2\.[A-Za-z_.]*" |---|---|---|---| | S1 | `io.py` | `cv2.VideoCapture` (`CAP_PROP_FRAME_COUNT`, `CAP_PROP_FRAME_WIDTH`, `CAP_PROP_FRAME_HEIGHT`) | Cross-check of frame count and size. Decoding and PTS use PyAV. | | S2 | `color.py` | `cv2.LUT` | Inverse transfer lookup table to float32 linear luminance. | -| S4 | `motion.py` | `cv2.goodFeaturesToTrack`, `cv2.calcOpticalFlowPyrLK`, `cv2.findHomography` with `cv2.USAC_MAGSAC`, `cv2.DISOpticalFlow_create` (`DISOPTICAL_FLOW_PRESET_FAST`), `cv2.dilate` | Homographies map each frame to the middle frame. | +| S4 | `motion.py` | `cv2.goodFeaturesToTrack`, `cv2.calcOpticalFlowPyrLK`, `cv2.findHomography` with `cv2.USAC_MAGSAC`, `cv2.DISOpticalFlow_create` (`DISOPTICAL_FLOW_PRESET_FAST`), `cv2.dilate` | Each frame is registered directly to the middle (reference) frame (D-022). | | S5 | `segment.py` | `cv2.dnn.readNet`, `cv2.dnn.blobFromImage`, `cv2.resize`, `cv2.cvtColor`, `cv2.dilate` | PP-HumanSeg ONNX model from opencv_zoo. | | S6 | `patches.py` | `cv2.warpPerspective` (`WARP_INVERSE_MAP`, `INTER_NEAREST`) | Patch masks mapped into each frame. Per-patch row sums use NumPy `np.add.reduceat` (not `cv2.reduce`). | | S7 | `rowsignal.py` | `cv2.warpPerspective` (`INTER_LINEAR`, `INTER_NEAREST`, `WARP_INVERSE_MAP`) | Temporal mean in scene coordinates. Masked moving average and division are NumPy (`np.divide` with `where`), float32. | | S8 | `phase.py`, `pipeline.py` | `cv2.solve` (`DECOMP_SVD`, float64), `cv2.dft` (`DFT_COMPLEX_OUTPUT`), `cv2.getOptimalDFTSize` | Least-squares phase fit; DFT cross-check. Detrending in S7 uses `np.linalg.lstsq`. | | S12 | `facecheck.py` | `cv2.FaceDetectorYN.create`, `cv2.cvtColor` | YuNet ONNX model from opencv_zoo. | | S14 | `overlay.py` | `cv2.applyColorMap` (`COLORMAP_TWILIGHT_SHIFTED`), `cv2.VideoWriter`, `cv2.VideoWriter_fourcc`, `cv2.resize`, `cv2.cvtColor`, `cv2.putText`, `cv2.polylines`, `cv2.line`, `cv2.rectangle` | Temporal band-pass at the flicker alias rate is a NumPy complex demodulation, not an OpenCV filter. | -| Calibration | `calibration_rig/fit_readout.py` (milestone M5) | Planned: band-period measurement with `cv2.dft` / `cv2.reduce` | Not yet implemented; update this row when M5 is accepted. | +| Calibration | `calibration_rig/fit_readout.py` (milestone M5) | `cv2.reduce` (`REDUCE_AVG`), `cv2.dft` (`DFT_COMPLEX_OUTPUT`), `cv2.getOptimalDFTSize`, `cv2.magnitude` | Band-period measurement on rig clips; readout time from the fitted band period. | ### Design choices recorded in `docs/DECISIONS.md` - Analysis grid: frames are block-averaged to a small grid; each block keeps the mean sensor row index of its rows, so timing is kept (D-006). - Flicker-compensated temporal mean (D-015). - Readout direction chosen by physical plausibility of the measured flicker frequency, then band SNR (D-016). -- Change-point detection: robust z-score of phase-residual differences (D-014). +- Motion: each frame is registered directly to the reference frame; chaining is only a fallback (D-022). +- Continuity: two-sided line fits of the phase at each frame boundary give the jump and its standard error (D-023, replaces D-014). ## 6. AWS deployment @@ -126,8 +128,8 @@ The infrastructure is AWS CDK v2 (Python) in `infra/` (SPEC Part 10.6). | Network | VPC, private subnets, endpoints for S3 and DynamoDB | | Storage | S3 buckets with SSE-KMS and lifecycle deletion | | Data | DynamoDB `Jobs`, `Reviews`, `GridLog`, `TraceIndex` | -| Compute | ECS cluster on Graviton `c8g` with COOL; SQS queue; autoscaling on queue depth | -| API | Fargate `linux/arm64` service behind a load balancer or HTTP API | +| Compute | ECS cluster on Graviton4 `c8g`, stock OpenCV 5 (COOL AMI used when `aws.cool_ami_id` is set); SQS queue with DLQ; autoscaling on queue depth | +| API | Fargate `linux/arm64` service behind an internal Application Load Balancer | | IoT | Thing type, certificates, policy, topic rule | | Auth | Cognito user pool with `reviewer` group | | Web | S3 + CloudFront; WAF rate limits | @@ -136,6 +138,29 @@ The infrastructure is AWS CDK v2 (Python) in `infra/` (SPEC Part 10.6). Region: `us-east-1` (DECISIONS D-005). `make deploy` creates all stacks; `make destroy` removes them. +### Deployment status (2026-10-27) + +Account 643700104680, region `us-east-1`, resource prefix `orchard` (D-021). + +| Stack | Status | Note | +|---|---|---| +| Network | Deployed | | +| Storage | Deployed | S3 buckets with SSE-KMS | +| Data | Deployed | DynamoDB `Jobs`, `Reviews`, `GridLog`, `TraceIndex` | +| AI | Deployed | IAM permission for the pinned Bedrock model. Bedrock model access waits for AWS account verification (H1); runs use the fallback planner. | +| Auth | Deployed | Cognito user pool with `reviewer` group | +| Compute | Deployed | ECS cluster; Graviton4 `c8g.xlarge` worker Auto Scaling group, at most one instance within the account vCPU quota (D-024); SQS queue and DLQ | +| API | Deployed | Fargate `linux/arm64` behind an internal ALB. A public HTTPS endpoint through Amazon API Gateway is in progress. | +| IoT | Deployed | IoT topic rule and ingest Lambda | +| Observability | Deployed | CloudWatch dashboard and alarms | +| Web | Not deployed | CloudFront requires AWS account verification (H1). | + +Images: `linux/arm64`, built on a Graviton EC2 build instance by `infra/build_images.py` and pushed to Amazon ECR. They run on Graviton. + +COOL: not active. The Marketplace subscription is pending (H1). The engine runs stock OpenCV 5.0.0 on Graviton. The COOL award is not claimed unless the benchmark in §7 runs. + +End-to-end runs on AWS are in progress. + x86 and Arm: production containers are `linux/arm64`. The `linux/amd64` image exists only for the x86 baseline in the COOL benchmark. Deployment status, instance configuration, and the endpoint URL are added after milestone M8. diff --git a/docs/VERIFICATION.md b/docs/VERIFICATION.md index b434606..cc25631 100644 --- a/docs/VERIFICATION.md +++ b/docs/VERIFICATION.md @@ -25,14 +25,17 @@ Each entry: source URL or command, result, date, decision. Status is one of `VER - Facts from these sources: distributed as an AWS Marketplace AMI (Ubuntu 24.04 LTS, precompiled OpenCV based on OpenCV 5, Arm KleidiCV optimizations), separate listings for Graviton3 and Graviton4. - NOT yet verified (direct page fetch failed: network/Cloudflare): exact version string, instance-type list, license/EULA text, accelerated function list, runtime proof method. - Runtime proof plan (implemented in `bench/cool_evidence.py`): `cv2.getBuildInformation()` text, `cv2.__file__` and loaded shared-library paths from `/proc/self/maps`, and a `perf record` capture of the hot stages, all saved under `results/bench//evidence/`. +- Additional facts (infra agent, Marketplace listing fetched 2026-10-26): Graviton4 listing says Ubuntu 24.04 LTS, "Built on OpenCV 5.0", Arm KleidiCV, compatible families c8g/m8g/r8g, Python venvs under `/opt/cool/venvs/` (3.10, 3.11, 3.12), C++ SDK under `/opt/cool/cpp_sdk/lib`, hourly software fee, 7-day trial. A separate COOL Docker listing exists (not opened). A Graviton5 listing targets c9g/m9g with OpenCV 5.1.0 (not used). +- Still unverified: whether COOL libraries load inside the Debian-based `python:3.11-slim` container (`bench/run_remote.md` step C3 tests this and gives a native fallback). - Needs H1 (Marketplace subscription). Until then the COOL award is not claimed. ## V6 — Claude with tool use on Amazon Bedrock — BLOCKED-HUMAN (partial) - Sources: AWS ML blog "Access Anthropic Claude models in India on Amazon Bedrock with global cross-Region inference" (`https://aws.amazon.com/blogs/machine-learning/access-anthropic-claude-models-in-india-on-amazon-bedrock-with-global-cross-region-inference`) names the global inference profile ID `global.anthropic.claude-sonnet-4-6`. Anthropic docs `https://platform.claude.com/docs/en/build-with-claude/claude-on-amazon-bedrock` state newer models need an inference profile ID and that tool use is supported through the Converse/Messages APIs. -- Not verified: availability for the account in `us-east-1` (no `gridprint` AWS profile exists on this machine: `aws configure list-profiles` has no `gridprint`). +- 2026-10-26, profile `gridprint` (account 643700104680, us-east-1): `aws bedrock list-inference-profiles` lists Claude profiles (e.g. `us.anthropic.claude-sonnet-5`, `us.anthropic.claude-haiku-4-5-20251001-v1:0`), but `converse` returns `ValidationException: Operation not allowed` (us-east-1, us-west-2) and `AccessDeniedException: Your account is currently being verified` (eu-central-1); `get-foundation-model-availability` reports `authorizationStatus: NOT_AUTHORIZED`. The second account (profile `vivek`) returns `INVALID_PAYMENT_INSTRUMENT`. +- Status: FALLBACK applied for now (deterministic planner, documented in D-021). Re-check after AWS completes the account verification and model access is granted (H1). - Decision: `config/default.yaml` holds `bedrock.model_id: global.anthropic.claude-sonnet-4-6`, `bedrock.enabled: false`. The deterministic fallback planner runs until H1 completes and `scripts/verify_aws.py` confirms the model with `bedrock:ListInferenceProfiles` and a one-call Converse test. -## V7 — Android Camera2 capabilities on test devices — BLOCKED-HUMAN +## V7 — Android Camera2 capabilities on test devices — BLOCKED-HUMAN (method ready) - Keys exist in the Android API reference (`CaptureRequest.CONTROL_AE_MODE`, `CONTROL_AE_ANTIBANDING_MODE_OFF`, `SENSOR_EXPOSURE_TIME`, `CaptureResult.SENSOR_ROLLING_SHUTTER_SKEW`, `SENSOR_TIMESTAMP`, `CameraCharacteristics.SENSOR_INFO_TIMESTAMP_SOURCE`). Per-device support must be checked on hardware. The Capture app writes a capability report on first run (H4). Fallback: calibration-rig readout (Part 10.5). ## V8 — Public grid-frequency data — VERIFIED (partial) @@ -52,8 +55,8 @@ Each entry: source URL or command, result, date, decision. Status is one of `VER ## V11 — GPS PPS with ESP32 and MQTT/TLS to IoT Core — BLOCKED-HUMAN - Needs hardware (H2). Firmware uses ESP-IDF GPIO interrupt + `esp_timer_get_time()` for PPS and zero-cross capture, and `esp-mqtt` with TLS client certificates. Validation: `node/selftest.md` checklist + `node/ingest/data_quality.py`. -## V12 — On-demand prices — BLOCKED-HUMAN -- Needs AWS account (H1). `bench/prices.py` reads prices through the AWS Price List API (`pricing:GetProducts`, endpoint region `us-east-1`) and stores the raw response with date in `results/bench//prices.json`. No price is written by hand. +## V12 — On-demand prices — OPEN +- The AWS account exists (H1). Prices are fetched by `bench/prices.py` when the benchmark runs. `bench/prices.py` reads prices through the AWS Price List API (`pricing:GetProducts`, endpoint region `us-east-1`) and stores the raw response with date in `results/bench//prices.json`. No price is written by hand. ## Extra — CI action versions — VERIFIED (2026-10-26) - Command: `curl https://api.github.com/repos//releases/latest` → `actions/checkout v7.0.1`, `astral-sh/setup-uv v10.3.2`, `actions/setup-node v7.1.0`, `aquasecurity/trivy-action v0.36.0`. Pinned in `.github/workflows/ci.yml`. diff --git a/docs/VIDEO_SCRIPT.md b/docs/VIDEO_SCRIPT.md index b422e6a..28c765a 100644 --- a/docs/VIDEO_SCRIPT.md +++ b/docs/VIDEO_SCRIPT.md @@ -2,10 +2,11 @@ This is the plan for human task H8 (`SPEC.md` Part 14.4). The final video must not be longer than five minutes. -Number rule: this script contains no measured number. Each place that needs a measured value says "[value from results]" and has a source tag. Before recording, replace each placeholder with the value from the named file and keep the tag in this file. The `` in each tag is the frozen results folder. +Number rule: this script contains no measured number. Each place that needs a measured value says "[value from results]" and has a source tag. Before recording, replace each "[value from results]" marker with the value from the named file and keep the tag in this file. The `` in each tag is the frozen results folder. Recording rules: -- Every result on screen comes from the deployed endpoint. No mock screens. +- Every result on screen comes from the live pipeline: the AWS endpoint, or `make local` when the endpoint is not public. No mock screens. +- Synthetic results on screen carry the label SYNTHETIC. - Show the verdict card text and icon, not only color. - Show the overlay label "Visualization of measured flicker. Not itself evidence." whenever the overlay is on screen. - Do not use the words in the forbidden-claims list (SPEC R5). @@ -16,7 +17,7 @@ Recording rules: | Item | Content | |---|---| -| Voice | "AI can fake a video. Now people call real videos fake. We prove a video is real with the light in the room." | +| Voice | "AI can fake a video. Now people call real videos fake. We check whether a video is real with the light in the room." | | Shot 1 | Close shot of a white wall under a mains-powered lamp, recorded with GRIDPRINT Capture. The band overlay shows bands moving across the wall. | | Shot 2 | Title card: "GRIDPRINT". | @@ -60,7 +61,8 @@ Recording rules: | Item | Content | |---|---| -| Voice | "The engine runs OpenCV 5 on AWS Graviton with COOL. The agent uses Claude on Amazon Bedrock and calls engine tools through MCP. The reference grid log comes from our GPS-timed Node." | +| Voice | "The engine runs OpenCV 5 on AWS Graviton. The agent plans each job and calls engine tools through MCP. Its planner is Claude on Amazon Bedrock, with a deterministic planner as fallback. The reference grid log comes from our GPS-timed Node." | +| Voice check | Before recording, match each sentence to the facts on that day: say "with COOL" only if the COOL benchmark ran (`results/bench/`); say "Claude on Amazon Bedrock" planned this job only if the shown trace has planner type `bedrock`; say "GPS-timed Node" only if the shown reference came from Node data (H2), otherwise name the source shown in the report. | | Shot 15 | Architecture diagram (`docs/diagrams/architecture.svg`). Highlight engine stages, Graviton workers, Node path. | | Shot 16 | Benchmark chart from `results/bench//`. Throughput for configurations A, B, C: [value from results] , [value from results] , [value from results] | | Shot 17 | Cost per hour of video for each configuration: [value from results] | diff --git a/docs/diagrams/agent_workflow.mmd b/docs/diagrams/agent_workflow.mmd index 9131dcc..6bdea81 100644 --- a/docs/diagrams/agent_workflow.mmd +++ b/docs/diagrams/agent_workflow.mmd @@ -14,7 +14,7 @@ flowchart TB end subgraph DECISION["Decision: planner + policies"] - PL{"Planner
Bedrock Claude or
deterministic fallback"} + PL{"Planner
Bedrock LLM (Nova Lite)
+ deterministic fallback
per step"} POL["policies.py
iteration and tool-call budget,
compute_verdict last,
no FINAL while PENDING_REVIEW,
no edits to thresholds or verdicts"] PL --- POL end diff --git a/docs/diagrams/agent_workflow.svg b/docs/diagrams/agent_workflow.svg index 54d621b..b56ba98 100644 --- a/docs/diagrams/agent_workflow.svg +++ b/docs/diagrams/agent_workflow.svg @@ -1 +1 @@ -

Action

Decision: planner + policies

Perception: OpenCV 5 stage outputs (MCP tool results)

B1: SIGNAL_WEAK after first extraction
re-select patches, switch axis or nominal Hz

B2: readout unknown
blind estimation

B2: blind estimate low confidence

B3: HIGH_MOTION
stricter motion or low-motion frames

B4: HIGH_COMPRESSION and weak signal

B5: no usable signal after budget
BUDGET_EXHAUSTED

analysis complete

new stage output

VERIFIED or INCONCLUSIVE

B6: CONTRADICTED
state PENDING_REVIEW

approve: REVIEWER_APPROVED

reject: INCONCLUSIVE, REVIEWER_REJECTED

Job from SQS

probe_video
timestamps, jitter, gaps, HDR flag,
axis info, compression

estimate_readout
readout time, source, confidence

analyze_motion
motion_level, static fraction

propose_patches
ranked patches, band SNR

extract_flicker
band SNR, per-frame stats, track

match_reference
rho1, rho2, margin, offset

check_continuity
EDIT_CANDIDATE, DROPPED_FRAMES_EXPLAINED

face_region_check
EXPERIMENTAL stats

Planner
Bedrock Claude or
deterministic fallback

policies.py
iteration and tool-call budget,
compute_verdict last,
no FINAL while PENDING_REVIEW,
no edits to thresholds or verdicts

Re-run a stage with new arguments

request_user_input
camera model, original file,
claimed time, longer clip

compute_verdict
deterministic S13

request_human_review

render_report

Trace per step
S3 JSON + DynamoDB TraceIndex
live in web UI

Job waits for user answer
POST /v1/jobs/{id}/input

Human reviewer
approve or reject

FINAL report

\ No newline at end of file +

Action

Decision: planner + policies

Perception: OpenCV 5 stage outputs (MCP tool results)

B1: SIGNAL_WEAK after first extraction
re-select patches, switch axis or nominal Hz

B2: readout unknown
blind estimation

B2: blind estimate low confidence

B3: HIGH_MOTION
stricter motion or low-motion frames

B4: HIGH_COMPRESSION and weak signal

B5: no usable signal after budget
BUDGET_EXHAUSTED

analysis complete

new stage output

VERIFIED or INCONCLUSIVE

B6: CONTRADICTED
state PENDING_REVIEW

approve: REVIEWER_APPROVED

reject: INCONCLUSIVE, REVIEWER_REJECTED

Job from SQS

probe_video
timestamps, jitter, gaps, HDR flag,
axis info, compression

estimate_readout
readout time, source, confidence

analyze_motion
motion_level, static fraction

propose_patches
ranked patches, band SNR

extract_flicker
band SNR, per-frame stats, track

match_reference
rho1, rho2, margin, offset

check_continuity
EDIT_CANDIDATE, DROPPED_FRAMES_EXPLAINED

face_region_check
EXPERIMENTAL stats

Planner
Bedrock Claude or
deterministic fallback
(fallback in use until
Bedrock access is enabled)

policies.py
iteration and tool-call budget,
compute_verdict last,
no FINAL while PENDING_REVIEW,
no edits to thresholds or verdicts

Re-run a stage with new arguments

request_user_input
camera model, original file,
claimed time, longer clip

compute_verdict
deterministic S13

request_human_review

render_report

Trace per step
S3 JSON + DynamoDB TraceIndex
live in web UI

Job waits for user answer
POST /v1/jobs/{id}/input

Human reviewer
approve or reject

FINAL report

\ No newline at end of file diff --git a/docs/diagrams/architecture.mmd b/docs/diagrams/architecture.mmd index 1067cf7..6a9ee79 100644 --- a/docs/diagrams/architecture.mmd +++ b/docs/diagrams/architecture.mmd @@ -6,7 +6,7 @@ flowchart LR REV["Reviewer
(login)"] end - subgraph Edge["AWS edge"] + subgraph Edge["AWS edge (CloudFront waits for account verification)"] WAF["AWS WAF
rate limits"] CF["Amazon CloudFront"] S3W["Amazon S3
static web"] @@ -19,16 +19,16 @@ flowchart LR SQS["Amazon SQS
jobs queue"] DDB["Amazon DynamoDB
Jobs, Reviews, TraceIndex"] S3R["Amazon S3 results
report PDF/JSON, overlay MP4,
plots, agent trace"] - BR["Amazon Bedrock
Claude planner (pinned model ID)"] + BR["Amazon Bedrock
planner model (Nova Lite,
cross-account role)"] CW["Amazon CloudWatch
logs, metrics, dashboards, alarms"] end - subgraph Worker["Engine worker task: ECS on Graviton c8g, linux/arm64"] + subgraph Worker["Engine worker task: ECS on Graviton4 c8g, linux/arm64"] direction TB ORCH["Agent orchestrator
planner: Bedrock or fallback
policies, trace"] MCPC["MCP client"] MCPS["MCP server gridprint-engine
(stdio)"] - subgraph ENG["GRIDPRINT Engine: OpenCV 5 + COOL"] + subgraph ENG["GRIDPRINT Engine: OpenCV 5 (stock wheel)
COOL AMI supported when aws.cool_ami_id is set"] direction TB S1["S1 ingest and timing
PyAV PTS, VideoCapture"] S2["S2 linear light
cv2.LUT"] diff --git a/docs/diagrams/architecture.svg b/docs/diagrams/architecture.svg index 1b5f3d2..988ed78 100644 --- a/docs/diagrams/architecture.svg +++ b/docs/diagrams/architecture.svg @@ -1 +1 @@ -

Reference grid data

Engine worker task: ECS on Graviton c8g, linux/arm64

AWS core (region pinned in config)

AWS edge

Clients

tool calls

1 sample per UTC second

topic rule

presigned upload

Converse with tools

read

live grid tile

same engine code and inputs

GRIDPRINT Engine: OpenCV 5 + COOL

S1 ingest and timing
PyAV PTS, VideoCapture

S2 linear light
cv2.LUT

S3 readout time

S4 motion
goodFeaturesToTrack, calcOpticalFlowPyrLK,
findHomography USAC_MAGSAC, DISOpticalFlow

S5 foreground mask
cv2.dnn.readNet PP-HumanSeg

S6 light patches
warpPerspective

S7 row signal
warpPerspective, float32

S8 phase and frequency
cv2.solve, cv2.dft

S9 clock-bias removal

S10 reference matching

S11 continuity

S12 face region EXPERIMENTAL
FaceDetectorYN

S13 verdict (deterministic)

S14 band overlay
applyColorMap, VideoWriter

S15 report

GRIDPRINT Capture
Android, Camera2
video + sidecar JSON

GRIDPRINT Verify
web app (React)

Reviewer
(login)

AWS WAF
rate limits

Amazon CloudFront

Amazon S3
static web

Amazon Cognito
reviewer group

API: FastAPI
ECS Fargate linux/arm64

Amazon S3 uploads
SSE-KMS, presigned URLs

Amazon SQS
jobs queue

Amazon DynamoDB
Jobs, Reviews, TraceIndex

Amazon S3 results
report PDF/JSON, overlay MP4,
plots, agent trace

Amazon Bedrock
Claude planner (pinned model ID)

Amazon CloudWatch
logs, metrics, dashboards, alarms

Agent orchestrator
planner: Bedrock or fallback
policies, trace

MCP client

MCP server gridprint-engine
(stdio)

Adapters
StorageAdapter, GridLogAdapter, JobStoreAdapter

GRIDPRINT Node
ESP32 + GPS PPS
plug-in AC-to-AC adapter

AWS IoT Core
MQTT over TLS

AWS Lambda
ingest

Amazon DynamoDB
GridLog

Benchmark only:
x86 instance, linux/amd64 image,
stock OpenCV 5 (config A)

\ No newline at end of file +

Reference grid data

Engine worker task: ECS on Graviton4 c8g, linux/arm64

AWS core (region pinned in config)

AWS edge (CloudFront waits for account verification)

Clients

tool calls

1 sample per UTC second

topic rule

presigned upload

Converse with tools

read

live grid tile

same engine code and inputs

GRIDPRINT Engine: OpenCV 5 (stock wheel)
COOL AMI supported when aws.cool_ami_id is set

S1 ingest and timing
PyAV PTS, VideoCapture

S2 linear light
cv2.LUT

S3 readout time

S4 motion
goodFeaturesToTrack, calcOpticalFlowPyrLK,
findHomography USAC_MAGSAC, DISOpticalFlow

S5 foreground mask
cv2.dnn.readNet PP-HumanSeg

S6 light patches
warpPerspective

S7 row signal
warpPerspective, float32

S8 phase and frequency
cv2.solve, cv2.dft

S9 clock-bias removal

S10 reference matching

S11 continuity

S12 face region EXPERIMENTAL
FaceDetectorYN

S13 verdict (deterministic)

S14 band overlay
applyColorMap, VideoWriter

S15 report

GRIDPRINT Capture
Android, Camera2
video + sidecar JSON

GRIDPRINT Verify
web app (React)

Reviewer
(login)

AWS WAF
rate limits

Amazon CloudFront

Amazon S3
static web

Amazon Cognito
reviewer group

API: FastAPI
ECS Fargate linux/arm64

Amazon S3 uploads
SSE-KMS, presigned URLs

Amazon SQS
jobs queue

Amazon DynamoDB
Jobs, Reviews, TraceIndex

Amazon S3 results
report PDF/JSON, overlay MP4,
plots, agent trace

Amazon Bedrock
Claude planner (pinned model ID)
access pending; fallback planner in use

Amazon CloudWatch
logs, metrics, dashboards, alarms

Agent orchestrator
planner: Bedrock or fallback
policies, trace

MCP client

MCP server gridprint-engine
(stdio)

Adapters
StorageAdapter, GridLogAdapter, JobStoreAdapter

GRIDPRINT Node
ESP32 + GPS PPS
plug-in AC-to-AC adapter

AWS IoT Core
MQTT over TLS

AWS Lambda
ingest

Amazon DynamoDB
GridLog

Benchmark only:
x86 instance, linux/amd64 image,
stock OpenCV 5 (config A)

\ No newline at end of file diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..fdaf62d --- /dev/null +++ b/docs/index.md @@ -0,0 +1,29 @@ +--- +title: GRIDPRINT +--- + +# GRIDPRINT + +GRIDPRINT checks whether a video is real by reading the power-grid frequency from the flicker of the lights in +the room. + +Mains-powered lights flicker at twice the grid frequency. A phone camera's rolling shutter reads the sensor one row +at a time, so the flicker becomes faint horizontal bands. GRIDPRINT measures the bands with OpenCV 5, rebuilds the +grid-frequency track (electric network frequency, ENF) and matches it against reference logs of the real grid. + +It returns one of three verdicts: + +- **VERIFIED** — a grid signal is present, matches the reference log at one time, and shows no unexplained break. +- **CONTRADICTED** — a grid signal is present and conflicts with the claim or shows an internal break. A human + reviewer must approve this verdict before the report is final. +- **INCONCLUSIVE** — no usable signal, no reference coverage, or no unique match. A missing signal is never + evidence that a video is fake. + +## Read more + +- [Technical report](TECHNICAL_REPORT.md) +- [Architecture](ARCHITECTURE.md) +- [Judge guide](JUDGE_GUIDE.md) +- [Responsible use](RESPONSIBLE_USE.md) +- [Progress and evidence](PROGRESS.md) +- [Source code on GitHub](https://github.com/vivekyarra/Alibi) diff --git a/engine/gridprint_engine/adapters/__init__.py b/engine/gridprint_engine/adapters/__init__.py index d2abb1e..e7d23a8 100644 --- a/engine/gridprint_engine/adapters/__init__.py +++ b/engine/gridprint_engine/adapters/__init__.py @@ -5,6 +5,8 @@ import csv import json import shutil +import time +import uuid from abc import ABC, abstractmethod from datetime import UTC, datetime from pathlib import Path @@ -140,12 +142,23 @@ def _p(self, table: str, key: str) -> Path: def put(self, table: str, key: str, item: dict[str, Any]) -> None: p = self._p(table, key) p.parent.mkdir(parents=True, exist_ok=True) - tmp = p.with_suffix(".tmp") + tmp = p.with_suffix(f".{uuid.uuid4().hex[:8]}.tmp") tmp.write_text(json.dumps(item, default=str, indent=1)) + for attempt in range(50): # Windows refuses replace while a reader holds the file open + try: + tmp.replace(p) + return + except PermissionError: + time.sleep(0.02 * (attempt + 1)) tmp.replace(p) def get(self, table: str, key: str) -> dict[str, Any] | None: p = self._p(table, key) + for attempt in range(50): + try: + return json.loads(p.read_text()) if p.exists() else None + except (PermissionError, json.JSONDecodeError): + time.sleep(0.02 * (attempt + 1)) return json.loads(p.read_text()) if p.exists() else None def scan(self, table: str) -> list[dict[str, Any]]: diff --git a/engine/gridprint_engine/continuity.py b/engine/gridprint_engine/continuity.py index cc50669..2b27d6b 100644 --- a/engine/gridprint_engine/continuity.py +++ b/engine/gridprint_engine/continuity.py @@ -1,9 +1,11 @@ """Stage S11 — continuity (SPEC 8.12). Works with or without reference data. -The flicker phase φ_k is measured against absolute frame time, so across a true time interval Δt it -advances by 2π·δf·Δt. Residual r_k = wrap(Δφ_k − 2π·δf_local·Δt_k). A cut that was re-timed to -look continuous leaves a residual of 2π·f·(removed time) mod 2π; dropped frames with true -timestamps leave none. Change points: robust z-score of r_k (DECISIONS D-014). +The flicker phase φ_k is measured against absolute frame time, so it follows a smooth curve (slope 2π·δf). +A cut that was re-timed to look continuous shifts the phase after the cut by 2π·f·(removed time) mod 2π; +dropped frames with true timestamps do not. At every frame boundary the phase is fitted with a straight line +on each side (frames within `jump_window_s`), both lines are evaluated at the boundary, and the difference is +the phase jump. Its standard error comes from the two fits. Change points are local maxima of |jump|/σ +(DECISIONS D-014, D-023). """ from __future__ import annotations @@ -25,53 +27,80 @@ class ContinuityResult: residual_t: np.ndarray residual_rad: np.ndarray reasons: list[Reason] - boundaries: list[dict[str, float]] = field(default_factory=list) # raw candidates (z > 3) for calibration + boundaries: list[dict[str, float]] = field(default_factory=list) # raw candidates for calibration + + +def _side_fits(t: np.ndarray, y: np.ndarray, lo: np.ndarray, hi: np.ndarray, tb: np.ndarray + ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """Least-squares line through samples [lo, hi) of (t, y) evaluated at tb: (value, variance, n). float64.""" + c = lambda a: np.concatenate([[0.0], np.cumsum(a)]) # noqa: E731 + St, Sy, Stt, Sty, Syy = c(t), c(y), c(t * t), c(t * y), c(y * y) + n = (hi - lo).astype(np.float64) + st, sy = St[hi] - St[lo], Sy[hi] - Sy[lo] + stt, sty, syy = Stt[hi] - Stt[lo], Sty[hi] - Sty[lo], Syy[hi] - Syy[lo] + with np.errstate(invalid="ignore", divide="ignore"): + tm, ym = st / n, sy / n + sxx = stt - n * tm * tm + sxy = sty - n * tm * ym + b = sxy / sxx + a = ym - b * tm + rss = np.maximum(syy - n * ym * ym - b * sxy, 0.0) + s2 = rss / np.maximum(n - 2, 1) + val = a + b * tb + var = s2 * (1.0 / n + (tb - tm) ** 2 / sxx) + return val, var, n def check_continuity(t_frame: np.ndarray, phi_unwrapped: np.ndarray, gaps: list[GapEvent], - smooth_window_s: float, thr: dict[str, float]) -> ContinuityResult: + jump_window_s: float, thr: dict[str, float], min_side_frames: int = 8) -> ContinuityResult: ok = ~np.isnan(phi_unwrapped) idx = np.flatnonzero(ok) - if len(idx) < 10: + if len(idx) < 2 * min_side_frames: return ContinuityResult([], np.zeros(0), np.zeros(0), []) - t = t_frame[idx] - phi = phi_unwrapped[idx] - dphi = _wrap(np.diff(phi)) - dt = np.diff(t) - rate = dphi / np.maximum(dt, 1e-9) # rad/s per interval - # smooth frequency model: running median of the phase rate over the smoothing window - mid = 0.5 * (t[1:] + t[:-1]) - loc = np.empty_like(rate) - for i in range(len(rate)): - sel = np.abs(mid - mid[i]) <= smooth_window_s / 2 - sel[i] = False - loc[i] = np.median(rate[sel]) if sel.any() else rate[i] - res = _wrap(dphi - loc * dt) - mad = float(np.median(np.abs(res - np.median(res)))) * 1.4826 + 1e-9 - z = np.abs(res - np.median(res)) / mad + t = t_frame[idx].astype(np.float64) + phi = phi_unwrapped[idx].astype(np.float64) + # boundary j lies between samples j and j+1 + tb = 0.5 * (t[1:] + t[:-1]) + j = np.arange(len(t) - 1) + left_lo = np.searchsorted(t, tb - jump_window_s, "left") + left_hi = j + 1 + right_lo = j + 1 + right_hi = np.searchsorted(t, tb + jump_window_s, "right") + vl, varl, nl = _side_fits(t, phi, left_lo, left_hi, tb) + vr, varr, nr = _side_fits(t, phi, right_lo, right_hi, tb) + jump = _wrap(vr - vl) + sigma = np.sqrt(np.maximum(varl + varr, 1e-12)) + z = np.where((nl >= min_side_frames) & (nr >= min_side_frames), np.abs(jump) / sigma, 0.0) + # local maxima of z within ±jump_window_s (a single jump raises z at its neighbours too) + peak = np.zeros(len(z), bool) + for cand in np.flatnonzero(z > 3.0): + near = np.abs(tb - tb[cand]) <= jump_window_s + peak[cand] = z[cand] >= z[near].max() sig_thr = float(thr["edit_significance_sigma"]) min_rad = float(thr["edit_min_rad"]) gap_at = {g.frame_index: g for g in gaps} events: list[ContinuityEvent] = [] - reasons: list[Reason] = [] - for j in range(len(res)): - k = int(idx[j + 1]) # boundary between idx[j] and idx[j+1] - gap = gap_at.get(k) # S1 timestamp gap at this boundary - significant = bool(z[j] > sig_thr and abs(res[j]) >= min_rad) + raw: list[dict[str, float]] = [] + for i in range(len(z)): + k = int(idx[i + 1]) + gap = gap_at.get(k) + if not peak[i] and gap is None: + continue + significant = bool(peak[i] and z[i] > sig_thr and abs(jump[i]) >= min_rad) + raw.append({"t_s": float(t[i + 1]), "z": float(z[i]), "rad": float(abs(jump[i])), "gap": gap is not None}) if gap is not None and not significant: - events.append(ContinuityEvent(kind="DROPPED_FRAMES_EXPLAINED", frame_index=k, t_s=float(t[j + 1]), - magnitude_rad=float(res[j]), significance=float(z[j]), - significant=False, gap_s=gap.size_s)) + events.append(ContinuityEvent(kind="DROPPED_FRAMES_EXPLAINED", frame_index=k, t_s=float(t[i + 1]), + magnitude_rad=float(jump[i]), significance=float(z[i]), significant=False, + gap_s=gap.size_s)) elif significant: - events.append(ContinuityEvent(kind="EDIT_CANDIDATE", frame_index=k, t_s=float(t[j + 1]), - magnitude_rad=float(res[j]), significance=float(z[j]), - significant=True, gap_s=gap.size_s if gap else None)) + events.append(ContinuityEvent(kind="EDIT_CANDIDATE", frame_index=k, t_s=float(t[i + 1]), + magnitude_rad=float(jump[i]), significance=float(z[i]), significant=True, + gap_s=gap.size_s if gap else None)) + reasons: list[Reason] = [] if any(e.kind == "EDIT_CANDIDATE" for e in events): reasons.append(Reason.EDIT_CANDIDATE) if any(e.kind == "DROPPED_FRAMES_EXPLAINED" for e in events): reasons.append(Reason.DROPPED_FRAMES_EXPLAINED) - raw = [{"t_s": float(t[j + 1]), "z": float(z[j]), "rad": float(abs(res[j])), - "gap": bool(int(idx[j + 1]) in gap_at)} for j in np.flatnonzero(z > 3.0)] - raw += [{"t_s": float(t[j + 1]), "z": float(z[j]), "rad": float(abs(res[j])), "gap": True} - for j in range(len(res)) if int(idx[j + 1]) in gap_at and z[j] <= 3.0] - return ContinuityResult(events, mid, res, reasons, raw) + # series for plots: the estimated phase jump at every boundary with enough frames on both sides + valid = (nl >= min_side_frames) & (nr >= min_side_frames) + return ContinuityResult(events, tb[valid], jump[valid], reasons, raw) diff --git a/engine/gridprint_engine/motion.py b/engine/gridprint_engine/motion.py index c2a6a67..8a84c8e 100644 --- a/engine/gridprint_engine/motion.py +++ b/engine/gridprint_engine/motion.py @@ -66,13 +66,31 @@ def analyze_motion(lin: np.ndarray, residual_limit_px: float, high_motion_px: fl step = [np.eye(3)] for i in range(1, k): step.append(_pair_homography(gray8[i - 1], gray8[i], max_features)) # (i-1) → i - # accumulate frame → frame-0, then re-reference to the middle frame - to0 = [np.eye(3)] - for i in range(1, k): - to0.append(to0[-1] @ np.linalg.inv(step[i])) + # Register every frame directly to the reference (middle) frame: features found once in the reference and + # tracked with pyramidal LK. Chaining per-frame homographies accumulates error into fake drift over + # hundreds of frames, so chaining is used only for a frame whose direct registration fails. ref = k // 2 - inv_ref = np.linalg.inv(to0[ref]) - homs = np.stack([inv_ref @ H for H in to0]).astype(np.float64) + homs_l: list[np.ndarray | None] = [None] * k + ref_pts = cv2.goodFeaturesToTrack(gray8[ref], maxCorners=max_features, qualityLevel=0.01, minDistance=5) + for i in range(k): + if i == ref: + homs_l[i] = np.eye(3) + continue + if ref_pts is None or len(ref_pts) < 8: + break + nxt, st, _ = cv2.calcOpticalFlowPyrLK(gray8[ref], gray8[i], ref_pts, None, winSize=(21, 21), maxLevel=3) + good = st.ravel() == 1 + if good.sum() >= 8: + Hm, inl = cv2.findHomography(nxt[good], ref_pts[good], cv2.USAC_MAGSAC, 1.0) # frame i → reference + if Hm is not None and inl is not None and inl.sum() >= 8: + homs_l[i] = Hm + for i in list(range(ref + 1, k)) + list(range(ref - 1, -1, -1)): # fallback: chain from a neighbour + if homs_l[i] is None: + if i > ref: + homs_l[i] = homs_l[i - 1] @ np.linalg.inv(step[i]) # type: ignore[operator] + else: + homs_l[i] = homs_l[i + 1] @ step[i + 1] # type: ignore[operator] + homs = np.stack(homs_l).astype(np.float64) # type: ignore[arg-type] homs /= homs[:, 2:3, 2:3] dis = cv2.DISOpticalFlow_create(cv2.DISOPTICAL_FLOW_PRESET_ULTRAFAST) # type: ignore[attr-defined] @@ -101,19 +119,18 @@ def analyze_motion(lin: np.ndarray, residual_limit_px: float, high_motion_px: fl for s_ in static]) homs = np.stack([scale @ Hm @ np.linalg.inv(scale) for Hm in homs]) h, w = H, W - max_disp = 0.0 corners = np.array([[0, 0, 1], [w - 1, 0, 1], [0, h - 1, 1], [w - 1, h - 1, 1]], np.float64) - for H in homs: - q = corners @ H.T + disp = [] + for Hm in homs: + q = corners @ Hm.T q = q[:, :2] / q[:, 2:3] - max_disp = max(max_disp, float(np.abs(q - corners[:, :2]).max())) + disp.append(float(np.abs(q - corners[:, :2]).max())) + max_disp = float(np.percentile(disp, 90)) if disp else 0.0 + moving = max_disp > max(1.0, 20 * camera_static_px) # 90th-percentile corner displacement (analysis px) subset = None if strictness == "strict" and resid_med: rm = np.array([resid_med[0]] + resid_med) subset = np.flatnonzero(rm <= np.percentile(rm, 50)) - # Chained per-frame homographies drift slowly even for a fixed camera. Warping is used only when the - # camera visibly moves (median per-frame displacement) and the accumulated displacement is real. - moving = cam > camera_static_px and max_disp > 1.0 return MotionResult( homs=homs if moving else None, static=static, diff --git a/engine/gridprint_engine/pipeline.py b/engine/gridprint_engine/pipeline.py index bf4f098..26629e1 100644 --- a/engine/gridprint_engine/pipeline.py +++ b/engine/gridprint_engine/pipeline.py @@ -456,9 +456,17 @@ def check_continuity(self, track_id: str | None = None) -> dict[str, Any]: tr = self.tracks.get(track_id) if track_id else self.primary_track() if tr is None or self.timing is None: raise JobError("no flicker track") + if Reason.SIGNAL_ABSENT in tr.reasons: + # without a signal the phase is noise; reporting phase breaks would be meaningless (and R2) + res = s11.ContinuityResult([], np.zeros(0), np.zeros(0), []) + self.continuity[tr.id] = res + return {"track_id": tr.id, "events": [], "edit_candidates": 0, "dropped_explained": 0, "reasons": [], + "skipped": "no signal"} + # only frames whose own flicker fit is usable enter the phase series + phi = np.where(tr.fits.snr >= self.E["continuity"]["min_frame_snr"], tr.track.phase_unwrapped, np.nan) with self._timed("S11_continuity"): - res = s11.check_continuity(self.timing.t_s, tr.track.phase_unwrapped, self.timing.gaps, - self.E["continuity"]["smooth_window_s"], self.cfg.thresholds["continuity"]) + res = s11.check_continuity(self.timing.t_s, phi, self.timing.gaps, + self.E["continuity"]["jump_window_s"], self.cfg.thresholds["continuity"]) self.continuity[tr.id] = res return {"track_id": tr.id, "events": [e.model_dump() for e in res.events][:50], "edit_candidates": sum(e.kind == "EDIT_CANDIDATE" for e in res.events), @@ -524,6 +532,9 @@ def compute_verdict(self, extra_reasons: list[Reason] | None = None) -> dict[str stage_timings_s={k: round(v, 4) for k, v in self.timings.items()}, artifacts=dict(self.artifacts), notes=list(self.notes)) self.result.quality["verdict_rule"] = out.rule + from gridprint_engine import report as s15 + + s15.write_json(self, self.workdir) # the verdict is stored even if no report is rendered return {"verdict": out.verdict.value, "lifecycle_state": out.state.value, "reasons": [r.value for r in out.reasons], "rule": out.rule} diff --git a/engine/gridprint_engine/report.py b/engine/gridprint_engine/report.py index 8e72c8c..32b559c 100644 --- a/engine/gridprint_engine/report.py +++ b/engine/gridprint_engine/report.py @@ -109,12 +109,14 @@ def _plots(job: Job, outdir: Path) -> list[dict[str, str]]: if e.kind == "EDIT_CANDIDATE": ax.axvline(e.t_s, color="#b00000", lw=1.5) ax.set_xlabel("time in video (s)") - ax.set_ylabel("phase residual (rad)") + ax.set_ylabel("phase jump at boundary (rad)") fig.tight_layout() _save(fig, outdir, "phase_residual") - plots.append({"file": "phase_residual.png", "alt": "Flicker phase residual over time. Solid vertical " - "lines mark edit candidates; dotted lines mark frame-timestamp gaps.", - "caption": "Phase residual. Solid line: edit candidate. Dotted line: timestamp gap."}) + plots.append({"file": "phase_residual.png", + "alt": "Estimated flicker phase jump at each frame boundary. Solid vertical lines mark edit " + "candidates; dotted lines mark frame-timestamp gaps.", + "caption": "Phase jump at frame boundaries. Solid line: edit candidate. " + "Dotted line: timestamp gap."}) # 4. per-frame band amplitude fig, ax = plt.subplots(figsize=(8, 2.4)) ax.plot(job.timing.t_s, np.where(tr.fits.ok, tr.fits.amp, np.nan), lw=0.8) diff --git a/engine/gridprint_engine/tests/test_video.py b/engine/gridprint_engine/tests/test_video.py new file mode 100644 index 0000000..ae0cab2 --- /dev/null +++ b/engine/gridprint_engine/tests/test_video.py @@ -0,0 +1,143 @@ +"""Video-level integration tests on SYNTHETIC clips (M2/M3 acceptance). + +Covers VFR timestamps, gaps, sensor-row axis, rotation metadata, HDR, linear light, ENF recovery and time +offset, estimator agreement, cuts, dropped frames, and report generation. Thresholds come from a test file +(the production file stays the only source for real verdicts, R8). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np +import pytest +import yaml + +from eval.simulator.render import SimConfig, simulate +from gridprint_engine import io +from gridprint_engine.adapters import LocalGridLog +from gridprint_engine.config import CONFIG_DIR, load_settings +from gridprint_engine.pipeline import Job, run_default +from gridprint_engine.schemas import Reason, Verdict + +SMALL = dict(width=160, height=120, ref_margin_s=300, flicker_depth=0.1, noise_sigma=0.8) +TEST_THR = {"signal": {"snr_absent_db": 3.0, "snr_weak_db": 6.0}, + "match": {"min_track_s": 20, "rho1_min": 0.8, "margin_min": 0.02}, + "continuity": {"edit_significance_sigma": 6.0, "edit_min_rad": 0.15}, + "facecheck": {"ratio_anomaly": 0.5}} + + +@pytest.fixture(scope="module") +def clips(tmp_path_factory: pytest.TempPathFactory) -> dict[str, tuple[Path, dict]]: + d = tmp_path_factory.mktemp("clips") + cfgs = { + "clean": SimConfig(seed=11, duration_s=40, sidecar=True, **SMALL), + "cut": SimConfig(seed=11, duration_s=40, edits=[{"type": "cut", "at_s": 17.3, "dur_s": 1.0}], **SMALL), + "dropped": SimConfig(seed=11, duration_s=40, drop_frames=[(300, 5), (800, 7)], **SMALL), + "cols": SimConfig(seed=12, duration_s=6, sensor_axis="cols", **SMALL), + "rot": SimConfig(seed=13, duration_s=3, rotation_deg=90, **SMALL), + "vfr": SimConfig(seed=14, duration_s=4, jitter_ms=2.0, drop_frames=[(50, 3)], **SMALL), + "hdr": SimConfig(seed=15, duration_s=1, transfer="hlg", **SMALL), + "nomains": SimConfig(seed=16, duration_s=6, **{**SMALL, "flicker_depth": 0.0}), + } + out = {} + for name, cfg in cfgs.items(): + tr = simulate(cfg, d, name) + out[name] = (d, tr) + thr = d / "thresholds.yaml" + thr.write_text(yaml.safe_dump(TEST_THR)) + out["_settings"] = load_settings(CONFIG_DIR, thr) # type: ignore[assignment] + return out + + +def _job(clips, name, **kw) -> Job: + d, tr = clips[name] + side = d / tr["sidecar"] if tr.get("sidecar") else None + return run_default(d / tr["video"], d / "work", LocalGridLog(d / tr["gridlog_root"]), sidecar=side, + cfg=clips["_settings"], job_id=name, **kw) + + +def test_vfr_timestamps_gaps_and_jitter(clips): + d, tr = clips["vfr"] + pts = io.read_pts(d / tr["video"]) + assert np.allclose(pts - pts[0], np.array(tr["frame_pts_s"]) - tr["frame_pts_s"][0], atol=2e-5) + tm = io.compute_timing(pts, None, 1.5) + assert any(g.frame_index == 50 for g in tm.gaps) + assert tm.jitter_p95_ms > 1.0 # 2 ms jitter was injected + assert np.allclose(io.read_pts_ffprobe(d / tr["video"]), pts, atol=1e-5) # V9 fallback agrees + + +def test_rotation_metadata_is_recorded_but_not_applied(clips): + d, tr = clips["rot"] + pr = io.probe_video(d / tr["video"]) + assert abs(pr.rotation) == 90 + fs = io.decode_frames(d / tr["video"], "bt709", 240, 320, 0.98, 0.02) + assert (fs.n_rows, fs.n_cols) == (120, 160) # coded (sensor) orientation kept + assert fs.lin.dtype == np.float32 # R6 + + +def test_sensor_axis_columns_detected(clips): + d, tr = clips["cols"] + j = Job(d / tr["video"], d / "work", LocalGridLog(d / tr["gridlog_root"]), cfg=clips["_settings"]) + j.probe_video() + j.estimate_readout("blind") + out = j.extract_flicker() + assert out["axis"] == "cols+" + assert j.readout is not None and j.readout.readout_s == pytest.approx(0.025, rel=0.1) + + +def test_hdr_stops_with_inconclusive(clips): + j = _job(clips, "hdr") + assert j.result is not None + assert j.result.verdict == Verdict.INCONCLUSIVE and Reason.HDR_UNSUPPORTED in j.result.reasons + + +def test_no_mains_light_is_inconclusive_never_contradicted(clips): + j = _job(clips, "nomains") + assert j.result is not None + assert j.result.verdict == Verdict.INCONCLUSIVE + assert Reason.SIGNAL_ABSENT in j.result.reasons + + +def test_clean_clip_verified_with_time_and_report(clips): + j = _job(clips, "clean", render=True) + r = j.result + assert r is not None + _, tr = clips["clean"] + assert r.verdict == Verdict.VERIFIED, r.reasons + assert abs(r.time_estimate_utc - tr["start_utc"]) < 1.0 + assert r.quality["estimators_agree"] is True + # ENF track recovered: compare with the ground-truth grid frequency + track = j.primary_track().track + gt = np.array(tr["enf_clip_hz"])[np.floor(track.tau_s).astype(int)] + err = (track.enf_hz - track.enf_hz.mean()) - (gt - gt.mean()) + assert np.sqrt(np.mean(err ** 2)) < 0.003 + for k in ("report_html", "report_pdf", "report_json", "overlay_mp4"): + assert Path(r.artifacts[k]).exists() and Path(r.artifacts[k]).stat().st_size > 0 + assert json.loads(Path(r.artifacts["report_json"]).read_text())["lifecycle_state"] == "FINAL" + html = Path(r.artifacts["report_html"]).read_text(encoding="utf-8") + from scripts.claims_lint import scan_text + + assert scan_text(html) == [] + assert "Limitations" in html and "alt=" in html + + +def test_cut_is_contradicted_and_localized(clips): + j = _job(clips, "cut", render=True) + r = j.result + assert r is not None + assert r.verdict == Verdict.CONTRADICTED and r.lifecycle_state.value == "PENDING_REVIEW" + edits = [e for e in r.continuity_events if e.kind == "EDIT_CANDIDATE"] + assert edits and min(abs(e.t_s - 17.3) for e in edits) < 0.2 + with pytest.raises(Exception, match="FINAL report blocked"): + j.render_report(final=True) + + +def test_dropped_frames_with_true_timestamps_not_flagged(clips): + j = _job(clips, "dropped") + r = j.result + assert r is not None + assert not any(e.kind == "EDIT_CANDIDATE" for e in r.continuity_events) + assert Reason.DROPPED_FRAMES_EXPLAINED in r.reasons + assert r.verdict != Verdict.CONTRADICTED diff --git a/eval/simulator/make_samples.py b/eval/simulator/make_samples.py new file mode 100644 index 0000000..0c3d439 --- /dev/null +++ b/eval/simulator/make_samples.py @@ -0,0 +1,35 @@ +"""Write the small committed sample set (SPEC M1) to results/synthetic_samples/. All clips are SYNTHETIC.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from eval.simulator.render import SimConfig, simulate + +REPO = Path(__file__).resolve().parents[2] +BASE: dict[str, Any] = dict(width=160, height=120, duration_s=6.0, ref_margin_s=60, seed=21) +SAMPLES: dict[str, dict[str, Any]] = { + "clean_with_sidecar": {"sidecar": True}, + "cut_0p5s": {"edits": [{"type": "cut", "at_s": 3.1, "dur_s": 0.5}]}, + "dropped_frames": {"drop_frames": [(90, 4)]}, + "no_mains_light": {"flicker_depth": 0.0}, + "camera_motion": {"camera_motion_px": 2.0, "moving_objects": 1}, + "cols_axis_messaging": {"sensor_axis": "cols", "compression": "messaging"}, +} + + +def main() -> None: + out = REPO / "results" / "synthetic_samples" + index = [] + for name, kw in SAMPLES.items(): + tr = simulate(SimConfig(**{**BASE, **kw}), out, name) + index.append({"name": name, "video": tr["video"], "truth": f"{name}.truth.json", "label": tr["label"], + "expected_class": tr["expected_class"], "video_sha256": tr["video_sha256"]}) + (out / "index.json").write_text(json.dumps(index, indent=1)) + print(json.dumps(index, indent=1)) + + +if __name__ == "__main__": + main() diff --git a/eval/simulator/render.py b/eval/simulator/render.py index d002cf6..d861770 100644 --- a/eval/simulator/render.py +++ b/eval/simulator/render.py @@ -138,7 +138,12 @@ def _schedule(cfg: SimConfig) -> tuple[np.ndarray, np.ndarray, list[dict[str, An if cfg.jitter_ms > 0: rng = np.random.default_rng(cfg.seed + 7) pts_a = pts_a + rng.normal(0, cfg.jitter_ms * 1e-3, len(pts_a)) - pts_a = np.maximum.accumulate(pts_a + np.arange(len(pts_a)) * 1e-6) + # keep timestamps strictly increasing by at least two 90 kHz ticks and non-negative, so the + # container never receives duplicate or negative PTS (which would drop frames) + tick = 2.0 / 90000 + for i in range(1, len(pts_a)): + pts_a[i] = max(pts_a[i], pts_a[i - 1] + tick) + pts_a -= min(0.0, float(pts_a[0])) return cap_a, pts_a, events diff --git a/eval/tests/test_simulator.py b/eval/tests/test_simulator.py new file mode 100644 index 0000000..35e6e85 --- /dev/null +++ b/eval/tests/test_simulator.py @@ -0,0 +1,57 @@ +"""Simulator (SPEC 11.1): every factor produces a video plus ground truth, labeled SYNTHETIC.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import av +import numpy as np + +from eval.simulator.enf_model import GridSignal, ou_track +from eval.simulator.render import SimConfig, simulate +from gridprint_engine.adapters import LocalGridLog + +TINY = dict(width=64, height=48, duration_s=3.0, ref_margin_s=20) + + +def test_grid_signal_integral_and_reference(): + g = GridSignal(1000, np.array([50.0, 50.1, 50.1, 49.9])) + assert g.grid_cycles(np.array([1000.0]))[0] == 0.0 + assert g.grid_cycles(np.array([1001.0]))[0] == np.float64(50.05) + t, f = g.reference_log() + assert list(t) == [1000, 1001, 1002] and np.allclose(f, [50.05, 50.1, 50.0]) + + +def test_ou_track_is_deterministic_and_near_nominal(): + a = ou_track(3600, 50.0, np.random.default_rng(1)) + b = ou_track(3600, 50.0, np.random.default_rng(1)) + assert np.array_equal(a, b) and abs(a.mean() - 50.0) < 0.05 and a.dtype == np.float64 + + +def test_every_factor_renders_with_ground_truth(tmp_path: Path): + factors = { + "depth": {"flicker_depth": 0.05}, "readout": {"readout_s": 0.015}, "exposure": {"exposure_s": 0.004}, + "jitter": {"jitter_ms": 1.0}, "dropped": {"drop_frames": [(20, 3)]}, "motion": {"camera_motion_px": 2.0}, + "objects": {"moving_objects": 1}, "noise": {"noise_sigma": 3.0}, "compression": {"compression": "messaging"}, + "cut": {"edits": [{"type": "cut", "at_s": 1.0, "dur_s": 0.5}]}, + "insert": {"edits": [{"type": "insert", "at_s": 1.0, "dur_s": 0.5, "from_offset_s": -10.0}]}, + "reorder": {"edits": [{"type": "reorder", "a_s": 0.2, "b_s": 1.6, "dur_s": 0.6}]}, + "axis": {"sensor_axis": "cols"}, "rotation": {"rotation_deg": 90}, "hdr": {"transfer": "hlg"}, + "sidecar": {"sidecar": True}, "clock": {"clock_ppm": 50.0}, + } + for name, kw in factors.items(): + tr = simulate(SimConfig(seed=3, **{**TINY, **kw}), tmp_path, name) + assert tr["label"] == "SYNTHETIC" + with av.open(str(tmp_path / tr["video"])) as c: + n = sum(1 for _ in c.decode(video=0)) + assert n == len(tr["frame_pts_s"]) == len(tr["frame_capture_t_s"]) + assert json.loads((tmp_path / f"{name}.truth.json").read_text())["video_sha256"] == tr["video_sha256"] + t, f = LocalGridLog(tmp_path / tr["gridlog_root"]).read("SIM50", tr["start_utc"] - 5, tr["start_utc"] + 5) + assert len(t) == 11 and np.all(np.abs(f - 50) < 0.5) + cut = json.loads((tmp_path / "cut.truth.json").read_text()) + assert cut["edits"][0]["frames_removed"] == 15 and cut["expected_class"] == "edited" + assert np.allclose(np.diff(cut["frame_pts_s"]), 1 / 30) # re-timed: looks continuous + dropped = json.loads((tmp_path / "dropped.truth.json").read_text()) + assert max(np.diff(dropped["frame_pts_s"])) > 0.1 # original timestamps kept + assert (tmp_path / "sidecar.sidecar.json").exists() diff --git a/infra/build_images.py b/infra/build_images.py new file mode 100644 index 0000000..8169e16 --- /dev/null +++ b/infra/build_images.py @@ -0,0 +1,247 @@ +"""Build the container images in AWS (CodeBuild on Graviton) and push them to ECR. + +Used by `make deploy` when no local Docker engine is available. The images are built from the committed tree +(`git archive HEAD`), so what runs in AWS is exactly what is in the repository. + + python infra/build_images.py # builds linux/arm64 engine + api images, prints their URIs + python infra/build_images.py --amd64 # also builds the linux/amd64 engine image (bench baseline A) + +Writes `infra/image_uris.json` (git-ignored); `infra/app.py` passes these URIs to the stacks. +""" + +from __future__ import annotations + +import argparse +import io +import json +import subprocess +import sys +import time +from pathlib import Path + +import boto3 +import yaml + +REPO = Path(__file__).resolve().parents[1] +CFG = yaml.safe_load((REPO / "config" / "default.yaml").read_text()) +PREFIX = CFG["aws"].get("resource_prefix", "gridprint") +IMAGES = {"engine": "infra/docker/engine.Dockerfile", "api": "infra/docker/api.Dockerfile"} +ARM_IMAGE = "aws/codebuild/amazonlinux-aarch64-standard:3.0" # listed by list-curated-environment-images +X86_IMAGE = "aws/codebuild/amazonlinux2-x86_64-standard:5.0" + +BUILDSPEC = """version: 0.2 +phases: + pre_build: + commands: + - aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $REGISTRY + build: + commands: + - docker build -f $DOCKERFILE -t $REPO_URI:$TAG . + - docker push $REPO_URI:$TAG +""" + + +def session() -> boto3.Session: + return boto3.Session(profile_name=CFG["aws"]["profile"], region_name=CFG["aws"]["region"]) + + +def ensure_repo(ecr, name: str) -> str: # type: ignore[no-untyped-def] + try: + r = ecr.describe_repositories(repositoryNames=[name])["repositories"][0] + except ecr.exceptions.RepositoryNotFoundException: + r = ecr.create_repository(repositoryName=name, imageScanningConfiguration={"scanOnPush": True}, + encryptionConfiguration={"encryptionType": "AES256"})["repository"] + return str(r["repositoryUri"]) + + +def ensure_bucket(s3, name: str) -> None: # type: ignore[no-untyped-def] + try: + s3.head_bucket(Bucket=name) + except Exception: + s3.create_bucket(Bucket=name) + s3.put_public_access_block(Bucket=name, PublicAccessBlockConfiguration={ + "BlockPublicAcls": True, "IgnorePublicAcls": True, "BlockPublicPolicy": True, + "RestrictPublicBuckets": True}) + s3.put_bucket_lifecycle_configuration(Bucket=name, LifecycleConfiguration={"Rules": [ + {"ID": "expire-sources", "Status": "Enabled", "Filter": {"Prefix": ""}, "Expiration": {"Days": 7}}]}) + + +def ensure_role(iam, account: str, bucket: str) -> str: # type: ignore[no-untyped-def] + name = f"{PREFIX}-image-build" + trust = {"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Principal": { + "Service": "codebuild.amazonaws.com"}, "Action": "sts:AssumeRole"}]} + try: + arn = iam.get_role(RoleName=name)["Role"]["Arn"] + except iam.exceptions.NoSuchEntityException: + arn = iam.create_role(RoleName=name, AssumeRolePolicyDocument=json.dumps(trust))["Role"]["Arn"] + time.sleep(10) # IAM propagation before CodeBuild can assume the role + policy = {"Version": "2012-10-17", "Statement": [ + {"Effect": "Allow", "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"], + "Resource": f"arn:aws:logs:*:{account}:log-group:/aws/codebuild/{PREFIX}-*"}, + {"Effect": "Allow", "Action": ["s3:GetObject", "s3:GetObjectVersion"], "Resource": f"arn:aws:s3:::{bucket}/*"}, + {"Effect": "Allow", "Action": "ecr:GetAuthorizationToken", "Resource": "*"}, + {"Effect": "Allow", "Action": ["ecr:BatchCheckLayerAvailability", "ecr:InitiateLayerUpload", + "ecr:UploadLayerPart", "ecr:CompleteLayerUpload", "ecr:PutImage", + "ecr:BatchGetImage", "ecr:GetDownloadUrlForLayer"], + "Resource": f"arn:aws:ecr:*:{account}:repository/{PREFIX}-*"}]} + iam.put_role_policy(RoleName=name, PolicyName="build", PolicyDocument=json.dumps(policy)) + return str(arn) + + +def source_zip() -> bytes: + return subprocess.run(["git", "archive", "--format=zip", "HEAD"], cwd=REPO, capture_output=True, + check=True).stdout + + +def build(arch: str, tag: str, account: str) -> dict[str, str]: + s = session() + ecr, s3, iam, cb = s.client("ecr"), s.client("s3"), s.client("iam"), s.client("codebuild") + bucket = f"{PREFIX}-build-{account}" + ensure_bucket(s3, bucket) + role = ensure_role(iam, account, bucket) + key = f"src/{tag}.zip" + s3.upload_fileobj(io.BytesIO(source_zip()), bucket, key) + registry = f"{account}.dkr.ecr.{s.region_name}.amazonaws.com" + uris: dict[str, str] = {} + for name, dockerfile in IMAGES.items(): + if arch == "amd64" and name != "engine": + continue + repo = ensure_repo(ecr, f"{PREFIX}-{name}") + project = f"{PREFIX}-{name}-{arch}" + env = {"type": "ARM_CONTAINER" if arch == "arm64" else "LINUX_CONTAINER", + "image": ARM_IMAGE if arch == "arm64" else X86_IMAGE, "computeType": "BUILD_GENERAL1_LARGE", + "privilegedMode": True} + args = {"name": project, "source": {"type": "S3", "location": f"{bucket}/{key}", "buildspec": BUILDSPEC}, + "artifacts": {"type": "NO_ARTIFACTS"}, "environment": env, "serviceRole": role, + "timeoutInMinutes": 60} + try: + cb.create_project(**args) + except cb.exceptions.ResourceAlreadyExistsException: + cb.update_project(**args) + img_tag = f"{tag}-{arch}" + b = cb.start_build(projectName=project, environmentVariablesOverride=[ + {"name": "DOCKERFILE", "value": dockerfile}, {"name": "REPO_URI", "value": repo}, + {"name": "TAG", "value": img_tag}, {"name": "REGISTRY", "value": registry}])["build"] + print(f"{project}: build {b['id']} started", flush=True) + while True: + st = cb.batch_get_builds(ids=[b["id"]])["builds"][0] + if st["buildStatus"] != "IN_PROGRESS": + break + time.sleep(20) + print(f"{project}: {st['buildStatus']} ({st.get('currentPhase')})", flush=True) + if st["buildStatus"] != "SUCCEEDED": + logs = st.get("logs", {}).get("deepLink") + raise SystemExit(f"image build failed: {project}; logs: {logs}") + uris[f"{name}_{arch}"] = f"{repo}:{img_tag}" + return uris + + +USER_DATA = """#!/bin/bash +set -x +exec > /var/log/image-build.log 2>&1 +dnf install -y docker unzip +systemctl start docker +aws s3 cp s3://{bucket}/{key} /tmp/src.zip && mkdir -p /src && cd /src && unzip -q /tmp/src.zip +aws ecr get-login-password --region {region} | docker login --username AWS --password-stdin {registry} +ok=1 +docker build -f infra/docker/engine.Dockerfile -t {engine}:{tag} . && docker push {engine}:{tag} || ok=0 +docker build -f infra/docker/api.Dockerfile -t {api}:{tag} . && docker push {api}:{tag} || ok=0 +aws s3 cp /var/log/image-build.log s3://{bucket}/logs/{tag}.log +if [ $ok = 1 ]; then echo ok > /tmp/status; else echo failed > /tmp/status; fi +aws s3 cp /tmp/status s3://{bucket}/logs/{tag}.status +shutdown -h now +""" + + +def ensure_instance_profile(iam, account: str, bucket: str) -> str: # type: ignore[no-untyped-def] + name = f"{PREFIX}-image-build-ec2" + trust = {"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Principal": {"Service": "ec2.amazonaws.com"}, + "Action": "sts:AssumeRole"}]} + try: + iam.get_role(RoleName=name) + except iam.exceptions.NoSuchEntityException: + iam.create_role(RoleName=name, AssumeRolePolicyDocument=json.dumps(trust)) + policy = {"Version": "2012-10-17", "Statement": [ + {"Effect": "Allow", "Action": ["s3:GetObject", "s3:PutObject"], "Resource": f"arn:aws:s3:::{bucket}/*"}, + {"Effect": "Allow", "Action": "ecr:GetAuthorizationToken", "Resource": "*"}, + {"Effect": "Allow", "Action": ["ecr:BatchCheckLayerAvailability", "ecr:InitiateLayerUpload", + "ecr:UploadLayerPart", "ecr:CompleteLayerUpload", "ecr:PutImage", + "ecr:BatchGetImage", "ecr:GetDownloadUrlForLayer"], + "Resource": f"arn:aws:ecr:*:{account}:repository/{PREFIX}-*"}]} + iam.put_role_policy(RoleName=name, PolicyName="build", PolicyDocument=json.dumps(policy)) + try: + iam.get_instance_profile(InstanceProfileName=name) + except iam.exceptions.NoSuchEntityException: + iam.create_instance_profile(InstanceProfileName=name) + iam.add_role_to_instance_profile(InstanceProfileName=name, RoleName=name) + time.sleep(15) # instance profile propagation + return name + + +def build_ec2(tag: str, account: str) -> dict[str, str]: + """arm64 images built on a short-lived Graviton instance that terminates itself.""" + s = session() + ecr, s3, iam, ec2, ssm = (s.client(n) for n in ("ecr", "s3", "iam", "ec2", "ssm")) + bucket = f"{PREFIX}-build-{account}" + ensure_bucket(s3, bucket) + profile = ensure_instance_profile(iam, account, bucket) + key = f"src/{tag}.zip" + s3.upload_fileobj(io.BytesIO(source_zip()), bucket, key) + repos = {n: ensure_repo(ecr, f"{PREFIX}-{n}") for n in IMAGES} + img_tag = f"{tag}-arm64" + ud = USER_DATA.format(bucket=bucket, key=key, region=s.region_name, tag=img_tag, + registry=f"{account}.dkr.ecr.{s.region_name}.amazonaws.com", engine=repos["engine"], + api=repos["api"]) + ami = ssm.get_parameter(Name="/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64") + for _ in range(6): + try: + inst = ec2.run_instances( + ImageId=ami["Parameter"]["Value"], InstanceType="c8g.large", MinCount=1, MaxCount=1, UserData=ud, + IamInstanceProfile={"Name": profile}, InstanceInitiatedShutdownBehavior="terminate", + BlockDeviceMappings=[{"DeviceName": "/dev/xvda", "Ebs": {"VolumeSize": 40, "VolumeType": "gp3"}}], + MetadataOptions={"HttpTokens": "required"}, + TagSpecifications=[{"ResourceType": "instance", "Tags": [{"Key": "Name", + "Value": f"{PREFIX}-image-build"}]}]) + break + except ec2.exceptions.ClientError as e: # instance profile may still be propagating + if "iamInstanceProfile" not in str(e): + raise + time.sleep(10) + iid = inst["Instances"][0]["InstanceId"] + print(f"build instance {iid} started", flush=True) + status_key = f"logs/{img_tag}.status" + for _ in range(180): # up to 60 minutes + time.sleep(20) + try: + body = s3.get_object(Bucket=bucket, Key=status_key)["Body"].read().decode().strip() + break + except s3.exceptions.NoSuchKey: + continue + else: + raise SystemExit(f"image build timed out; instance {iid}") + print(f"build status: {body}; log s3://{bucket}/logs/{img_tag}.log", flush=True) + if body != "ok": + raise SystemExit("image build failed; see the log above") + return {f"{n}_arm64": f"{repos[n]}:{img_tag}" for n in IMAGES} + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--amd64", action="store_true") + ap.add_argument("--backend", choices=["codebuild", "ec2"], default="ec2") + a = ap.parse_args() + account = session().client("sts").get_caller_identity()["Account"] + tag = subprocess.run(["git", "rev-parse", "--short=12", "HEAD"], cwd=REPO, capture_output=True, text=True, + check=True).stdout.strip() + out = REPO / "infra" / "image_uris.json" + uris = json.loads(out.read_text()) if out.exists() else {} + uris.update(build_ec2(tag, account) if a.backend == "ec2" else build("arm64", tag, account)) + if a.amd64: + uris.update(build("amd64", tag, account)) + out.write_text(json.dumps(uris, indent=1)) + print(json.dumps(uris, indent=1)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/infra/docker/api.Dockerfile b/infra/docker/api.Dockerfile index 250afb6..c7fc332 100644 --- a/infra/docker/api.Dockerfile +++ b/infra/docker/api.Dockerfile @@ -1,5 +1,12 @@ # GRIDPRINT API image (FastAPI on arm64 Fargate, SPEC 10.1). Build context: repository root. # docker buildx build --platform linux/arm64 -f infra/docker/api.Dockerfile -t gridprint-api . +FROM node:24-slim AS web +WORKDIR /web +COPY web/package.json web/package-lock.json ./ +RUN npm ci --no-audit --no-fund +COPY web/ ./ +RUN npm run build + FROM ghcr.io/astral-sh/uv:0.11.8 AS uv FROM python:3.11-slim @@ -19,11 +26,15 @@ ENV UV_PROJECT_ENVIRONMENT=/opt/venv \ WORKDIR /app COPY pyproject.toml uv.lock ./ -RUN uv sync --frozen --no-dev --no-install-project +RUN uv sync --frozen --no-dev --group agent --no-install-project COPY config ./config COPY engine ./engine +COPY agent ./agent +COPY mcp_server ./mcp_server +COPY node/ingest ./node/ingest COPY api ./api +COPY --from=web /web/dist ./web/dist ENV PATH=/opt/venv/bin:$PATH \ PYTHONPATH=/app:/app/engine diff --git a/infra/docker/engine.Dockerfile b/infra/docker/engine.Dockerfile index 7d2d0a9..eec7cdb 100644 --- a/infra/docker/engine.Dockerfile +++ b/infra/docker/engine.Dockerfile @@ -28,8 +28,11 @@ COPY config ./config COPY engine ./engine COPY agent ./agent COPY mcp_server ./mcp_server +COPY node/ingest ./node/ingest COPY api ./api -COPY models ./models +COPY scripts/fetch_models.py ./scripts/fetch_models.py +# DNN models (S5, S12) are downloaded from opencv_zoo at build time; licenses and SHA-256 are recorded +RUN /opt/venv/bin/python scripts/fetch_models.py ENV PATH=/opt/venv/bin:$PATH \ PYTHONPATH=/app:/app/engine diff --git a/infra/gridprint_infra/ai.py b/infra/gridprint_infra/ai.py index 7989f9d..d9055e2 100644 --- a/infra/gridprint_infra/ai.py +++ b/infra/gridprint_infra/ai.py @@ -41,3 +41,4 @@ def __init__(self, scope: Construct, cid: str, *, cfg: InfraConfig, **kwargs) -> resources=bedrock_resource_arns(self, cfg.bedrock_model_id), )], ) + diff --git a/infra/gridprint_infra/api.py b/infra/gridprint_infra/api.py index deae762..f751ebf 100644 --- a/infra/gridprint_infra/api.py +++ b/infra/gridprint_infra/api.py @@ -9,7 +9,9 @@ from pathlib import Path -from aws_cdk import Duration, IgnoreMode, Stack +from aws_cdk import CfnOutput, Duration, IgnoreMode, Stack +from aws_cdk import aws_apigatewayv2 as apigw +from aws_cdk import aws_apigatewayv2_integrations as apigw_int from aws_cdk import aws_ec2 as ec2 from aws_cdk import aws_ecr_assets as ecr_assets from aws_cdk import aws_ecs as ecs @@ -18,8 +20,8 @@ from constructs import Construct from .auth import AuthStack -from .compute import API_CONTEXT, METRIC_NAMESPACE, ComputeStack -from .config import REPO, InfraConfig +from .compute import API_CONTEXT, METRIC_NAMESPACE, ComputeStack, container_image +from .config import REPO, InfraConfig, prebuilt_image from .data import DataStack from .runtime_env import API_COMMAND, service_env from .storage import StorageStack @@ -31,7 +33,8 @@ class ApiStack(Stack): def __init__(self, scope: Construct, cid: str, *, cfg: InfraConfig, vpc: ec2.IVpc, storage: StorageStack, data: DataStack, compute: ComputeStack, auth: AuthStack, **kwargs) -> None: super().__init__(scope, cid, **kwargs) - image = ecr_assets.DockerImageAsset( + prebuilt = prebuilt_image("api_arm64") + image = None if prebuilt else ecr_assets.DockerImageAsset( self, "ApiImageArm64", directory=str(REPO), file=str(Path("infra") / "docker" / "api.Dockerfile"), @@ -53,7 +56,7 @@ def __init__(self, scope: Construct, cid: str, *, cfg: InfraConfig, vpc: ec2.IVp operating_system_family=ecs.OperatingSystemFamily.LINUX), circuit_breaker=ecs.DeploymentCircuitBreaker(rollback=True), task_image_options=ecs_patterns.ApplicationLoadBalancedTaskImageOptions( - image=ecs.ContainerImage.from_docker_image_asset(image), + image=container_image(self, "ApiRepo", prebuilt, image), container_port=API_PORT, log_driver=ecs.LogDrivers.aws_logs(stream_prefix="api", log_group=self.log_group, mode=ecs.AwsLogDriverMode.NON_BLOCKING), @@ -70,6 +73,26 @@ def __init__(self, scope: Construct, cid: str, *, cfg: InfraConfig, vpc: ec2.IVp self.service.load_balancer.connections.allow_from(ec2.Peer.ipv4(vpc.vpc_cidr_block), ec2.Port.tcp(80), "CloudFront VPC origin inside the VPC") + # Public HTTPS entry point without CloudFront (D-025): API Gateway HTTP API → VPC link → internal ALB. + # The VPC link ENIs sit inside the VPC, so the CIDR rule above admits them. Stage throttling limits + # abuse of anonymous upload creation (WAF cannot attach to HTTP APIs). + # The VPC's default security group has no rules (CDK hardening), so the VPC link gets its own group with + # egress to the ALB only. + link_sg = ec2.SecurityGroup(self, "VpcLinkSg", vpc=vpc, allow_all_outbound=False, + description="API Gateway VPC link to the internal API load balancer") + link_sg.add_egress_rule(ec2.Peer.ipv4(vpc.vpc_cidr_block), ec2.Port.tcp(80), "to the internal ALB") + vpc_link = apigw.VpcLink(self, "PublicApiLink", vpc=vpc, security_groups=[link_sg], + subnets=ec2.SubnetSelection(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS)) + self.http_api = apigw.HttpApi( + self, "PublicApi", + default_integration=apigw_int.HttpAlbIntegration("AlbIntegration", self.service.listener, + vpc_link=vpc_link), + ) + stage = self.http_api.default_stage.node.default_child # type: ignore[union-attr] + stage.default_route_settings = apigw.CfnStage.RouteSettingsProperty( # type: ignore[union-attr] + throttling_burst_limit=cfg.waf_job_create_limit_5min * 2, throttling_rate_limit=10) + CfnOutput(self, "PublicEndpoint", value=self.http_api.api_endpoint) + role = self.service.task_definition.task_role storage.uploads.grant_put(role) storage.uploads.grant_read(role) diff --git a/infra/gridprint_infra/app.py b/infra/gridprint_infra/app.py index 2b3e710..9644297 100644 --- a/infra/gridprint_infra/app.py +++ b/infra/gridprint_infra/app.py @@ -25,7 +25,7 @@ def build_app(account: str | None = None, config_dir: Path | None = None, cfg = config_mod.load(config_dir or config_mod.CONFIG_DIR) app = cdk.App(outdir=outdir, context=context) if outdir or context else cdk.App() env = cdk.Environment(account=account, region=cfg.region) - p = "Gridprint" + p = cfg.resource_prefix.capitalize() network = NetworkStack(app, f"{p}Network", env=env) storage = StorageStack(app, f"{p}Storage", cfg=cfg, env=env) @@ -43,6 +43,6 @@ def build_app(account: str | None = None, config_dir: Path | None = None, if str(app.node.try_get_context("bench")).lower() == "true": BenchImagesStack(app, f"{p}BenchImages", env=env) - for key, value in {"project": "gridprint", "managed-by": "cdk"}.items(): + for key, value in {"project": cfg.resource_prefix, "managed-by": "cdk"}.items(): cdk.Tags.of(app).add(key, value) return app diff --git a/infra/gridprint_infra/compute.py b/infra/gridprint_infra/compute.py index b1f88b6..04dd020 100644 --- a/infra/gridprint_infra/compute.py +++ b/infra/gridprint_infra/compute.py @@ -18,6 +18,7 @@ from aws_cdk import aws_autoscaling as autoscaling from aws_cdk import aws_cloudwatch as cw from aws_cdk import aws_ec2 as ec2 +from aws_cdk import aws_ecr as ecr from aws_cdk import aws_ecr_assets as ecr_assets from aws_cdk import aws_ecs as ecs from aws_cdk import aws_iam as iam @@ -27,7 +28,7 @@ from .ai import AiStack from .auth import AuthStack -from .config import REPO, InfraConfig +from .config import PREFIX, REPO, InfraConfig, prebuilt_image from .data import DataStack from .runtime_env import WORKER_COMMAND, service_env from .storage import StorageStack @@ -44,16 +45,17 @@ def docker_context(dockerfile: str, paths: list[str]) -> list[str]: ENGINE_CONTEXT = docker_context("engine.Dockerfile", - ["config", "engine", "agent", "mcp_server", "api", "models"]) -API_CONTEXT = docker_context("api.Dockerfile", ["config", "engine", "api"]) + ["config", "engine", "agent", "mcp_server", "api", "node", "scripts"]) +API_CONTEXT = docker_context("api.Dockerfile", ["config", "engine", "agent", "mcp_server", "node", "api", "web"]) # Engineering constants for the worker task on c8g.2xlarge (8 vCPU, 16 GiB). Headroom is left for the # ECS agent and the OS. Documented in docs/agent_notes/infra.md. -WORKER_CPU_UNITS = 7168 -WORKER_MEMORY_MIB = 13312 +# Worker task size per instance type: whole instance minus headroom for the ECS agent and OS. +WORKER_SIZES = {"c8g.xlarge": (3584, 6656), "c8g.2xlarge": (7168, 13312), "c8g.4xlarge": (15360, 28672)} +WORKER_CPU_UNITS, WORKER_MEMORY_MIB = WORKER_SIZES["c8g.2xlarge"] JOB_VISIBILITY_TIMEOUT_S = 1800 # > limits.decode_timeout_s plus analysis time DLQ_MAX_RECEIVES = 3 -METRIC_NAMESPACE = "GRIDPRINT" +METRIC_NAMESPACE = PREFIX.capitalize() ARM64_CONSTRAINT = "attribute:ecs.cpu-architecture == arm64" @@ -68,6 +70,17 @@ def engine_image(scope: Construct, cid: str, platform: ecr_assets.Platform) -> e ) +def container_image(scope: Construct, cid: str, uri: str | None, + asset: ecr_assets.DockerImageAsset | None) -> ecs.ContainerImage: + """Pre-built ECR image (infra/build_images.py) when available, else the local Docker build asset.""" + if uri: + repo_name, tag = uri.split("/", 1)[1].split(":", 1) + repo = ecr.Repository.from_repository_name(scope, cid, repo_name) + return ecs.ContainerImage.from_ecr_repository(repo, tag) + assert asset is not None + return ecs.ContainerImage.from_docker_image_asset(asset) + + def ubuntu_ecs_agent_user_data(region: str) -> list[str]: """Commands from the ECS developer guide, 'Installing the Amazon ECS container agent on a non-Amazon Linux EC2 instance' (deb, arm64). ECS_CLUSTER is appended later by the cluster.""" @@ -113,7 +126,7 @@ def __init__(self, scope: Construct, cid: str, *, cfg: InfraConfig, vpc: ec2.IVp user_data = ec2.UserData.for_linux() self.host_sg = ec2.SecurityGroup(self, "WorkerHostSg", vpc=vpc, allow_all_outbound=True, - description="GRIDPRINT worker hosts: no inbound") + description=f"{PREFIX} worker hosts: no inbound") launch_template = ec2.LaunchTemplate( self, "WorkerLaunchTemplate", instance_type=ec2.InstanceType(cfg.worker_instance_type), @@ -172,13 +185,15 @@ def __init__(self, scope: Construct, cid: str, *, cfg: InfraConfig, vpc: ec2.IVp if self.uses_cool_ami: env["PYTHONPATH"] = f"{cfg.cool_pythonpath}:/app:/app/engine" env["LD_LIBRARY_PATH"] = cfg.cool_ld_library_path - self.worker_image = engine_image(self, "EngineImageArm64", ecr_assets.Platform.LINUX_ARM64) + prebuilt = prebuilt_image("engine_arm64") + self.worker_image = None if prebuilt else engine_image(self, "EngineImageArm64", + ecr_assets.Platform.LINUX_ARM64) container = self.task_def.add_container( "engine", - image=ecs.ContainerImage.from_docker_image_asset(self.worker_image), + image=container_image(self, "EngineRepo", prebuilt, self.worker_image), command=WORKER_COMMAND, - cpu=WORKER_CPU_UNITS, - memory_limit_mib=WORKER_MEMORY_MIB, + cpu=WORKER_SIZES.get(cfg.worker_instance_type, WORKER_SIZES["c8g.2xlarge"])[0], + memory_limit_mib=WORKER_SIZES.get(cfg.worker_instance_type, WORKER_SIZES["c8g.2xlarge"])[1], environment=env, readonly_root_filesystem=False, logging=ecs.LogDrivers.aws_logs(stream_prefix="worker", log_group=self.log_group, @@ -192,6 +207,9 @@ def __init__(self, scope: Construct, cid: str, *, cfg: InfraConfig, vpc: ec2.IVp read_only=True)) # Least privilege for the worker. + if cfg.bedrock_role_arn: # planner model hosted in another account: assume only that role (D-026) + self.task_role.add_to_policy(iam.PolicyStatement(actions=["sts:AssumeRole"], + resources=[cfg.bedrock_role_arn])) storage.uploads.grant_read_write(self.task_role) storage.results.grant_read_write(self.task_role) for table in data.tables: diff --git a/infra/gridprint_infra/config.py b/infra/gridprint_infra/config.py index b27d491..2a45434 100644 --- a/infra/gridprint_infra/config.py +++ b/infra/gridprint_infra/config.py @@ -23,6 +23,7 @@ DEFAULT_COOL_PYTHONPATH = "/opt/cool/python_3.11/site-packages" DEFAULT_COOL_LD_LIBRARY_PATH = "/opt/cool/cpp_sdk/lib" DEFAULT_NODE_ZONES = ("IN",) +DEFAULT_RESOURCE_PREFIX = "gridprint" def upload_retention_days(hours: float) -> int: @@ -48,6 +49,8 @@ class InfraConfig: node_zones: tuple[str, ...] cool_pythonpath: str cool_ld_library_path: str + resource_prefix: str = DEFAULT_RESOURCE_PREFIX + bedrock_role_arn: str | None = None cool_host_paths: tuple[str, ...] = field(default=DEFAULT_COOL_HOST_PATHS) @property @@ -65,7 +68,8 @@ def load(config_dir: Path = CONFIG_DIR) -> InfraConfig: return InfraConfig( region=aws["region"], profile=aws["profile"], - bedrock_model_id=raw["bedrock"]["model_id"], + # in-account Claude model kept for the AI stack policy (planner model may live in another account, D-026) + bedrock_model_id=raw["bedrock"].get("in_account_model_id", raw["bedrock"]["model_id"]), upload_retention_hours=float(limits["upload_retention_hours"]), results_retention_days=int(limits["results_retention_days"]), presign_expiry_s=int(limits["presign_expiry_s"]), @@ -78,6 +82,8 @@ def load(config_dir: Path = CONFIG_DIR) -> InfraConfig: waf_job_create_limit_5min=int(aws.get("waf_job_create_limit_5min", DEFAULT_WAF_JOB_CREATE_LIMIT_5MIN)), node_zones=tuple(grid.get("node_zones", DEFAULT_NODE_ZONES)), + resource_prefix=str(aws.get("resource_prefix", DEFAULT_RESOURCE_PREFIX)), + bedrock_role_arn=(raw.get("bedrock") or {}).get("role_arn"), cool_pythonpath=str(aws.get("cool_pythonpath", DEFAULT_COOL_PYTHONPATH)), cool_ld_library_path=str(aws.get("cool_ld_library_path", DEFAULT_COOL_LD_LIBRARY_PATH)), ) @@ -92,3 +98,15 @@ def foundation_model_id(model_or_profile_id: str) -> str: if rest and head in {"global", "us", "eu", "apac", "jp", "au", "ca", "us-gov"}: return rest return model_or_profile_id + + +# Prefix for every physical resource name (stacks, tables, IoT names, dashboard, KMS alias); `aws.resource_prefix`. +PREFIX = load().resource_prefix + + +def prebuilt_image(key: str) -> str | None: + """ECR image URI built by infra/build_images.py (no local Docker needed), or None to build locally.""" + import json + + f = REPO / "infra" / "image_uris.json" + return json.loads(f.read_text()).get(key) if f.exists() else None diff --git a/infra/gridprint_infra/data.py b/infra/gridprint_infra/data.py index 7955dd2..6c4a300 100644 --- a/infra/gridprint_infra/data.py +++ b/infra/gridprint_infra/data.py @@ -6,6 +6,8 @@ from aws_cdk import aws_dynamodb as ddb from constructs import Construct +from .config import PREFIX + class DataStack(Stack): def __init__(self, scope: Construct, cid: str, **kwargs) -> None: @@ -14,7 +16,7 @@ def __init__(self, scope: Construct, cid: str, **kwargs) -> None: def table(name: str, pk: str, sk: tuple[str, ddb.AttributeType] | None = None) -> ddb.Table: return ddb.Table( self, name, - table_name=f"gridprint-{name}", + table_name=f"{PREFIX}-{name}", partition_key=ddb.Attribute(name=pk, type=ddb.AttributeType.STRING), sort_key=ddb.Attribute(name=sk[0], type=sk[1]) if sk else None, billing_mode=ddb.BillingMode.PAY_PER_REQUEST, diff --git a/infra/gridprint_infra/iot.py b/infra/gridprint_infra/iot.py index 93059f1..080f987 100644 --- a/infra/gridprint_infra/iot.py +++ b/infra/gridprint_infra/iot.py @@ -16,11 +16,11 @@ from constructs import Construct from .compute import METRIC_NAMESPACE -from .config import REPO +from .config import PREFIX, REPO from .data import DataStack -TOPIC_PREFIX = "gridprint" -THING_TYPE = "gridprint-node" +TOPIC_PREFIX = PREFIX +THING_TYPE = f"{PREFIX}-node" INGEST_DIR = REPO / "node" / "ingest" @@ -53,13 +53,13 @@ def __init__(self, scope: Construct, cid: str, *, data: DataStack, **kwargs) -> self, "NodeThingType", thing_type_name=THING_TYPE, thing_type_properties=iot.CfnThingType.ThingTypePropertiesProperty( - thing_type_description="GRIDPRINT grid-frequency Node (ESP32 + GPS PPS)", + thing_type_description=f"{PREFIX} sensor node", searchable_attributes=["zone"], ), ) self.node_policy = iot.CfnPolicy( self, "NodePolicy", - policy_name="gridprint-node", + policy_name=f"{PREFIX}-node", policy_document=node_policy_document(self.partition, self.region, self.account), ) diff --git a/infra/gridprint_infra/observability.py b/infra/gridprint_infra/observability.py index 1c0155a..8a1cb96 100644 --- a/infra/gridprint_infra/observability.py +++ b/infra/gridprint_infra/observability.py @@ -28,7 +28,7 @@ from .api import ApiStack from .compute import METRIC_NAMESPACE, ComputeStack -from .config import InfraConfig +from .config import PREFIX, InfraConfig from .iot import IotStack STAGES = ["S1", "S2", "S3", "S4", "S5", "S6", "S7", "S8", "S9", "S10", "S11", "S12", "S13", "S14", "S15"] @@ -46,7 +46,7 @@ def __init__(self, scope: Construct, cid: str, *, cfg: InfraConfig, compute: Com iot: IotStack, **kwargs) -> None: super().__init__(scope, cid, **kwargs) # Subscriptions (email of the on-call person) are added by a human (H1). - self.alarm_topic = sns.Topic(self, "Alarms", display_name="GRIDPRINT alarms", + self.alarm_topic = sns.Topic(self, "Alarms", display_name=f"{PREFIX} alarms", enforce_ssl=True) action = cw_actions.SnsAction(self.alarm_topic) @@ -88,7 +88,7 @@ def __init__(self, scope: Construct, cid: str, *, cfg: InfraConfig, compute: Com a.add_alarm_action(action) self.alarms = alarms - dash = cw.Dashboard(self, "Dashboard", dashboard_name="gridprint") + dash = cw.Dashboard(self, "Dashboard", dashboard_name=PREFIX) dash.add_widgets( cw.GraphWidget(title="Jobs", left=[m("JobsCreated"), m("JobsCompleted")], right=[compute.queue.metric_approximate_number_of_messages_visible(), diff --git a/infra/gridprint_infra/storage.py b/infra/gridprint_infra/storage.py index a7d0006..28a76aa 100644 --- a/infra/gridprint_infra/storage.py +++ b/infra/gridprint_infra/storage.py @@ -7,7 +7,7 @@ from aws_cdk import aws_s3 as s3 from constructs import Construct -from .config import InfraConfig +from .config import PREFIX, InfraConfig class StorageStack(Stack): @@ -15,7 +15,7 @@ def __init__(self, scope: Construct, cid: str, *, cfg: InfraConfig, **kwargs) -> super().__init__(scope, cid, **kwargs) self.key = kms.Key( self, "DataKey", - alias="alias/gridprint-data", + alias=f"alias/{PREFIX}-data", enable_key_rotation=True, removal_policy=RemovalPolicy.DESTROY, # `make destroy` removes everything (SPEC 10.6) pending_window=Duration.days(7), @@ -37,6 +37,8 @@ def __init__(self, scope: Construct, cid: str, *, cfg: InfraConfig, **kwargs) -> # S3 lifecycle expiration counts whole days. limits.upload_retention_hours is rounded up. self.uploads = s3.Bucket( self, "Uploads", + cors=[s3.CorsRule(allowed_methods=[s3.HttpMethods.PUT], allowed_origins=["*"], allowed_headers=["*"], + max_age=3000)], # presigned PUT from the browser (D-025) lifecycle_rules=[s3.LifecycleRule( id="expire-uploads", expiration=Duration.days(cfg.upload_retention_days), diff --git a/infra/gridprint_infra/web.py b/infra/gridprint_infra/web.py index fad04a4..3b769b9 100644 --- a/infra/gridprint_infra/web.py +++ b/infra/gridprint_infra/web.py @@ -17,7 +17,7 @@ from constructs import Construct from .api import ApiStack -from .config import REPO, InfraConfig +from .config import PREFIX, REPO, InfraConfig from .storage import StorageStack WEB_DIST = REPO / "web" / "dist" @@ -72,7 +72,7 @@ def __init__(self, scope: Construct, cid: str, *, cfg: InfraConfig, api: ApiStac scope="CLOUDFRONT", default_action=wafv2.CfnWebACL.DefaultActionProperty(allow={}), visibility_config=wafv2.CfnWebACL.VisibilityConfigProperty( - cloud_watch_metrics_enabled=True, metric_name="gridprint-web", sampled_requests_enabled=True), + cloud_watch_metrics_enabled=True, metric_name=f"{PREFIX}-web", sampled_requests_enabled=True), rules=[ rate_rule("rate-all-per-ip", 0, cfg.waf_rate_limit_5min), rate_rule("rate-job-create-per-ip", 1, cfg.waf_job_create_limit_5min, @@ -121,7 +121,7 @@ def __init__(self, scope: Construct, cid: str, *, cfg: InfraConfig, api: ApiStac "MaxAgeSeconds": 3000, }]}, }, - physical_resource_id=cr.PhysicalResourceId.of("gridprint-uploads-cors"), + physical_resource_id=cr.PhysicalResourceId.of(f"{PREFIX}-uploads-cors"), ) cr.AwsCustomResource( self, "UploadsCors", diff --git a/infra/tests/test_stacks.py b/infra/tests/test_stacks.py index 80e0229..6bd4cda 100644 --- a/infra/tests/test_stacks.py +++ b/infra/tests/test_stacks.py @@ -21,6 +21,9 @@ from gridprint_infra import config as cfgmod # noqa: E402 from gridprint_infra.app import build_app # noqa: E402 +from gridprint_infra.config import PREFIX # noqa: E402 + +P = PREFIX.capitalize() ACCOUNT = "123456789012" @@ -58,22 +61,22 @@ def test_docker_context_is_allow_list(tmp_path_factory: pytest.TempPathFactory, assert ENGINE_CONTEXT[0] == "*" and API_CONTEXT[0] == "*" assert "!infra/docker/engine.Dockerfile" in ENGINE_CONTEXT assert "!api" in ENGINE_CONTEXT and "!agent" in ENGINE_CONTEXT - assert "!agent" not in API_CONTEXT + assert "!agent" in API_CONTEXT and "!node" in API_CONTEXT # the API imports the worker (agent) and node adapters def test_all_stacks_present(stacks: dict[str, Template]) -> None: for name in ["Network", "Storage", "Data", "Compute", "Api", "Iot", "Auth", "Web", "Observability", "Ai"]: - assert f"Gridprint{name}" in stacks + assert f"{P}{name}" in stacks def test_vpc_gateway_endpoints(stacks: dict[str, Template]) -> None: - t = stacks["GridprintNetwork"] + t = stacks[f"{P}Network"] t.resource_count_is("AWS::EC2::VPCEndpoint", 2) t.has_resource_properties("AWS::EC2::VPCEndpoint", {"VpcEndpointType": "Gateway"}) def test_buckets_kms_tls_public_block_and_lifecycle(stacks: dict[str, Template]) -> None: - t = stacks["GridprintStorage"] + t = stacks[f"{P}Storage"] buckets = t.find_resources("AWS::S3::Bucket") assert len(buckets) == 2 cfg = cfgmod.load() @@ -95,25 +98,25 @@ def test_buckets_kms_tls_public_block_and_lifecycle(stacks: dict[str, Template]) def test_four_tables_with_pitr_and_gridlog_keys(stacks: dict[str, Template]) -> None: - t = stacks["GridprintData"] + t = stacks[f"{P}Data"] t.resource_count_is("AWS::DynamoDB::Table", 4) for name in ["Jobs", "Reviews", "GridLog", "TraceIndex"]: t.has_resource_properties("AWS::DynamoDB::Table", { - "TableName": f"gridprint-{name}", + "TableName": f"{PREFIX}-{name}", "PointInTimeRecoverySpecification": {"PointInTimeRecoveryEnabled": True}, }) t.has_resource_properties("AWS::DynamoDB::Table", { - "TableName": "gridprint-GridLog", + "TableName": f"{PREFIX}-GridLog", "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}, {"AttributeName": "t", "KeyType": "RANGE"}], "AttributeDefinitions": Match.array_with([{"AttributeName": "t", "AttributeType": "N"}]), }) def test_lead_key_schema_for_jobs_reviews_trace(stacks: dict[str, Template]) -> None: - t = stacks["GridprintData"] + t = stacks[f"{P}Data"] for name in ["Jobs", "Reviews", "TraceIndex"]: t.has_resource_properties("AWS::DynamoDB::Table", { - "TableName": f"gridprint-{name}", + "TableName": f"{PREFIX}-{name}", "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], "BillingMode": "PAY_PER_REQUEST", @@ -130,8 +133,8 @@ def _container(t: Template) -> dict: @pytest.mark.parametrize("stack,command", [ - ("GridprintApi", ["uvicorn", "api.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8000"]), - ("GridprintCompute", ["python", "-m", "api.worker", "--sqs"]), + (f"{P}Api", ["uvicorn", "api.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8000"]), + (f"{P}Compute", ["python", "-m", "api.worker", "--sqs"]), ]) def test_container_command_and_env(stacks: dict[str, Template], stack: str, command: list[str]) -> None: c = _container(stacks[stack]) @@ -143,32 +146,32 @@ def test_container_command_and_env(stacks: dict[str, Template], stack: str, comm def test_uploads_cors_put_from_cloudfront(stacks: dict[str, Template]) -> None: - t = stacks["GridprintWeb"] + t = stacks[f"{P}Web"] cr = json.dumps(t.find_resources("Custom::AWS")) assert "putBucketCors" in cr assert "PUT" in cr assert "DomainName" in cr # origin is built from the distribution domain - storage_buckets = stacks["GridprintStorage"].find_resources("AWS::S3::Bucket") + storage_buckets = stacks[f"{P}Storage"].find_resources("AWS::S3::Bucket") assert all("CorsConfiguration" not in b["Properties"] for b in storage_buckets.values()) def test_worker_can_put_uploads_and_write_all_tables(stacks: dict[str, Template]) -> None: - pols = json.dumps(stacks["GridprintCompute"].find_resources("AWS::IAM::Policy")) + pols = json.dumps(stacks[f"{P}Compute"].find_resources("AWS::IAM::Policy")) for action in ["s3:PutObject", "s3:GetObject*", "dynamodb:PutItem", "sqs:ReceiveMessage", "sqs:DeleteMessage"]: assert action in pols for table in ["Jobs", "Reviews", "GridLog", "TraceIndex"]: - assert f"GridprintData:ExportsOutputFnGetAtt{table}" in pols + assert f"{P}Data:ExportsOutputFnGetAtt{table}" in pols def test_cognito_reviewer_group(stacks: dict[str, Template]) -> None: - t = stacks["GridprintAuth"] + t = stacks[f"{P}Auth"] t.has_resource_properties("AWS::Cognito::UserPoolGroup", {"GroupName": "reviewer"}) t.has_resource_properties("AWS::Cognito::UserPool", {"MfaConfiguration": "ON"}) def test_waf_rate_rules_on_cloudfront(stacks: dict[str, Template]) -> None: - t = stacks["GridprintWeb"] + t = stacks[f"{P}Web"] t.has_resource_properties("AWS::WAFv2::WebACL", { "Scope": "CLOUDFRONT", "Rules": Match.array_with([Match.object_like({ @@ -179,14 +182,14 @@ def test_waf_rate_rules_on_cloudfront(stacks: dict[str, Template]) -> None: def test_arm64_task_definitions(stacks: dict[str, Template]) -> None: - worker = stacks["GridprintCompute"] + worker = stacks[f"{P}Compute"] worker.has_resource_properties("AWS::ECS::TaskDefinition", { "RequiresCompatibilities": ["EC2"], "PlacementConstraints": [{"Type": "memberOf", "Expression": "attribute:ecs.cpu-architecture == arm64"}], }) worker.has_resource_properties("AWS::ECS::Service", {"CapacityProviderStrategy": Match.any_value()}) - api = stacks["GridprintApi"] + api = stacks[f"{P}Api"] api.has_resource_properties("AWS::ECS::TaskDefinition", { "RequiresCompatibilities": ["FARGATE"], "RuntimePlatform": {"CpuArchitecture": "ARM64", "OperatingSystemFamily": "LINUX"}, @@ -195,7 +198,7 @@ def test_arm64_task_definitions(stacks: dict[str, Template]) -> None: def test_graviton_capacity_and_queue(stacks: dict[str, Template]) -> None: - t = stacks["GridprintCompute"] + t = stacks[f"{P}Compute"] t.has_resource_properties("AWS::EC2::LaunchTemplate", { "LaunchTemplateData": Match.object_like({"InstanceType": "c8g.2xlarge"})}) t.resource_count_is("AWS::ECS::CapacityProvider", 1) @@ -208,7 +211,7 @@ def test_graviton_capacity_and_queue(stacks: dict[str, Template]) -> None: def test_bedrock_policy_is_pinned(stacks: dict[str, Template]) -> None: - t = stacks["GridprintAi"] + t = stacks[f"{P}Ai"] pol = next(iter(t.find_resources("AWS::IAM::ManagedPolicy").values())) stmts = pol["Properties"]["PolicyDocument"]["Statement"] assert len(stmts) == 1 @@ -221,9 +224,9 @@ def test_bedrock_policy_is_pinned(stacks: dict[str, Template]) -> None: def test_iot_policy_topic_and_arm64_lambda(stacks: dict[str, Template]) -> None: - t = stacks["GridprintIot"] + t = stacks[f"{P}Iot"] doc = json.dumps(t.find_resources("AWS::IoT::Policy")) - assert "topic/gridprint/${iot:Connection.Thing.Attributes[zone]}/${iot:Connection.Thing.ThingName}" in doc + assert f"topic/{PREFIX}/${{iot:Connection.Thing.Attributes[zone]}}/${{iot:Connection.Thing.ThingName}}" in doc t.has_resource_properties("AWS::Lambda::Function", {"Architectures": ["arm64"], "Handler": "handler.handler"}) t.resource_count_is("AWS::IoT::TopicRule", 1) @@ -231,7 +234,7 @@ def test_iot_policy_topic_and_arm64_lambda(stacks: dict[str, Template]) -> None: def test_observability_alarms(stacks: dict[str, Template]) -> None: - t = stacks["GridprintObservability"] + t = stacks[f"{P}Observability"] t.resource_count_is("AWS::CloudWatch::Dashboard", 1) limit = cfgmod.load().freshness_limit_s t.has_resource_properties("AWS::CloudWatch::Alarm", { @@ -239,7 +242,7 @@ def test_observability_alarms(stacks: dict[str, Template]) -> None: def test_cool_ami_variant(cool_stacks: dict[str, Template]) -> None: - t = cool_stacks["GridprintCompute"] + t = cool_stacks[f"{P}Compute"] lt = json.dumps(t.find_resources("AWS::EC2::LaunchTemplate")) assert "amazon-ecs-init-latest.arm64.deb" in lt t.has_resource_properties("AWS::ECS::TaskDefinition", { @@ -248,4 +251,4 @@ def test_cool_ami_variant(cool_stacks: dict[str, Template]) -> None: [Match.object_like({"ContainerPath": "/opt/cool", "ReadOnly": True})]), })], }) - assert "GridprintBenchImages" in cool_stacks + assert f"{P}BenchImages" in cool_stacks diff --git a/mcp_server/tools.py b/mcp_server/tools.py index bd79e76..e6f2406 100644 --- a/mcp_server/tools.py +++ b/mcp_server/tools.py @@ -32,6 +32,13 @@ def __init__(self, workdir: Path, gridlog: GridLogAdapter, store: JobStoreAdapte @classmethod def from_env(cls) -> EngineTools: work = Path(os.environ.get("GRIDPRINT_WORKDIR", REPO / "work")) + if os.environ.get("GRIDPRINT_MODE") == "aws": # same adapters as the API and worker (SPEC 5) + from api.aws_adapters import DynamoJobStore + from node.ingest.aws_gridlog import DynamoGridLog + + region = os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") or "us-east-1" + return cls(work, DynamoGridLog(table_name=os.environ["GRIDLOG_TABLE"], region=region), + DynamoJobStore(region)) grid = Path(os.environ.get("GRIDPRINT_GRIDLOG", REPO / "data" / "gridlog")) store = Path(os.environ.get("GRIDPRINT_STORE", REPO / "work" / "store")) return cls(work, LocalGridLog(grid), LocalJobStore(store)) diff --git a/node/firmware/FLASHING.md b/node/firmware/FLASHING.md index 0fce843..af327fb 100644 --- a/node/firmware/FLASHING.md +++ b/node/firmware/FLASHING.md @@ -92,3 +92,8 @@ Topic `gridprint//`, QoS 1, one message per UTC second: `quality` is `gps` (PPS-disciplined timing) or `ntp` (SNTP fallback, larger timing uncertainty; `gps_lock` is then false). The ingest Lambda (`node/ingest/handler.py`) rejects any other value. + +## MQTT topic prefix + +The Node publishes to `//`. Set "MQTT topic prefix" in `idf.py menuconfig` to the value +of `aws.resource_prefix` in `config/default.yaml` (default `orchard`). `node/provision.py` prints the exact topic. diff --git a/node/firmware/main/Kconfig.projbuild b/node/firmware/main/Kconfig.projbuild index b70de7b..e9463e0 100644 --- a/node/firmware/main/Kconfig.projbuild +++ b/node/firmware/main/Kconfig.projbuild @@ -18,6 +18,10 @@ menu "GRIDPRINT Node" string "Grid zone id (config/zones.yaml)" default "IN" + config GRIDPRINT_TOPIC_PREFIX + string "MQTT topic prefix (aws.resource_prefix in config/default.yaml)" + default "orchard" + config GRIDPRINT_NODE_ID string "Node id (IoT thing name and MQTT client id)" default "node-01" diff --git a/node/firmware/main/main.c b/node/firmware/main/main.c index 4cfa303..f39563d 100644 --- a/node/firmware/main/main.c +++ b/node/firmware/main/main.c @@ -6,7 +6,7 @@ // 2. For each UTC second the crossings between two PPS edges are filtered (glitch, band and median // filters in freq_calc.c) and converted to a frequency. // 3. Each second a JSON message {zone, node_id, t_utc_s, freq_hz, quality, gps_lock} is published with -// QoS 1 over MQTT/TLS (esp-mqtt) to AWS IoT Core topic gridprint//. +// QoS 1 over MQTT/TLS (esp-mqtt) to AWS IoT Core topic // (prefix = aws.resource_prefix). // 4. Messages go through a RAM FIFO. When offline they stay in the FIFO; after reconnect the FIFO is // drained oldest first (back-fill), at most CONFIG_GRIDPRINT_BACKFILL_RATE messages per second. // Fallback (SPEC V11): without GPS lock, and if enabled, seconds are framed by SNTP time and sent with @@ -336,7 +336,8 @@ static void uplink_task(void *arg) { char topic[96]; char payload[200]; - snprintf(topic, sizeof(topic), "gridprint/%s/%s", CONFIG_GRIDPRINT_ZONE, CONFIG_GRIDPRINT_NODE_ID); + snprintf(topic, sizeof(topic), "%s/%s/%s", CONFIG_GRIDPRINT_TOPIC_PREFIX, CONFIG_GRIDPRINT_ZONE, + CONFIG_GRIDPRINT_NODE_ID); const int gap_ms = 1000 / CONFIG_GRIDPRINT_BACKFILL_RATE; for (;;) { sample_t s; diff --git a/node/ingest/aws_gridlog.py b/node/ingest/aws_gridlog.py index 4152a59..2e7a314 100644 --- a/node/ingest/aws_gridlog.py +++ b/node/ingest/aws_gridlog.py @@ -15,7 +15,7 @@ from node.ingest.handler import DynamoWriter, day_key DAY_S = 86400 -DEFAULT_TABLE = os.environ.get("GRIDPRINT_GRIDLOG_TABLE", "gridprint-GridLog") # infra data stack name +DEFAULT_TABLE = os.environ.get("GRIDLOG_TABLE") or os.environ.get("GRIDPRINT_GRIDLOG_TABLE") or "gridprint-GridLog" def _query_all(table: Any, **kw: Any) -> list[dict[str, Any]]: diff --git a/node/ingest/handler.py b/node/ingest/handler.py index 58eab0a..c0a6905 100644 --- a/node/ingest/handler.py +++ b/node/ingest/handler.py @@ -35,7 +35,7 @@ LOG = logging.getLogger("gridprint.ingest") LOG.setLevel(logging.INFO) -QUALITIES = ("gps", "ntp") +QUALITIES = ("gps", "ntp", "sim") # "sim": generated by simulated_publisher.py, never a measurement NODE_SOURCES = ("node", "sim") ZONE_RE = re.compile(r"^[A-Z0-9][A-Z0-9-]{0,15}$") NODE_RE = re.compile(r"^[A-Za-z0-9_-]{1,64}$") diff --git a/node/ingest/import_wav_reference.py b/node/ingest/import_wav_reference.py new file mode 100644 index 0000000..7c93d2e --- /dev/null +++ b/node/ingest/import_wav_reference.py @@ -0,0 +1,89 @@ +"""Import a recorded mains waveform (.wav) as a 1 s grid-frequency reference (SPEC 10.4 importers). + +Public ENF datasets ship the mains reference as audio-rate voltage recordings (e.g. EV-ENFD `ENF_Reference`, +one hour per file). Frequency per second: rising zero crossings of the band-limited waveform, timed with +linear interpolation between samples; the mean period inside each second gives that second's frequency. + + python node/ingest/import_wav_reference.py REF.wav --zone CN --t0-utc 2023-01-01T00:00:00Z --source ev-enfd +The file has no absolute time, so the start time is given by the caller (documented with the dataset). +""" + +from __future__ import annotations + +import argparse +import sys +import wave +from datetime import datetime +from pathlib import Path + +import numpy as np + +REPO = Path(__file__).resolve().parents[2] +sys.path[:0] = [str(REPO), str(REPO / "engine")] + + +def read_wav(path: Path) -> tuple[np.ndarray, int]: + with wave.open(str(path), "rb") as w: + n, ch, width, fs = w.getnframes(), w.getnchannels(), w.getsampwidth(), w.getframerate() + raw = w.readframes(n) + dt = {1: np.uint8, 2: np.int16, 4: np.int32}[width] + x = np.frombuffer(raw, dtype=dt).astype(np.float64).reshape(-1, ch)[:, 0] + if width == 1: + x -= 128.0 + return x, fs + + +def bandpass(x: np.ndarray, fs: int, nominal: float, half_width: float = 5.0) -> np.ndarray: + """FFT band-pass around the nominal frequency (float64). Removes DC and harmonics before crossings.""" + spec = np.fft.rfft(x - x.mean()) + f = np.fft.rfftfreq(len(x), 1.0 / fs) + spec[(f < nominal - half_width) | (f > nominal + half_width)] = 0 + return np.fft.irfft(spec, len(x)) + + +def per_second_frequency(x: np.ndarray, fs: int, nominal: float) -> tuple[np.ndarray, np.ndarray]: + """Return (second index from file start, frequency Hz) for every whole second with crossings.""" + y = bandpass(x, fs, nominal) + s0, s1 = y[:-1], y[1:] + idx = np.flatnonzero((s0 < 0) & (s1 >= 0)) + tc = (idx + (-s0[idx]) / (s1[idx] - s0[idx])) / fs # crossing times, linear interpolation + periods = np.diff(tc) + mids = 0.5 * (tc[1:] + tc[:-1]) + ok = np.abs(periods - 1.0 / nominal) < 0.1 / nominal # reject glitches (±10 % of a period) + secs = np.floor(mids[ok]).astype(np.int64) + out_s, out_f = [], [] + last = int(len(x) / fs) - 1 + for s in np.unique(secs): + if s <= 0 or s >= last: # the FFT band-pass biases the first and last second (edge effects) + continue + p = periods[ok][secs == s] + if len(p) >= 0.8 * nominal: + out_s.append(int(s)) + out_f.append(1.0 / float(p.mean())) + return np.array(out_s, np.int64), np.array(out_f, np.float64) + + +def main(argv: list[str] | None = None) -> int: + from gridprint_engine.adapters import LocalGridLog + + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("wav", nargs="+") + ap.add_argument("--zone", required=True) + ap.add_argument("--nominal", type=float, default=50.0) + ap.add_argument("--t0-utc", required=True, help="UTC start time of the first file (ISO 8601)") + ap.add_argument("--source", default="wav-import") + ap.add_argument("--root", default=str(REPO / "data" / "gridlog")) + a = ap.parse_args(argv) + t0 = int(datetime.fromisoformat(a.t0_utc.replace("Z", "+00:00")).timestamp()) + gl = LocalGridLog(Path(a.root)) + for p in a.wav: + x, fs = read_wav(Path(p)) + s, f = per_second_frequency(x, fs, a.nominal) + gl.write(a.zone, t0 + s, f, source=a.source, node=Path(p).stem, quality="reference-wav") + print(f"{p}: {len(s)} seconds, {fs} Hz sample rate, frequency {f.min():.4f}–{f.max():.4f} Hz") + t0 += int(len(x) / fs) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/node/ingest/simulated_publisher.py b/node/ingest/simulated_publisher.py index 4a0311e..82b8fc6 100644 --- a/node/ingest/simulated_publisher.py +++ b/node/ingest/simulated_publisher.py @@ -52,7 +52,7 @@ def ou_track(n: int, f0: float, seed: int, theta: float = THETA, sigma: float = def messages(zone: str, nodes: list[str], t0: int, seconds: int, seed: int = 0, - quality: str = "gps", drop_fraction: float = 0.0) -> Iterator[dict[str, Any]]: + quality: str = "sim", drop_fraction: float = 0.0) -> Iterator[dict[str, Any]]: """Yield node messages in time order. `drop_fraction` removes random seconds (to test gap reports).""" f0 = nominal_hz(zone) track = ou_track(seconds, f0, seed) @@ -132,7 +132,7 @@ def main(argv: list[str] | None = None) -> int: ap.add_argument("--seconds", type=int, default=3600) ap.add_argument("--start", type=int, default=None, help="first UTC epoch second (default: now - seconds)") ap.add_argument("--seed", type=int, default=0) - ap.add_argument("--quality", choices=["gps", "ntp"], default="gps") + ap.add_argument("--quality", choices=["sim", "gps", "ntp"], default="sim") g = ap.add_mutually_exclusive_group(required=True) g.add_argument("--local", type=Path, help="LocalGridLog root, for example work/gridlog") g.add_argument("--iot", action="store_true", help="publish to AWS IoT Core") diff --git a/node/ingest/tests/test_import_wav_reference.py b/node/ingest/tests/test_import_wav_reference.py new file mode 100644 index 0000000..f57b210 --- /dev/null +++ b/node/ingest/tests/test_import_wav_reference.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import wave +from pathlib import Path + +import numpy as np + +from gridprint_engine.adapters import LocalGridLog +from node.ingest import import_wav_reference as iw + + +def _wav(path: Path, freq: np.ndarray, fs: int = 8000) -> None: + t = np.arange(len(freq) * fs) / fs + f_inst = np.repeat(freq, fs) + phase = 2 * np.pi * np.cumsum(f_inst) / fs + x = 0.8 * np.sin(phase) + 0.1 * np.sin(3 * phase) + 0.01 * np.random.default_rng(0).normal(size=len(t)) + with wave.open(str(path), "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(fs) + w.writeframes((x * 30000).astype(np.int16).tobytes()) + + +def test_per_second_frequency_recovers_track(tmp_path): + truth = 50 + 0.02 * np.sin(np.arange(30) / 5) + _wav(tmp_path / "r.wav", truth) + x, fs = iw.read_wav(tmp_path / "r.wav") + s, f = iw.per_second_frequency(x, fs, 50.0) + assert len(s) >= 28 + assert np.max(np.abs(f[2:-2] - truth[s[2:-2]])) < 0.002 + + +def test_cli_writes_gridlog(tmp_path): + _wav(tmp_path / "r.wav", np.full(10, 50.01)) + assert iw.main([str(tmp_path / "r.wav"), "--zone", "CN", "--t0-utc", "2023-01-01T00:00:00Z", + "--root", str(tmp_path / "gl")]) == 0 + t, f = LocalGridLog(tmp_path / "gl").read("CN", 1672531200, 1672531210) + assert len(t) >= 8 and np.allclose(f, 50.01, atol=1e-3) diff --git a/node/ingest/tests/test_ingest_data_quality.py b/node/ingest/tests/test_ingest_data_quality.py index 6788a24..c085fc6 100644 --- a/node/ingest/tests/test_ingest_data_quality.py +++ b/node/ingest/tests/test_ingest_data_quality.py @@ -24,7 +24,7 @@ def test_gaps_coverage_and_outliers(): def test_cli_exit_codes(tmp_path, capsys): - sp.run_local(tmp_path, sp.messages("SIM50", ["n1"], T, 3600, seed=2)) + sp.run_local(tmp_path, sp.messages("SIM50", ["n1"], T, 3600, seed=2, quality="gps")) # GPS-labelled test rows assert dq.main(["--zone", "SIM50", "--hours", "1", "--end", str(T + 3599), "--root", str(tmp_path)]) == 0 out = capsys.readouterr().out assert "coverage_pct=100.000" in out and "gps_lock_pct=100.000" in out and "PASS" in out diff --git a/node/ingest/tests/test_ingest_e2e.py b/node/ingest/tests/test_ingest_e2e.py index e20149d..e887c1f 100644 --- a/node/ingest/tests/test_ingest_e2e.py +++ b/node/ingest/tests/test_ingest_e2e.py @@ -34,7 +34,8 @@ def test_simulated_publisher_end_to_end(tmp_path): rows = data_quality.read_local_rows(tmp_path, "SIM50", T, T + 599) assert {r["source"] for r in rows} == {"sim"} rep = data_quality.analyze(rows, T, T + 599, 50.0) - assert rep["coverage_pct"] == 100.0 and rep["gps_lock_pct"] == 100.0 and rep["gaps"] == [] + # simulated rows are labelled "sim", so they never count as GPS-locked + assert rep["coverage_pct"] == 100.0 and rep["gps_lock_pct"] == 0.0 and rep["gaps"] == [] def test_invalid_messages_are_not_written(tmp_path): diff --git a/node/ingest/tests/test_node_provision.py b/node/ingest/tests/test_node_provision.py index 008c751..0fdeaa5 100644 --- a/node/ingest/tests/test_node_provision.py +++ b/node/ingest/tests/test_node_provision.py @@ -16,7 +16,7 @@ def __init__(self, stack_deployed=False): self.stack_deployed = stack_deployed def get_policy(self, policyName): - if self.stack_deployed and policyName == "gridprint-node": + if self.stack_deployed and policyName == f"{prov.PREFIX}-node": return {} raise ClientError({"Error": {"Code": "ResourceNotFoundException", "Message": ""}}, "GetPolicy") @@ -47,7 +47,7 @@ def test_policy_is_scoped_to_node_topic(): doc = prov.policy_document("us-east-1", "123456789012", "IN", "node-01") res = [s["Resource"] for s in doc["Statement"]] assert res == ["arn:aws:iot:us-east-1:123456789012:client/node-01", - "arn:aws:iot:us-east-1:123456789012:topic/gridprint/IN/node-01"] + f"arn:aws:iot:us-east-1:123456789012:topic/{prov.PREFIX}/IN/node-01"] def test_provision_writes_certs(tmp_path): @@ -56,9 +56,9 @@ def test_provision_writes_certs(tmp_path): names = [c[0] for c in iot.calls] assert names == ["create_policy", "create_thing", "create_keys_and_certificate", "attach_policy", "attach_thing_principal", "describe_endpoint"] - assert iot.calls[3][1]["policyName"] == "gridprint-node-node-01" + assert iot.calls[3][1]["policyName"] == f"{prov.PREFIX}-node-node-01" assert (tmp_path / "private.pem.key").read_text() == "KEY" - assert r["topic"] == "gridprint/IN/node-01" + assert r["topic"] == f"{prov.PREFIX}/IN/node-01" def test_provision_uses_stack_policy_and_thing_type(tmp_path): @@ -66,6 +66,6 @@ def test_provision_uses_stack_policy_and_thing_type(tmp_path): r = prov.provision(iot, FakeSts(), "us-east-1", "IN", "node-01", tmp_path) calls = dict(iot.calls) assert "create_policy" not in calls - assert calls["create_thing"]["thingTypeName"] == "gridprint-node" + assert calls["create_thing"]["thingTypeName"] == f"{prov.PREFIX}-node" assert calls["create_thing"]["attributePayload"]["attributes"]["zone"] == "IN" - assert calls["attach_policy"]["policyName"] == "gridprint-node" and r["policy"] == "gridprint-node" + assert calls["attach_policy"]["policyName"] == f"{prov.PREFIX}-node" and r["policy"] == f"{prov.PREFIX}-node" diff --git a/node/provision.py b/node/provision.py index bd25ac8..3db592c 100644 --- a/node/provision.py +++ b/node/provision.py @@ -18,9 +18,13 @@ from pathlib import Path from typing import Any +import yaml + CERT_DIR = Path(__file__).resolve().parent / "firmware" / "certs" -STACK_POLICY = "gridprint-node" # created by infra/gridprint_infra/iot.py -THING_TYPE = "gridprint-node" +PREFIX = str((yaml.safe_load((Path(__file__).resolve().parents[1] / "config" / "default.yaml").read_text()) + .get("aws") or {}).get("resource_prefix", "gridprint")) # aws.resource_prefix +STACK_POLICY = f"{PREFIX}-node" # created by infra/gridprint_infra/iot.py +THING_TYPE = f"{PREFIX}-node" def policy_document(region: str, account: str, zone: str, node_id: str) -> dict[str, Any]: @@ -30,7 +34,7 @@ def policy_document(region: str, account: str, zone: str, node_id: str) -> dict[ "Statement": [ {"Effect": "Allow", "Action": "iot:Connect", "Resource": f"{base}:client/{node_id}"}, {"Effect": "Allow", "Action": "iot:Publish", - "Resource": f"{base}:topic/gridprint/{zone}/{node_id}"}, + "Resource": f"{base}:topic/{PREFIX}/{zone}/{node_id}"}, ], } @@ -53,7 +57,7 @@ def provision(iot: Any, sts: Any, region: str, zone: str, node_id: str, if _exists(lambda: iot.get_policy(policyName=STACK_POLICY)): policy_name = STACK_POLICY else: - policy_name = f"gridprint-node-{node_id}" + policy_name = f"{PREFIX}-node-{node_id}" if not _exists(lambda: iot.get_policy(policyName=policy_name)): iot.create_policy(policyName=policy_name, policyDocument=json.dumps(policy_document(region, account, zone, node_id))) @@ -73,7 +77,7 @@ def provision(iot: Any, sts: Any, region: str, zone: str, node_id: str, (out_dir / "endpoint.txt").write_text(endpoint + "\n") (out_dir / "certificate_arn.txt").write_text(cert["certificateArn"] + "\n") return {"thing": node_id, "policy": policy_name, "certificate_arn": cert["certificateArn"], - "endpoint": endpoint, "topic": f"gridprint/{zone}/{node_id}", "cert_dir": str(out_dir)} + "endpoint": endpoint, "topic": f"{PREFIX}/{zone}/{node_id}", "cert_dir": str(out_dir)} def main(argv: list[str] | None = None) -> int: diff --git a/scripts/check_numbers_sourced.py b/scripts/check_numbers_sourced.py index baf02a3..661f597 100644 --- a/scripts/check_numbers_sourced.py +++ b/scripts/check_numbers_sourced.py @@ -24,7 +24,7 @@ re.compile(r"\b(?:OpenCV|Python|WCAG|ES|Graviton|python|opencv)\s?\d+(?:\.\d+)*\b"), re.compile(r"\b(?:c8g|c7g|c7i|m7g|m7i|c8i)\.\w+\b"), re.compile(r"\b[MHVRDSB]\d+(?:\.\d+)*\b"), # milestone/rule/stage ids - re.compile(r"\b(?:Part|Section|§)\s?\d+(?:\.\d+)*\b"), + re.compile(r"\b(?:Part|Section|§|SPEC)\s?\d+(?:\.\d+)*\b"), re.compile(r"\b\d:\d{2}(?:[–-]\d:\d{2})?\b"), # video timecodes re.compile(r"`[^`]*`"), # code spans re.compile(r"\]\([^)]*\)"), # link targets diff --git a/scripts/seed_aws.py b/scripts/seed_aws.py new file mode 100644 index 0000000..f58d1fa --- /dev/null +++ b/scripts/seed_aws.py @@ -0,0 +1,59 @@ +"""Seed the deployed system: demo-case inputs to the results bucket (`demo/`) and reference GridLog data +from the local GridLog folder into the DynamoDB GridLog table. Uses the `gridprint` profile only. + + python scripts/seed_aws.py --results-bucket --gridlog-table [--zone SIM50] +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +import boto3 +import numpy as np +import yaml + +REPO = Path(__file__).resolve().parents[1] +sys.path[:0] = [str(REPO), str(REPO / "engine")] + + +def main() -> int: + from gridprint_engine.adapters import LocalGridLog + from node.ingest.aws_gridlog import DynamoGridLog + + ap = argparse.ArgumentParser() + ap.add_argument("--results-bucket", required=True) + ap.add_argument("--gridlog-table", required=True) + ap.add_argument("--zone", default="SIM50") + ap.add_argument("--data-root", default=str(REPO), help="folder that holds data/demo and data/gridlog") + ap.add_argument("--gridlog-root", default=None) + a = ap.parse_args() + root = Path(a.data_root) + a.gridlog_root = a.gridlog_root or str(root / "data" / "gridlog") + cfg = yaml.safe_load((REPO / "config" / "default.yaml").read_text()) + s = boto3.Session(profile_name=cfg["aws"]["profile"], region_name=cfg["aws"]["region"]) + s3 = s.client("s3") + cases = yaml.safe_load((REPO / "config" / "demo_cases.yaml").read_text())["cases"] + for c in cases: + for field in ("video", "sidecar"): + rel = c.get(field) + if rel and (root / rel).exists(): + s3.upload_file(str(root / rel), a.results_bucket, f"demo/{Path(rel).name}") + print(f"uploaded demo/{Path(rel).name}") + local = LocalGridLog(Path(a.gridlog_root)) + files = sorted((Path(a.gridlog_root) / a.zone).glob("*.csv")) + remote = DynamoGridLog(table_name=a.gridlog_table, profile=cfg["aws"]["profile"], region=cfg["aws"]["region"]) + total = 0 + for f in files: + day0 = int(np.datetime64(f"{f.stem[:4]}-{f.stem[4:6]}-{f.stem[6:]}").astype("datetime64[s]").astype(int)) + t, fr = local.read(a.zone, day0, day0 + 86399) + if len(t): + remote.write(a.zone, t, fr, source="simulator", node="seed") + total += len(t) + print(f"GridLog {a.zone}: {total} seconds written to {a.gridlog_table}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_stress.py b/tests/test_stress.py new file mode 100644 index 0000000..f36e7c2 --- /dev/null +++ b/tests/test_stress.py @@ -0,0 +1,158 @@ +"""Stress and fuzz tests: invariants that must hold for any input. + +- S13 verdict logic over random reason sets and flags (R1, R2, R3, R9). +- Agent policies under a random (adversarial) planner: verdict always computed, never FINAL for CONTRADICTED, + budgets respected, no analysis after the verdict. +- Engine on malformed and edge-case files: no crash with a wrong verdict; never CONTRADICTED without signal. +""" + +from __future__ import annotations + +import asyncio +import random +from pathlib import Path +from typing import Any + +import av +import numpy as np +import pytest + +from agent.orchestrator import run_job +from agent.policies import ALL_TOOLS, Action +from gridprint_engine.adapters import LocalGridLog +from gridprint_engine.pipeline import run_default +from gridprint_engine.schemas import LifecycleState, Reason, Verdict +from gridprint_engine.verdict import STOP_REASONS, VerdictInputs, decide + +ALL_REASONS = list(Reason) + + +def test_verdict_invariants_fuzz(): + rng = random.Random(0) + for _ in range(20000): + rs = set(rng.sample(ALL_REASONS, rng.randint(0, 8))) + inp = VerdictInputs(reasons=rs, match_passes=rng.random() < 0.5, claim_present=rng.random() < 0.3, + claim_within_tolerance=rng.choice([None, True, False]), + significant_edit=rng.random() < 0.3, face_anomaly=rng.random() < 0.3) + out = decide(inp) + assert out.verdict in (Verdict.VERIFIED, Verdict.CONTRADICTED, Verdict.INCONCLUSIVE) # R1 + if rs & STOP_REASONS: + assert out.verdict == Verdict.INCONCLUSIVE # R2 (absence of signal) and S13.1 + if out.verdict == Verdict.CONTRADICTED: + assert out.state == LifecycleState.PENDING_REVIEW # R3 + assert inp.significant_edit or Reason.SEGMENT_OFFSET_INCONSISTENT in rs or ( + inp.claim_present and inp.match_passes and inp.claim_within_tolerance is False) # R9 + else: + assert out.state == LifecycleState.DRAFT + if out.verdict == Verdict.VERIFIED: + assert inp.match_passes and not inp.significant_edit + + +class RandomPlanner: + """Adversarial planner double: random tools, random (often disallowed) arguments, random stops.""" + + kind = "fallback" + model_id = None + + def __init__(self, seed: int): + self.rng = random.Random(seed) + + def next_action(self, history: list[Any]) -> Action: + r = self.rng.random() + if r < 0.05: + return Action(None, {}, "random stop") + tool = self.rng.choice(sorted(ALL_TOOLS) + ["delete_everything", "set_threshold"]) + args = self.rng.choice([{}, {"final": True}, {"snr_weak_db": -100}, {"verdict": "VERIFIED"}, + {"strategy": "top"}, {"kind": "camera_model", "message": "m"}]) + return Action(tool, args, "random") + + +class RecordingTools: + def __init__(self, seed: int): + self.rng = random.Random(seed) + self.calls: list[tuple[str, dict[str, Any]]] = [] + + async def call(self, tool: str, args: dict[str, Any]) -> dict[str, Any]: + self.calls.append((tool, args)) + if self.rng.random() < 0.1: + raise RuntimeError("tool failure") + if tool == "compute_verdict": + return {"verdict": self.rng.choice(["VERIFIED", "CONTRADICTED", "INCONCLUSIVE"])} + return {} + + +@pytest.mark.parametrize("seed", range(200)) +def test_policies_hold_under_random_planner(seed): + tools = RecordingTools(seed) + max_calls = 20 + trace, hist = asyncio.run(run_job("j", RandomPlanner(seed), tools, 4, max_calls)) + names = [c[0] for c in tools.calls] + assert set(names) <= ALL_TOOLS + assert len(names) <= max_calls + 3 + assert sum(n == "extract_flicker" for n in names) <= 4 + for _, args in tools.calls: + assert not ({"snr_weak_db", "verdict"} & set(args)) + # history and tool calls align one to one; find the first verdict that succeeded + first = next((i for i, h in enumerate(hist) if h.tool == "compute_verdict" and h.error is None), None) + if first is not None: + analysis = {"probe_video", "estimate_readout", "analyze_motion", "propose_patches", "extract_flicker", + "match_reference", "check_continuity", "face_region_check", "compute_verdict"} + assert all(n not in analysis for n, _ in tools.calls[first + 1:]) + if hist[first].result["verdict"] == "CONTRADICTED": + assert not [a for n, a in tools.calls[first + 1:] if n == "render_report" and a.get("final", True)] + for st in trace.steps: + assert st.t_start and st.t_end and st.planner and st.tool + + +# ---------------------------------------------------------------- malformed and edge-case inputs +def _write(path: Path, frames: list[np.ndarray], fps: int = 30) -> Path: + with av.open(str(path), "w") as c: + s = c.add_stream("libx264", rate=fps) + s.height, s.width = frames[0].shape + s.pix_fmt = "yuv420p" + for f in frames: + for p in s.encode(av.VideoFrame.from_ndarray(np.stack([f] * 3, -1), format="rgb24")): + c.mux(p) + for p in s.encode(): + c.mux(p) + return path + + +EDGE = { + "black": lambda: [np.zeros((48, 64), np.uint8)] * 30, + "saturated": lambda: [np.full((48, 64), 255, np.uint8)] * 30, + "noise": lambda: [np.random.default_rng(i).integers(0, 255, (48, 64), dtype=np.uint8) for i in range(30)], + "one_frame": lambda: [np.full((48, 64), 120, np.uint8)], + "flat_grey": lambda: [np.full((48, 64), 120, np.uint8)] * 60, +} + + +@pytest.mark.parametrize("name", sorted(EDGE)) +def test_edge_case_videos_never_contradicted(tmp_path, name): + v = _write(tmp_path / f"{name}.mp4", EDGE[name]()) + job = run_default(v, tmp_path / "work", LocalGridLog(tmp_path / "gridlog")) + assert job.result is not None + assert job.result.verdict == Verdict.INCONCLUSIVE + + +@pytest.mark.parametrize("payload", [b"", b"\x00" * 10, b"not a video", b"\x00\x00\x00\x18ftypisom" + b"\x01" * 200]) +def test_malformed_files_raise_clean_errors(tmp_path, payload): + p = tmp_path / "bad.mp4" + p.write_bytes(payload) + with pytest.raises(Exception) as ei: + run_default(p, tmp_path / "work", LocalGridLog(tmp_path / "gridlog")) + assert not isinstance(ei.value, (SystemExit, KeyboardInterrupt)) + + +def test_truncated_real_video_does_not_verify(tmp_path): + from eval.simulator.render import SimConfig, simulate + + tr = simulate(SimConfig(seed=5, duration_s=3, width=64, height=48, ref_margin_s=20), tmp_path, "t") + data = (tmp_path / tr["video"]).read_bytes() + cut = tmp_path / "trunc.mp4" + cut.write_bytes(data[: len(data) // 2]) + try: + job = run_default(cut, tmp_path / "work", LocalGridLog(tmp_path / tr["gridlog_root"])) + except Exception: + return # a clean failure is acceptable for a broken container + assert job.result is not None and job.result.verdict != Verdict.CONTRADICTED