From 3e1740aa9daa9a2ce65e73394bafb05ab5f73498 Mon Sep 17 00:00:00 2001 From: Ishaan Date: Wed, 8 Jul 2026 14:02:34 -0400 Subject: [PATCH 1/3] docs --- .gitignore | 2 ++ AGENTS.md | 5 ++--- README.md | 8 ++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 0751f1c..bf8901b 100644 --- a/.gitignore +++ b/.gitignore @@ -48,5 +48,7 @@ studies/ # Personal study manifests and worker scripts train_*.yaml +*.hpo.yaml +bridge_crack_unet_v2.yaml colab_worker.py diff --git a/AGENTS.md b/AGENTS.md index 47c32b6..a1504de 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -55,7 +55,7 @@ The worker contract is exactly three calls; full reference in `docs/INTEGRATION. ### Statistical confidence (caveat, not a gate) -`get_study_data` includes `statistical_confidence` derived from completed-trial count: +The `hpo://studies/{name}/packet` resource includes `statistical_confidence` derived from completed-trial count: | Tier | Completed trials | Agent behavior | |------|------------------|----------------| @@ -69,7 +69,7 @@ Reviews are never hard-blocked at low confidence — the dashboard shows a banne When the user asks about study progress, trial results, or health: -1. Call `get_study_data(study_name)` to retrieve trial telemetry, health tier, fANOVA importances, best trials, and OOM patterns. +1. Read the `hpo://studies/{name}/packet` resource to retrieve trial telemetry, health tier, fANOVA importances, best trials, and OOM patterns. 2. Summarize: current best score, health status, stagnation indicators, and any OOM warnings. 3. If health is `watch` or `intervene`, explain the reason and offer to explore the data further. 4. If `vram_telemetry.oom_count > 0`, inspect the `oom_trials` list: @@ -84,7 +84,6 @@ When a completed trial is reported or the background daemon polls health, the sy **Agent Action (Cursor, Antigravity, and Claude Code)**: * Eagerly check for the existence of `.hpo_status.json` in the project root. * If `health_tier` is `"watch"` or `"intervene"`, immediately notify the user in your first response, explain the `health_reason`, and proactively offer to inspect the study data. -* Offer to use the `/goal` slash command to help coordinate and automate resolution of search space stagnation or failures. ## Guardrails (always) diff --git a/README.md b/README.md index 020baee..6a43cb8 100644 --- a/README.md +++ b/README.md @@ -100,11 +100,11 @@ Pathfinder exposes MCP tools that let your IDE agent (Cursor, Claude Code, Antig ### Inspection -1. Agent calls `get_study_data` to retrieve trial telemetry, health tier, fANOVA importances, and trial data +1. Agent reads the `hpo://studies/{name}/packet` resource to retrieve trial telemetry, health tier, fANOVA importances, and trial data 2. Agent summarizes: current best score, health status, OOM rate, stagnation warnings 3. Recommended search space adjustments happen by you through the dashboard Settings UI or `hpo_cli.py` -Key MCP tools: `validate_manifest`, `init_from_manifest`, `get_study_data`, `get_study_cards`, `export_manifest`. +Key MCP tools: `validate_manifest`, `init_from_manifest`. Data is retrieved via resources (`hpo://studies/{name}/packet`, `hpo://studies/{name}/cards`). Trigger phrases: say **"integrate HPO"** or **"wire hyperparameter tuning"** to onboard. Say **"show study health"** or **"check HPO progress"** to inspect. @@ -209,7 +209,7 @@ Name: `pathfinder`, Type: `command`, Command: `source .venv/bin/activate && pyth "command": "python3", "args": ["hpo_mcp_server.py"], "env": { - "HPO_DATABASE_URL": "sqlite:///./hpo_studies.db" + "HPO_DATABASE_URL": "sqlite:///./.data/hpo_studies.db" } } } @@ -258,7 +258,7 @@ pytest tests/ -q ## Dev Notes -- MCP tool design to inspect telemetry and modify training scripts, refactored the architecture to decouple agentic workflows from deterministic optimization path +- MCP chosen over CLI invocation for IDE-agent integration because typed auto-discovery (tools with schemas, resources with URI templates) provides a better contract than agents string-matching CLI output - Implemented concurrency patterns for distributed workers, real-time detection of crashed processes - Optimized SQLite backend performance using Write-Ahead Logging; allowing concurrent broker writes, dashboard rendering, and MCP queries without read-write blocks From 284bb9a9b5268e3ba85554e7b951a168e4e5e7e3 Mon Sep 17 00:00:00 2001 From: Ishaan Date: Wed, 8 Jul 2026 14:02:49 -0400 Subject: [PATCH 2/3] quickstart + mcp tests --- tests/test_http_auth.py | 127 ++++++++++++++++++++++++++++- tests/test_mcp_server.py | 167 ++++++++------------------------------ tests/test_performance.py | 122 ++++++++++++++++++++++++++++ tests/test_quickstart.py | 39 +++++++++ 4 files changed, 321 insertions(+), 134 deletions(-) create mode 100644 tests/test_performance.py create mode 100644 tests/test_quickstart.py diff --git a/tests/test_http_auth.py b/tests/test_http_auth.py index 4da551f..33b2137 100644 --- a/tests/test_http_auth.py +++ b/tests/test_http_auth.py @@ -1,6 +1,14 @@ """Auth and lease-ownership tests for the broker HTTP surface.""" + +import os import uuid +import pytest + + +# --------------------------------------------------------------------------- +# Lease ownership (worker-level auth) +# --------------------------------------------------------------------------- def test_report_requires_lease_ownership(client, initialized_study): owner = str(uuid.uuid4()) @@ -10,7 +18,6 @@ def test_report_requires_lease_ownership(client, initialized_study): assert sug.status_code == 200, sug.text trial_id = sug.json()["trial_id"] - # Intruder (no lease) cannot report. bad = client.post( "/api/report_epoch", json={"study_name": initialized_study, "trial_id": trial_id, "worker_id": intruder, @@ -18,7 +25,6 @@ def test_report_requires_lease_ownership(client, initialized_study): ) assert bad.status_code == 403 - # Missing worker_id also cannot report. nobody = client.post( "/api/report_epoch", json={"study_name": initialized_study, "trial_id": trial_id, @@ -26,7 +32,6 @@ def test_report_requires_lease_ownership(client, initialized_study): ) assert nobody.status_code == 403 - # Owner can report. ok = client.post( "/api/report_epoch", json={"study_name": initialized_study, "trial_id": trial_id, "worker_id": owner, @@ -49,3 +54,119 @@ def test_complete_requires_lease_for_inflight_trial(client, initialized_study): "history": [{"epoch": 1, "score": 0.6, "loss": 0.4}], "state": "COMPLETE"}, ) assert bad.status_code == 403 + + +# --------------------------------------------------------------------------- +# Token auth (broker middleware) +# --------------------------------------------------------------------------- + +TEST_TOKEN = "test_auth_token_123" + + +@pytest.fixture +def enable_auth(): + os.environ["HPO_SECRET_TOKEN"] = TEST_TOKEN + yield + os.environ.pop("HPO_SECRET_TOKEN", None) + + +def test_bypass_routes_work_without_token(enable_auth, client): + """Health, root, styles.css, and /api/login are explicitly bypassed.""" + assert client.get("/health").status_code == 200 + assert client.get("/styles.css").status_code != 401 + # POST /api/login requires a body; GET returns 405 but importantly NOT 401 + assert client.get("/api/login").status_code != 401 + + +def test_protected_route_returns_401_without_token(enable_auth, client): + """Protected API routes require a token.""" + resp = client.get("/api/hpo_config?study_name=does_not_exist") + assert resp.status_code == 401, resp.text + assert "Unauthorized" in resp.json()["error"] + + +def test_x_hpo_token_header_passes_auth(enable_auth, client, initialized_study): + """X-HPO-Token header is accepted.""" + resp = client.get( + f"/api/study_details?study_name={initialized_study}", + headers={"X-HPO-Token": TEST_TOKEN}, + ) + assert resp.status_code == 200, resp.text + + +def test_authorization_bearer_header_passes_auth(enable_auth, client, initialized_study): + """Authorization: Bearer header is accepted.""" + resp = client.get( + f"/api/study_details?study_name={initialized_study}", + headers={"Authorization": f"Bearer {TEST_TOKEN}"}, + ) + assert resp.status_code == 200, resp.text + + +def test_authorization_header_without_bearer_prefix_passes_auth(enable_auth, client, initialized_study): + """Authorization header without Bearer prefix falls through to raw comparison.""" + resp = client.get( + f"/api/study_details?study_name={initialized_study}", + headers={"Authorization": TEST_TOKEN}, + ) + assert resp.status_code == 200, resp.text + + +def test_wrong_token_returns_401(enable_auth, client): + """An incorrect token is rejected.""" + resp = client.get( + "/api/hpo_config?study_name=does_not_exist", + headers={"X-HPO-Token": "wrong_token"}, + ) + assert resp.status_code == 401 + + +def test_empty_token_header_returns_401(enable_auth, client): + """An empty X-HPO-Token header is treated as missing token.""" + resp = client.get( + "/api/hpo_config?study_name=does_not_exist", + headers={"X-HPO-Token": ""}, + ) + assert resp.status_code == 401 + + +def test_login_with_correct_token_sets_cookie(enable_auth, client): + """POST /api/login with correct token returns a session cookie.""" + resp = client.post("/api/login", json={"token": TEST_TOKEN}) + assert resp.status_code == 200, resp.text + data = resp.json() + assert data.get("success") is True + assert data.get("auth_required") is True + + cookies = resp.headers.get("set-cookie", "") + assert "hpo_session" in cookies + assert "HttpOnly" in cookies + + +def test_login_with_wrong_token_returns_401(enable_auth, client): + """POST /api/login with wrong token is rejected.""" + resp = client.post("/api/login", json={"token": "bogus"}) + assert resp.status_code == 401 + + +def test_cookie_based_auth_passes_protected_route(enable_auth, client, initialized_study): + """A request carrying the hpo_session cookie passes auth on protected routes.""" + login_resp = client.post("/api/login", json={"token": TEST_TOKEN}) + assert login_resp.status_code == 200 + cookie = login_resp.headers.get("set-cookie", "") + assert "hpo_session" in cookie + + resp = client.get( + f"/api/study_details?study_name={initialized_study}", + headers={"Cookie": cookie}, + ) + assert resp.status_code == 200, resp.text + + +def test_login_returns_auth_required_false_when_no_token_configured(client): + """When no HPO_SECRET_TOKEN is set, login reports auth is not required.""" + assert "HPO_SECRET_TOKEN" not in os.environ + resp = client.post("/api/login", json={"token": "anything"}) + assert resp.status_code == 200 + data = resp.json() + assert data.get("auth_required") is False diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 20dfae6..274e5af 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -1,14 +1,13 @@ -"""Tests for the MCP server tools — verify unified error handling, return shapes, and edge cases.""" +"""Tests for the MCP server tools and resources — verify tool return shapes and resource smoke checks.""" +import json import yaml from hpo_mcp_server import ( - get_study_data, - get_study_cards, validate_manifest, init_from_manifest, - export_manifest, - resource_grill, + study_packet_resource, + study_cards_resource, ) # --------------------------------------------------------------------------- @@ -38,92 +37,6 @@ """ -# --------------------------------------------------------------------------- -# get_study_data -# --------------------------------------------------------------------------- - -def test_get_study_data_valid_study_with_completed_trial(client, initialized_study): - """Return a valid packet with counts and health when study has completed trials.""" - study_name = initialized_study - - resp = client.post( - "/api/suggest_trial", - json={"study_name": study_name, "worker_id": "w1"}, - ) - assert resp.status_code == 200 - trial_id = resp.json()["trial_id"] - - resp = client.post( - "/api/complete_trial", - json={ - "study_name": study_name, - "trial_id": trial_id, - "worker_id": "w1", - "epoch": 1, - "score": 0.80, - "loss": 0.20, - "weights_path": "model.pt", - "history": [{"epoch": 1, "score": 0.80, "loss": 0.20}], - "state": "COMPLETE", - }, - ) - assert resp.status_code == 200 - - result = get_study_data(study_name) - assert isinstance(result, dict) - assert result.get("study_name") == study_name - assert "counts" in result - assert result["counts"].get("complete", 0) >= 1 - assert "trial_bins" in result - assert "health" in result - - -def test_get_study_data_nonexistent_study(): - """Returns success=False, error when study doesn't exist.""" - result = get_study_data("nonexistent_study_xyz") - assert isinstance(result, dict) - assert result.get("success") is False - assert "error" in result - - -def test_get_study_data_empty_study(client, initialized_study): - """Returns a valid packet with zero completed trials for a fresh study.""" - result = get_study_data(initialized_study) - assert isinstance(result, dict) - assert result.get("study_name") == initialized_study - assert result.get("counts", {}).get("complete") == 0 - - -# --------------------------------------------------------------------------- -# get_study_cards -# --------------------------------------------------------------------------- - -def test_get_study_cards_valid_study_no_cards(initialized_study): - """Returns success=True, cards=[] when study exists but has no cards.""" - result = get_study_cards(initialized_study) - assert isinstance(result, dict) - assert result.get("success") is True - assert isinstance(result.get("cards"), list) - assert result.get("cards") == [] - - -def test_get_study_cards_nonexistent_study(): - """Returns success=False, error when study doesn't exist.""" - result = get_study_cards("nonexistent_study_xyz") - assert isinstance(result, dict) - assert result.get("success") is False - assert "error" in result - assert "not found" in result["error"] - - -def test_get_study_cards_no_argument(): - """Returns success=True with a list when no study_name is passed.""" - result = get_study_cards() - assert isinstance(result, dict) - assert result.get("success") is True - assert isinstance(result.get("cards"), list) - - # --------------------------------------------------------------------------- # validate_manifest # --------------------------------------------------------------------------- @@ -219,46 +132,38 @@ def test_init_from_manifest_schema_errors(): # --------------------------------------------------------------------------- -# export_manifest -# --------------------------------------------------------------------------- - -def test_export_manifest_valid_study(): - """Returns success=True with parseable YAML string for an existing study.""" - data = yaml.safe_load(VALID_MANIFEST_YAML) - study_name = "mcp_test_export_valid" - data["study_name"] = study_name - yaml_str = yaml.dump(data) - init_from_manifest(yaml_str, force=True) - - result = export_manifest(study_name) - assert isinstance(result, dict) - assert result.get("success") is True - assert "yaml_str" in result - yaml_str = result["yaml_str"] - assert isinstance(yaml_str, str) - assert len(yaml_str) > 0 - - reparsed = yaml.safe_load(yaml_str) - assert reparsed.get("study_name") == study_name - - -def test_export_manifest_nonexistent_study(): - """Returns success=False with error for a non-existent study (no longer raises).""" - result = export_manifest("nonexistent_study_xyz") - assert isinstance(result, dict) - assert result.get("success") is False - assert "error" in result - assert "not found" in result["error"].lower() - - -# --------------------------------------------------------------------------- -# resource_grill +# Resource smoke tests # --------------------------------------------------------------------------- -def test_resource_grill_static_content(): - """Resource returns a non-empty string containing expected onboarding keywords.""" - content = resource_grill() +def test_study_packet_resource_existing_study(initialized_study): + """Resource returns JSON string that parses to the expected packet shape.""" + content = study_packet_resource(initialized_study) assert isinstance(content, str) - assert len(content) > 0 - assert "validate_manifest" in content - assert "AGENTS.md" in content + packet = json.loads(content) + assert isinstance(packet, dict) + assert packet.get("study_name") == initialized_study + assert "health" in packet + assert "counts" in packet + + +def test_study_packet_resource_nonexistent_study(): + """Resource returns JSON string with error payload for missing study.""" + content = study_packet_resource("nonexistent_study_xyz") + packet = json.loads(content) + assert packet.get("success") is False + assert "error" in packet + + +def test_study_cards_resource_existing_study(initialized_study): + """Resource returns JSON string that parses to a list (or empty list).""" + content = study_cards_resource(initialized_study) + cards = json.loads(content) + assert isinstance(cards, list) + + +def test_study_cards_resource_nonexistent_study(): + """Resource returns empty list for missing study.""" + content = study_cards_resource("nonexistent_study_xyz") + cards = json.loads(content) + assert isinstance(cards, list) + assert cards == [] diff --git a/tests/test_performance.py b/tests/test_performance.py new file mode 100644 index 0000000..de15942 --- /dev/null +++ b/tests/test_performance.py @@ -0,0 +1,122 @@ +"""Performance benchmarks — backs the README claim: "Suggests hyperparameters in <10ms." + +Benchmarks two paths: + 1. Pure TPE sampling via optuna.study.ask() — the core suggestion engine. + 2. Full broker suggest via handle_api_suggest_trial — includes lease management, search-space + resolution, and categorical repair, so it carries additional bookkeeping beyond the + TPE call itself. The <10ms claim refers to (1); (2) is included for completeness. +""" + +import statistics +import time +import uuid +import optuna + +from src.db_manager import DATABASE_URL + + +MAX_P50_MS = 10 + + +def _init_study_with_trials(num_complete: int = 50) -> str: + """Create a new study, suggest + complete N trials so TPE has historical data to work with.""" + from src.onboarding import initialize_study + + study_name = f"perf_{uuid.uuid4().hex[:12]}" + space = { + "learning_rate": {"min": 1e-5, "max": 1e-2, "type": "float_log"}, + "batch_size": {"options": [2, 4, 8, 16], "active": [2, 4, 8, 16], "type": "categorical"}, + "resolution": {"options": [256, 512, 1024], "active": [256, 512, 1024], "type": "categorical"}, + "loss_weight_ratio": {"min": 0.0, "max": 1.0, "type": "float"}, + } + config = {"metric_score_label": "Score", "metric_loss_label": "Loss", "eval_protocol": {"enabled": False}} + ctx = {"hypothesis": "perf benchmark", "gpu_model": "CPU", "gpu_capacity_gb": 8.0} + initialize_study(study_name, space, config, ctx, multi_objective=True) + + study = optuna.load_study(study_name=study_name, storage=DATABASE_URL) + + for _ in range(num_complete): + trial = study.ask() + study.tell(trial.number, values=[random_loss(), random_score()]) + + return study_name + + +def random_loss() -> float: + import random + return round(random.uniform(0.05, 0.95), 4) + + +def random_score() -> float: + import random + return round(random.uniform(0.55, 0.99), 4) + + +# --------------------------------------------------------------------------- +# Benchmark 1 — pure TPE sampling +# --------------------------------------------------------------------------- + +def test_tpe_suggest_latency(): + """Assert p50 of optuna study.ask() is under the README-claimed 10ms threshold.""" + study_name = _init_study_with_trials(50) + study = optuna.load_study(study_name=study_name, storage=DATABASE_URL) + + # Warmup — let the sampler finish any JIT / one-off allocations + for _ in range(50): + trial = study.ask() + study.tell(trial.number, values=[random_loss(), random_score()]) + + timings_ms: list[float] = [] + for _ in range(200): + start = time.perf_counter_ns() + trial = study.ask() + elapsed_ns = time.perf_counter_ns() - start + timings_ms.append(elapsed_ns / 1_000_000) + # Avoid accumulating unlimited trials in memory + study.tell(trial.number, values=[random_loss(), random_score()]) + + p50 = statistics.median(timings_ms) + p95 = sorted(timings_ms)[int(len(timings_ms) * 0.95)] + p99 = sorted(timings_ms)[int(len(timings_ms) * 0.99)] + + print( + f"\n TPE ask() — {len(timings_ms):,} samples after warmup\n" + f" p50 = {p50:.2f} ms p95 = {p95:.2f} ms p99 = {p99:.2f} ms" + ) + + assert p50 < MAX_P50_MS, ( + f"TPE p50 latency {p50:.2f} ms exceeds {MAX_P50_MS} ms threshold.\n" + f"Either the benchmark environment is noisy or the claim needs updating in README." + ) + + +# --------------------------------------------------------------------------- +# Benchmark 2 — full broker suggest path +# --------------------------------------------------------------------------- + +def test_broker_suggest_latency(): + """Full handle_api_suggest_trial wall time. Not bound by the 10ms README claim, but tracked.""" + from src.suggest import SuggestRequest, handle_api_suggest_trial + + study_name = _init_study_with_trials(50) + + # Warmup + for _ in range(20): + req = SuggestRequest(study_name=study_name, worker_id="perf-w") + handle_api_suggest_trial(req) + + timings_ms: list[float] = [] + for _ in range(100): + req = SuggestRequest(study_name=study_name, worker_id="perf-w") + start = time.perf_counter_ns() + handle_api_suggest_trial(req) + elapsed_ns = time.perf_counter_ns() - start + timings_ms.append(elapsed_ns / 1_000_000) + + p50 = statistics.median(timings_ms) + p95 = sorted(timings_ms)[int(len(timings_ms) * 0.95)] + + print( + f"\n Broker full suggest — {len(timings_ms):,} samples after warmup\n" + f" p50 = {p50:.2f} ms p95 = {p95:.2f} ms" + ) diff --git a/tests/test_quickstart.py b/tests/test_quickstart.py new file mode 100644 index 0000000..0b8ff4f --- /dev/null +++ b/tests/test_quickstart.py @@ -0,0 +1,39 @@ +"""Tests for the quickstart demo endpoint — the zero-friction onboarding wizard reached from the dashboard.""" + +import optuna +from src.db_manager import DATABASE_URL + + +DEMO_STUDY = "demo_segmentation_study" + + +def test_quickstart_demo_creates_study(client): + """First call to /api/quickstart_demo registers the study and returns success.""" + resp = client.post("/api/quickstart_demo") + assert resp.status_code == 200, resp.text + data = resp.json() + assert data.get("success") is True + assert data["study_name"] == DEMO_STUDY + + study = optuna.load_study(study_name=DEMO_STUDY, storage=DATABASE_URL) + assert study is not None + + +def test_quickstart_demo_idempotent(client): + """Calling /api/quickstart_demo again returns success without error.""" + client.post("/api/quickstart_demo") + resp = client.post("/api/quickstart_demo") + assert resp.status_code == 200, resp.text + data = resp.json() + assert data.get("success") is True + assert data["study_name"] == DEMO_STUDY + + +def test_quickstart_demo_study_visible_in_config(client): + """After the demo is initialized, the study details endpoint returns config.""" + client.post("/api/quickstart_demo") + + resp = client.get(f"/api/hpo_config?study_name={DEMO_STUDY}") + assert resp.status_code == 200, resp.text + data = resp.json() + assert "metric_loss_label" in data or "metric_score_label" in data From 0cd2d606a45f738cd9055ddc791a3e1a6650d4b7 Mon Sep 17 00:00:00 2001 From: Ishaan Date: Wed, 8 Jul 2026 14:03:22 -0400 Subject: [PATCH 3/3] clean up MCP server design --- hpo_mcp_server.py | 65 ++++++++--------------------------- simulators/training_worker.py | 44 +++++++++++++++++++++--- src/db_manager.py | 5 +++ 3 files changed, 59 insertions(+), 55 deletions(-) diff --git a/hpo_mcp_server.py b/hpo_mcp_server.py index dc282d4..45e2634 100644 --- a/hpo_mcp_server.py +++ b/hpo_mcp_server.py @@ -1,35 +1,13 @@ -from typing import Optional, Dict, Any +import json +from typing import Dict, Any from mcp.server.fastmcp import FastMCP -mcp = FastMCP("Pathfinder") - from src.db_manager import init_db -# --- MCP TOOLS --- - -@mcp.tool() -def get_study_data(study_name: str) -> Dict[str, Any]: - """Returns the compacted HPO review packet, utilizing a lazy materialization cache layer.""" - from src.analytics import build_study_packet - return build_study_packet(study_name) - - -@mcp.tool() -def get_study_cards(study_name: Optional[str] = None) -> Dict[str, Any]: - """Retrieves generated study cards (model cards, recaps) from the database to enable cross-study queries.""" - from src.analytics import load_study_cards - - if study_name is not None: - import optuna - from src.db_manager import DATABASE_URL - try: - optuna.load_study(study_name=study_name, storage=DATABASE_URL) - except KeyError: - return {"success": False, "error": f"Study '{study_name}' not found."} +mcp = FastMCP("Pathfinder") - cards = load_study_cards(study_name) - return {"success": True, "cards": cards} +# --- MCP TOOLS (state-changing operations) --- @mcp.tool() def validate_manifest(yaml_str: str) -> Dict[str, Any]: @@ -70,33 +48,20 @@ def init_from_manifest(yaml_str: str, force: bool = False) -> Dict[str, Any]: return {"success": False, "error": str(e)} -@mcp.tool() -def export_manifest(study_name: str) -> Dict[str, Any]: - """Export the active search space, HPO config, and context of an existing study as a valid manifest YAML string.""" - from src.manifest import export_manifest_yaml - try: - result = export_manifest_yaml(study_name) - return {"success": True, "yaml_str": result} - except Exception as e: - return {"success": False, "error": str(e)} - - -# --- MCP PROMPT RESOURCES --- - -@mcp.resource("hpo://prompts/grill") -def resource_grill() -> str: - """Onboarding checklist: interview, then manifest loop.""" - return """# Pathfinder Onboarding (Grill + Manifest Loop) +# --- MCP RESOURCES (data retrieval) --- -See AGENTS.md for the full procedure. After interviewing the user (metrics, GPU, bounds, hypothesis): +@mcp.resource("hpo://studies/{study_name}/packet") +def study_packet_resource(study_name: str) -> str: + """Compacted HPO review packet: trial telemetry, health tier, fANOVA importances, OOM patterns.""" + from src.analytics import build_study_packet + return json.dumps(build_study_packet(study_name), indent=2) -1. Draft a YAML manifest configuration (e.g. `train.hpo.yaml`). -2. Call `validate_manifest(yaml_str)` to check for errors/warnings mechanically. -3. Call `init_from_manifest(yaml_str)` to register the study in SQLite and Optuna. -4. Call `get_study_data(study_name)` to confirm the study is accessible and healthy. -Worker integration reference: `docs/INTEGRATION.md`. Do not write json space config files to disk. -""" +@mcp.resource("hpo://studies/{study_name}/cards") +def study_cards_resource(study_name: str) -> str: + """Generated study cards (model cards, recaps) from the database.""" + from src.analytics import load_study_cards + return json.dumps(load_study_cards(study_name), indent=2) if __name__ == "__main__": diff --git a/simulators/training_worker.py b/simulators/training_worker.py index e577f11..3d3e0a4 100644 --- a/simulators/training_worker.py +++ b/simulators/training_worker.py @@ -128,26 +128,44 @@ def run_training_worker( val_history = [] final_score = 0.0 final_loss = 999.0 + final_score_fixed = 0.0 + final_loss_fixed = 999.0 pruned = False + # Prepare parameters for fixed resolution evaluation (fixed at 512px) + params_fixed = dict(params) + params_fixed["resolution"] = 512 + # 2. Run Epoch training loop for epoch in range(1, epochs_per_trial + 1): # Simulate training/val forward pass score, loss = simulate_training_epoch(epoch, params) - val_history.append({"epoch": epoch, "score": score, "loss": loss}) + score_fixed, loss_fixed = simulate_training_epoch(epoch, params_fixed) + + val_history.append({ + "epoch": epoch, + "score": score, + "loss": loss, + "score_eval_fixed": score_fixed, + "loss_eval_fixed": loss_fixed + }) - print(f" Epoch {epoch:02d}/{epochs_per_trial:02d} | Score: {score:.4f} | Loss: {loss:.4f}") + print(f" Epoch {epoch:02d}/{epochs_per_trial:02d} | Score (train): {score:.4f} | Loss: {loss:.4f} | Score (fixed eval @512px): {score_fixed:.4f}") # Record final metrics final_score = score final_loss = loss + final_score_fixed = score_fixed + final_loss_fixed = loss_fixed # 3. Intermediate epoch reporting & pruning evaluation try: should_prune = session.report_epoch( epoch=epoch, score=score, - loss=loss + loss=loss, + score_eval_fixed=score_fixed, + loss_eval_fixed=loss_fixed ) except Exception as rep_err: print(f"Error reporting epoch: {rep_err}") @@ -162,6 +180,8 @@ def run_training_worker( epoch=epoch, score=score, loss=loss, + score_eval_fixed=score_fixed, + loss_eval_fixed=loss_fixed, state="PRUNED" ) except Exception as prune_err: @@ -179,6 +199,8 @@ def run_training_worker( epoch=epochs_per_trial, score=final_score, loss=final_loss, + score_eval_fixed=final_score_fixed, + loss_eval_fixed=final_loss_fixed, weights_path=weights_path, history=val_history, state="COMPLETE" @@ -191,5 +213,17 @@ def run_training_worker( if __name__ == "__main__": - # Runs a default study local simulation of 5 trials - run_training_worker(study_name="unet_crack_segmentation", max_trials=5) + import argparse + parser = argparse.ArgumentParser(description="Simulated Pathfinder Training Worker") + parser.add_argument("--study_name", default="unet_crack_segmentation", help="Study name") + parser.add_argument("--max_trials", type=int, default=5, help="Number of trials to run") + parser.add_argument("--epochs_per_trial", type=int, default=10, help="Epochs per trial") + parser.add_argument("--broker_url", default=None, help="Broker URL") + args = parser.parse_args() + + run_training_worker( + study_name=args.study_name, + max_trials=args.max_trials, + epochs_per_trial=args.epochs_per_trial, + broker_url=args.broker_url + ) diff --git a/src/db_manager.py b/src/db_manager.py index a51e993..266d684 100644 --- a/src/db_manager.py +++ b/src/db_manager.py @@ -85,6 +85,11 @@ def init_db(): def _apply_additive_migrations(): + """Deliberately additive-only (no down-migrations, no version table). + + Sufficient for solo-dev column additions. Alembic would be the upgrade path for + multi-contributor development or type-altering/restructuring migrations. + """ from sqlalchemy import inspect, text try: