From 0fc486fed2802e4a46b39c0c9bef030c62c627c7 Mon Sep 17 00:00:00 2001 From: Ishaan Date: Thu, 9 Jul 2026 13:30:09 -0400 Subject: [PATCH 1/6] add cli quickstart test --- tests/test_quickstart.py | 58 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/tests/test_quickstart.py b/tests/test_quickstart.py index 0b8ff4f..1a15bbe 100644 --- a/tests/test_quickstart.py +++ b/tests/test_quickstart.py @@ -1,5 +1,7 @@ """Tests for the quickstart demo endpoint — the zero-friction onboarding wizard reached from the dashboard.""" +import pytest +from unittest.mock import patch import optuna from src.db_manager import DATABASE_URL @@ -7,6 +9,13 @@ DEMO_STUDY = "demo_segmentation_study" +@pytest.fixture(autouse=True) +def mock_run_training_worker(): + """Mock out the simulation worker to prevent spawning a background thread that pollutes/locks the DB.""" + with patch("simulators.training_worker.run_training_worker") as mock: + yield mock + + def test_quickstart_demo_creates_study(client): """First call to /api/quickstart_demo registers the study and returns success.""" resp = client.post("/api/quickstart_demo") @@ -37,3 +46,52 @@ def test_quickstart_demo_study_visible_in_config(client): assert resp.status_code == 200, resp.text data = resp.json() assert "metric_loss_label" in data or "metric_score_label" in data + + +def test_cli_quickstart_wizard_creates_files(monkeypatch, tmp_path): + """The interactive CLI quickstart wizard generates manifest and worker, and registers the study.""" + import os + import optuna + import yaml + from unittest.mock import patch + from hpo_cli import cmd_quickstart + from src.db_manager import DATABASE_URL + + # Change working directory to a temporary path so we don't pollute the workspace + monkeypatch.chdir(tmp_path) + + # Mock inputs: + # 1. Study name base -> "cli_quickstart_test" + # 2. Hyperparameter name -> "my_param" + # 3. Min bound -> "-5.0" + # 4. Max bound -> "5.0" + # 5. Direction -> "maximize" + inputs = iter(["cli_quickstart_test", "my_param", "-5.0", "5.0", "maximize"]) + monkeypatch.setattr("builtins.input", lambda prompt="": next(inputs)) + + # Catch the sys.exit(0) call to prevent pytest from exiting + with patch("sys.exit") as mock_exit: + cmd_quickstart(args=None) + + # Verify sys.exit was called with 0 + mock_exit.assert_called_once_with(0) + + # Verify files were generated in the temp directory + yaml_path = os.path.join(tmp_path, "quickstart.hpo.yaml") + worker_path = os.path.join(tmp_path, "quickstart_worker.py") + assert os.path.exists(yaml_path) + assert os.path.exists(worker_path) + + # Verify YAML content structure + with open(yaml_path) as f: + manifest_data = yaml.safe_load(f) + assert manifest_data["study_name"] == "cli_quickstart_test" + assert manifest_data["metrics"]["primary_score"] == "score" + assert manifest_data["params"][0]["name"] == "my_param" + assert manifest_data["params"][0]["min"] == -5.0 + assert manifest_data["params"][0]["max"] == 5.0 + + # Verify database study was registered + study = optuna.load_study(study_name="cli_quickstart_test", storage=DATABASE_URL) + assert study is not None + From 23167aa5cd819b3e07f7896278ac9f0a58b521a4 Mon Sep 17 00:00:00 2001 From: Ishaan Date: Thu, 9 Jul 2026 13:30:53 -0400 Subject: [PATCH 2/6] remove automatic login from URL query params --- docs/INTEGRATION.md | 6 ++++-- src/tunneling.py | 10 ++++++---- web/js/auth.js | 25 ------------------------- 3 files changed, 10 insertions(+), 31 deletions(-) diff --git a/docs/INTEGRATION.md b/docs/INTEGRATION.md index 0ef976f..a6e1560 100644 --- a/docs/INTEGRATION.md +++ b/docs/INTEGRATION.md @@ -37,9 +37,11 @@ Save that token and use the **same** value in three places: | Where | How | |-------|-----| | Colab / worker | `os.environ["HPO_SECRET_TOKEN"] = "…"` — `TrialSession` sends `X-HPO-Token` | -| Dashboard | First visit to the tunnel URL prompts once; stored in a session cookie | +| Dashboard | Open the tunnel URL; enter the token when prompted (stored in a session cookie) | | CLI / MCP | `export HPO_SECRET_TOKEN=…` when tools hit the tunneled broker | +Auth uses a shared secret for personal tunnel setups. Loopback (`127.0.0.1`) needs no token. + Worker downloads also require the token header when auth is on. The dashboard **Worker Setup** tab generates copy-paste snippets. @@ -227,7 +229,7 @@ Safe to run while the broker is running. | Variable | Default | Description | |---|---|---|---| -| `HPO_DATABASE_URL` | `sqlite:///hpo_studies.db` | SQLite connection string. | +| `HPO_DATABASE_URL` | `sqlite:///.data/hpo_studies.db` | SQLite connection string. | | `HPO_BROKER_URL` | `http://localhost:8000` | URL the worker uses to reach the broker. | | `HPO_STUDY_NAME` | *(none)* | Default study name when not passed explicitly. | | `HPO_SECRET_TOKEN` | *(none)* | Bearer token required when `--tunnel` auth is enabled. | diff --git a/src/tunneling.py b/src/tunneling.py index e83566c..8c0fceb 100644 --- a/src/tunneling.py +++ b/src/tunneling.py @@ -66,7 +66,8 @@ def _persist_tunnel_url(url: str, secret_token: Optional[str], label: str = "") print(f"\n{'='*50}") print(f"\U0001f525 Remote broker URL established{label_text}: {url}") if secret_token: - print(f" Auto-login Link: {url}/?token={secret_token}") + print(f" Dashboard: {url}") + print(" Enter the access token when the dashboard prompts.") print(f"{'='*50}\n") except Exception as db_err: print(f"Error saving remote broker URL: {db_err}") @@ -152,18 +153,19 @@ def ensure_secret_token(args_host: str, tunnel_requested: bool) -> Optional[str] def print_security_banner(secret_token: Optional[str], host: str, port: int) -> None: - """Print the token and login link banner on startup.""" + """Print the token and dashboard URL banner on startup.""" if not secret_token: return is_loopback = host in ("127.0.0.1", "localhost", "::1", "0.0.0.0", "::") host_display = "localhost" if is_loopback else host port_suffix = f":{port}" if port != 80 else "" - local_login_url = f"http://{host_display}{port_suffix}/?token={secret_token}" + dashboard_url = f"http://{host_display}{port_suffix}/" print("\n" + "=" * 80) print("\U0001f511 PATHFINDER DASHBOARD SECURITY ACTIVE") print(f" Access Token: {secret_token}") - print(f" Auto-login Link: {local_login_url}") + print(f" Dashboard: {dashboard_url}") + print(" Enter the access token when the dashboard prompts.") if not is_loopback: print(f"\U0001f512 Secure Private VPN/Tailscale Network Mode enabled. Binding to {host}:{port}") print("=" * 80 + "\n") diff --git a/web/js/auth.js b/web/js/auth.js index 56bc19e..79dbe28 100644 --- a/web/js/auth.js +++ b/web/js/auth.js @@ -63,29 +63,4 @@ } return res; }; - - // Auto-login from query parameter '?token=...' - const urlParams = new URLSearchParams(window.location.search); - const urlToken = urlParams.get('token'); - if (urlToken) { - // Strip the token from the URL search parameters immediately to prevent it from leaking in history/referrers - const cleanUrl = new URL(window.location.href); - cleanUrl.searchParams.delete('token'); - window.history.replaceState({}, document.title, cleanUrl.toString()); - - originalFetch("/api/login", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ token: urlToken.trim() }), - }).then(res => { - if (res.ok) { - // Reload the page to retry all initial API calls with the session cookie set - window.location.reload(); - } else { - console.error("Auto-login token was invalid."); - } - }).catch(err => { - console.error("Auto-login failed:", err); - }); - } })(); From 449e79bafdde898f6fb9c7d4fbedc2cf7b8c78cf Mon Sep 17 00:00:00 2001 From: Ishaan Date: Thu, 9 Jul 2026 13:31:32 -0400 Subject: [PATCH 3/6] update terminology --- src/leases.py | 2 +- src/reporting.py | 2 +- src/routers/dashboard.py | 2 +- src/search_space.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/leases.py b/src/leases.py index 7f6072b..27cb680 100644 --- a/src/leases.py +++ b/src/leases.py @@ -71,7 +71,7 @@ def _try_claim_lease(session, study_name: str, trial_id: int, worker_id: str, already held by this worker. If no row matched (no lease yet), INSERT and win on the trial_id primary key; a losing racer hits IntegrityError and returns False. This is the sole mechanism preventing two workers from being handed the same trial. Safe under - concurrency: SQLite (WAL + busy_timeout) serializes writers and Postgres locks the row. + concurrency: SQLite (WAL + busy_timeout) serializes writers. """ now = datetime.now(timezone.utc).replace(tzinfo=None) new_expiry = now + timedelta(seconds=ttl_seconds) diff --git a/src/reporting.py b/src/reporting.py index 06fcb74..aa8fe77 100644 --- a/src/reporting.py +++ b/src/reporting.py @@ -451,7 +451,7 @@ def handle_api_complete_trial(req: CompleteTrialRequest): write_ide_status_file(req.study_name, health_tier, health_reason, study) except Exception as err: - print(f"Error updating coordinator health status: {err}") + print(f"Error updating study health status: {err}") # Fetch completed scores for sparkline completed_scores = [] diff --git a/src/routers/dashboard.py b/src/routers/dashboard.py index 35bccce..0babb68 100644 --- a/src/routers/dashboard.py +++ b/src/routers/dashboard.py @@ -279,7 +279,7 @@ def api_fanova(study_name: str): @router.get("/study_packet") def api_study_packet(study_name: str): - """Read-only context for the IDE coordinator: Pareto, fANOVA, eval insights, drift reasons.""" + """Read-only context for the IDE agent: Pareto, fANOVA, eval insights, drift reasons.""" try: return build_study_packet(study_name) except Exception as e: diff --git a/src/search_space.py b/src/search_space.py index a2cb9e2..cf563c7 100644 --- a/src/search_space.py +++ b/src/search_space.py @@ -214,7 +214,7 @@ def suggest_params_from_space(study, trial, space: Dict[str, Any]) -> Dict[str, def _apply_search_space_patch(patch: Dict[str, Any], space: Dict[str, Any], study_name: str) -> str: - """Validate + persist active-bound narrowing from a coordinator review.""" + """Validate + persist active-bound narrowing from a search-space patch.""" for param, new_val in patch.items(): if param not in space: return f"Unknown parameter '{param}'." From 1787cc35ce8d2fc32fd22c752d281ec436711dde Mon Sep 17 00:00:00 2001 From: Ishaan Date: Thu, 9 Jul 2026 13:34:22 -0400 Subject: [PATCH 4/6] add demo study template --- README.md | 26 ++++++++++++++++++++++--- simulators/training_worker.py | 2 +- templates/demo_study.yaml | 36 +++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 templates/demo_study.yaml diff --git a/README.md b/README.md index b87a975..bdcb7ba 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -# Pathfinder   [![Build Status](https://github.com/Ishaan1402/pathfinder/actions/workflows/integration.yml/badge.svg)](https://github.com/Ishaan1402/pathfinder/actions) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg?style=flat-square)](LICENSE) +# Pathfinder   [![Build Status](https://github.com/Ishaan1402/pathfinder/actions/workflows/ci.yml/badge.svg)](https://github.com/Ishaan1402/pathfinder/actions) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg?style=flat-square)](LICENSE) --- -Your coding agents architect training pipelines, but the optimization loop still runs completely out of their sight. Pathfinder brings that loop back in view. +Pathfinder is a HPO broker for 1–2 objective deep learning runs, not an LLM that picks hyperparameters. The broker suggests via Optuna TPE; your IDE agent then inspects results only when you ask @@ -38,13 +38,23 @@ An MCP server gives your IDE agent read-only visibility into trial history, heal ```bash python3 -m venv .venv source .venv/bin/activate -pip install -r requirements.txt +pip install -r requirements.txt # or: pip install -r requirements-dev.lock for pinned CI deps python broker.py --daemon # Dashboard: http://127.0.0.1:8000 ``` Using the dashboard is optional; CLI and your IDE agent can do everything. +### Try it without your own training script + +```bash +python hpo_cli.py init templates/demo_study.yaml +export HPO_BROKER_URL=http://localhost:8000 +export HPO_STUDY_NAME=demo_study +python simulators/training_worker.py --study_name demo_study --max_trials 3 +# Open http://127.0.0.1:8000 to watch trials +``` + ### Step 2: Connect Your Workers **Local worker (same machine)** @@ -256,6 +266,16 @@ pytest tests/ -q +## Ops (personal use) + +- Study state lives in SQLite under `.data/` (override with `HPO_DATABASE_URL`). +- Backup anytime: `python hpo_cli.py backup --output backup.db` +- Auth: loopback needs no token. With `--tunnel`, set `HPO_SECRET_TOKEN` and pass the same value to workers (`X-HPO-Token`). This is a shared secret for a single operator. + +--- + + + ## Dev Notes - MCP is implemented instead of giving the agent direct CLI execution access because structured APIs (schemas for tools and URIs for data) are much more reliable for AI tools than parsing raw command-line output. diff --git a/simulators/training_worker.py b/simulators/training_worker.py index 3d3e0a4..96e9240 100644 --- a/simulators/training_worker.py +++ b/simulators/training_worker.py @@ -215,7 +215,7 @@ def run_training_worker( if __name__ == "__main__": import argparse parser = argparse.ArgumentParser(description="Simulated Pathfinder Training Worker") - parser.add_argument("--study_name", default="unet_crack_segmentation", help="Study name") + parser.add_argument("--study_name", default="demo_study", help="Study name") parser.add_argument("--max_trials", type=int, default=5, help="Number of trials to run") parser.add_argument("--epochs_per_trial", type=int, default=10, help="Epochs per trial") parser.add_argument("--broker_url", default=None, help="Broker URL") diff --git a/templates/demo_study.yaml b/templates/demo_study.yaml new file mode 100644 index 0000000..19ad161 --- /dev/null +++ b/templates/demo_study.yaml @@ -0,0 +1,36 @@ +# Demo study for simulators/training_worker.py — register with: +# python hpo_cli.py init templates/demo_study.yaml + +study_name: demo_study + +metrics: + primary_score: score + objectives: + - name: score + direction: maximize + label: "Score" + - name: loss + direction: minimize + label: "Loss" + +params: + - name: learning_rate + type: float_log + min: 1.0e-5 + max: 1.0e-2 + - name: batch_size + type: categorical + options: [2, 4, 8, 16] + - name: resolution + type: categorical + options: [256, 512, 1024] + - name: model_capacity + type: categorical + options: ["narrow", "wide"] + - name: loss_weight_ratio + type: float + min: 0.0 + max: 1.0 + +worker: + entrypoint: python simulators/training_worker.py From 95854a6f682535f5357ec431b69971d177880784 Mon Sep 17 00:00:00 2001 From: Ishaan Date: Thu, 9 Jul 2026 13:34:40 -0400 Subject: [PATCH 5/6] update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bdcb7ba..4e719da 100644 --- a/README.md +++ b/README.md @@ -266,7 +266,7 @@ pytest tests/ -q -## Ops (personal use) +## Notes for running locally - Study state lives in SQLite under `.data/` (override with `HPO_DATABASE_URL`). - Backup anytime: `python hpo_cli.py backup --output backup.db` From ba1b57a9b0a7209ea1485902d676af942b3ae794 Mon Sep 17 00:00:00 2001 From: Ishaan Date: Thu, 9 Jul 2026 13:34:54 -0400 Subject: [PATCH 6/6] add lockfile and update ci --- .github/workflows/ci.yml | 7 +- .github/workflows/integration.yml | 23 ----- pyproject.toml | 4 +- requirements-dev.lock | 166 ++++++++++++++++++++++++++++++ requirements-dev.txt | 5 + requirements.txt | 8 +- 6 files changed, 183 insertions(+), 30 deletions(-) delete mode 100644 .github/workflows/integration.yml create mode 100644 requirements-dev.lock diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc4eea8..6c51646 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,10 +24,13 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements-dev.txt + pip install -r requirements-dev.lock + + - name: Smoke import + run: python -c "import broker; import hpo_cli" - name: Lint - run: ruff check src/ hpo_cli.py hpo_mcp_server.py broker.py --select F + run: ruff check src/ hpo_cli.py hpo_mcp_server.py broker.py --select F,E9 - name: Run tests (SQLite) run: pytest tests/ -q diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml deleted file mode 100644 index bcdc2a6..0000000 --- a/.github/workflows/integration.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Integration Test - -on: - push: - branches: [main, master] - pull_request: - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up Python 3.11 - uses: actions/setup-python@v5 - with: - python-version: "3.11" - cache: pip - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements-dev.txt - - name: Run tests - run: pytest tests/ -q diff --git a/pyproject.toml b/pyproject.toml index 5e013df..0570954 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "pathfinder" version = "1.0.0" -description = "Decoupled hyperparameter optimization (HPO) framework with periodically scheduled AI coordinator reviews" +description = "Local HPO broker (Optuna TPE + FastAPI) with MCP tools for agent onboarding and study inspection" readme = "README.md" requires-python = ">=3.10" license = {text = "MIT"} @@ -23,7 +23,7 @@ dependencies = [ "sqlalchemy>=2.0,<3", "pydantic>=2.0,<3", - "numpy>=1.20,<3", + "numpy>=1.20,<2.3", "fastapi>=0.110,<1", "uvicorn>=0.23,<1", "requests>=2.28,<3", diff --git a/requirements-dev.lock b/requirements-dev.lock new file mode 100644 index 0000000..8180c77 --- /dev/null +++ b/requirements-dev.lock @@ -0,0 +1,166 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=requirements-dev.lock --strip-extras requirements-dev.txt +# +alembic==1.18.5 + # via optuna +annotated-doc==0.0.4 + # via fastapi +annotated-types==0.7.0 + # via pydantic +anyio==4.14.1 + # via + # httpx + # mcp + # sse-starlette + # starlette +attrs==26.1.0 + # via + # jsonschema + # referencing +certifi==2026.6.17 + # via + # httpcore + # httpx + # requests +cffi==2.1.0 + # via cryptography +charset-normalizer==3.4.9 + # via requests +click==8.4.2 + # via uvicorn +colorlog==6.10.1 + # via optuna +cryptography==49.0.0 + # via pyjwt +exceptiongroup==1.3.1 + # via + # anyio + # pytest +fastapi==0.139.0 + # via -r requirements.txt +h11==0.16.0 + # via + # httpcore + # uvicorn +httpcore==1.0.9 + # via httpx +httpx==0.28.1 + # via mcp +httpx-sse==0.4.3 + # via mcp +idna==3.18 + # via + # anyio + # httpx + # requests +iniconfig==2.3.0 + # via pytest +jsonschema==4.26.0 + # via mcp +jsonschema-specifications==2025.9.1 + # via jsonschema +mako==1.3.12 + # via alembic +markupsafe==3.0.3 + # via mako +mcp==1.28.1 + # via -r requirements.txt +numpy==2.2.6 + # via + # -r requirements.txt + # optuna +optuna==4.9.0 + # via -r requirements.txt +packaging==26.2 + # via + # optuna + # pytest +pluggy==1.6.0 + # via pytest +pycparser==3.0 + # via cffi +pydantic==2.13.4 + # via + # -r requirements.txt + # fastapi + # mcp + # pydantic-settings +pydantic-core==2.46.4 + # via pydantic +pydantic-settings==2.14.2 + # via mcp +pygments==2.20.0 + # via pytest +pyjwt==2.13.0 + # via mcp +pytest==9.1.1 + # via -r requirements-dev.txt +python-dotenv==1.2.2 + # via pydantic-settings +python-multipart==0.0.32 + # via mcp +pyyaml==6.0.3 + # via + # -r requirements.txt + # optuna +referencing==0.37.0 + # via + # jsonschema + # jsonschema-specifications +requests==2.34.2 + # via -r requirements.txt +rpds-py==0.30.0 + # via + # jsonschema + # referencing +ruff==0.15.20 + # via -r requirements-dev.txt +sqlalchemy==2.0.51 + # via + # -r requirements.txt + # alembic + # optuna +sse-starlette==3.4.5 + # via mcp +starlette==1.3.1 + # via + # fastapi + # mcp + # sse-starlette +tomli==2.4.1 + # via + # alembic + # pytest +tqdm==4.68.4 + # via optuna +typing-extensions==4.16.0 + # via + # alembic + # anyio + # cryptography + # exceptiongroup + # fastapi + # mcp + # pydantic + # pydantic-core + # pyjwt + # referencing + # sqlalchemy + # starlette + # typing-inspection + # uvicorn +typing-inspection==0.4.2 + # via + # fastapi + # mcp + # pydantic + # pydantic-settings +urllib3==2.7.0 + # via requests +uvicorn==0.51.0 + # via + # -r requirements.txt + # mcp diff --git a/requirements-dev.txt b/requirements-dev.txt index 3b41a5b..356fcf7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,3 +2,8 @@ -r requirements.txt pytest>=8.0 ruff>=0.4 + +# CI installs the lock (pinned). Users can use this file or requirements.txt with ranges. +# Regenerate on the oldest supported Python so the matrix stays installable: +# python3.10 -m pip install pip-tools +# python3.10 -m piptools compile requirements-dev.txt -o requirements-dev.lock --strip-extras diff --git a/requirements.txt b/requirements.txt index 1e6aa10..7ab60f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,13 @@ -# Runtime dependencies. Bounds cap the next major version to avoid surprise breaking changes -# while still allowing security/patch updates. Dev/test extras live in requirements-dev.txt. +# Editable bounds. CI pins via requirements-dev.lock (compile on the oldest supported Python). +# python3.10 -m pip install pip-tools +# python3.10 -m piptools compile requirements-dev.txt -o requirements-dev.lock --strip-extras mcp>=1.1,<2 optuna>=3.6,<5 sqlalchemy>=2.0,<3 pydantic>=2.0,<3 -numpy>=1.20,<3 +# NumPy 2.3+ dropped 3.10; keep the ceiling so the 3.10 CI matrix stays installable. +numpy>=1.20,<2.3 fastapi>=0.110,<1 uvicorn>=0.23,<1 requests>=2.28,<3