Revision 7 9

.github/workflows/ci.yml

@@ -24,10 +24,13 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements-dev.txt
+          pip install -r requirements-dev.lock
+
+      - name: Smoke import
+        run: python -c "import broker; import hpo_cli"
 
       - name: Lint
-        run: ruff check src/ hpo_cli.py hpo_mcp_server.py broker.py --select F
+        run: ruff check src/ hpo_cli.py hpo_mcp_server.py broker.py --select F,E9
 
       - name: Run tests (SQLite)
         run: pytest tests/ -q

README.md

@@ -1,8 +1,8 @@
-# Pathfinder   [![Build Status](https://github.com/Ishaan1402/pathfinder/actions/workflows/integration.yml/badge.svg)](https://github.com/Ishaan1402/pathfinder/actions) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg?style=flat-square)](LICENSE)
+# Pathfinder   [![Build Status](https://github.com/Ishaan1402/pathfinder/actions/workflows/ci.yml/badge.svg)](https://github.com/Ishaan1402/pathfinder/actions) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg?style=flat-square)](LICENSE)
 
 ---
 
-Your coding agents architect training pipelines, but the optimization loop still runs completely out of their sight. Pathfinder brings that loop back in view.
+Pathfinder is a HPO broker for 1–2 objective deep learning runs, not an LLM that picks hyperparameters. The broker suggests via Optuna TPE; your IDE agent then inspects results only when you ask
 
 
 <table border="0">
@@ -38,13 +38,23 @@ An MCP server gives your IDE agent read-only visibility into trial history, heal
 ```bash
 python3 -m venv .venv
 source .venv/bin/activate
-pip install -r requirements.txt
+pip install -r requirements.txt   # or: pip install -r requirements-dev.lock for pinned CI deps
 python broker.py --daemon
 # Dashboard: http://127.0.0.1:8000
 ```
 
 Using the dashboard is optional; CLI and your IDE agent can do everything.
 
+### Try it without your own training script
+
+```bash
+python hpo_cli.py init templates/demo_study.yaml
+export HPO_BROKER_URL=http://localhost:8000
+export HPO_STUDY_NAME=demo_study
+python simulators/training_worker.py --study_name demo_study --max_trials 3
+# Open http://127.0.0.1:8000 to watch trials
+```
+
 ### Step 2: Connect Your Workers
 
 **Local worker (same machine)**
@@ -256,6 +266,16 @@ pytest tests/ -q
 
 
 
+## Notes for running locally
+
+- Study state lives in SQLite under `.data/` (override with `HPO_DATABASE_URL`).
+- Backup anytime: `python hpo_cli.py backup --output backup.db`
+- Auth: loopback needs no token. With `--tunnel`, set `HPO_SECRET_TOKEN` and pass the same value to workers (`X-HPO-Token`). This is a shared secret for a single operator.
+
+---
+
+
+
 ## Dev Notes
 
 - MCP is implemented instead of giving the agent direct CLI execution access because structured APIs (schemas for tools and URIs for data) are much more reliable for AI tools than parsing raw command-line output.

docs/INTEGRATION.md

@@ -37,9 +37,11 @@ Save that token and use the **same** value in three places:
 | Where | How |
 |-------|-----|
 | Colab / worker | `os.environ["HPO_SECRET_TOKEN"] = "…"` — `TrialSession` sends `X-HPO-Token` |
-| Dashboard | First visit to the tunnel URL prompts once; stored in a session cookie |
+| Dashboard | Open the tunnel URL; enter the token when prompted (stored in a session cookie) |
 | CLI / MCP | `export HPO_SECRET_TOKEN=…` when tools hit the tunneled broker |
 
+Auth uses a shared secret for personal tunnel setups. Loopback (`127.0.0.1`) needs no token.
+
 Worker downloads also require the token header when auth is on. The dashboard **Worker Setup** tab
 generates copy-paste snippets.
 
@@ -227,7 +229,7 @@ Safe to run while the broker is running.
 
 | Variable | Default | Description |
 |---|---|---|---|
-| `HPO_DATABASE_URL` | `sqlite:///hpo_studies.db` | SQLite connection string. |
+| `HPO_DATABASE_URL` | `sqlite:///.data/hpo_studies.db` | SQLite connection string. |
 | `HPO_BROKER_URL` | `http://localhost:8000` | URL the worker uses to reach the broker. |
 | `HPO_STUDY_NAME` | *(none)* | Default study name when not passed explicitly. |
 | `HPO_SECRET_TOKEN` | *(none)* | Bearer token required when `--tunnel` auth is enabled. |

pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "pathfinder"
 version = "1.0.0"
-description = "Decoupled hyperparameter optimization (HPO) framework with periodically scheduled AI coordinator reviews"
+description = "Local HPO broker (Optuna TPE + FastAPI) with MCP tools for agent onboarding and study inspection"
 readme = "README.md"
 requires-python = ">=3.10"
 license = {text = "MIT"}
@@ -23,7 +23,7 @@ dependencies = [
     "sqlalchemy>=2.0,<3",
 
     "pydantic>=2.0,<3",
-    "numpy>=1.20,<3",
+    "numpy>=1.20,<2.3",
     "fastapi>=0.110,<1",
     "uvicorn>=0.23,<1",
     "requests>=2.28,<3",

requirements-dev.lock

@@ -0,0 +1,166 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile --output-file=requirements-dev.lock --strip-extras requirements-dev.txt
+#
+alembic==1.18.5
+    # via optuna
+annotated-doc==0.0.4
+    # via fastapi
+annotated-types==0.7.0
+    # via pydantic
+anyio==4.14.1
+    # via
+    #   httpx
+    #   mcp
+    #   sse-starlette
+    #   starlette
+attrs==26.1.0
+    # via
+    #   jsonschema
+    #   referencing
+certifi==2026.6.17
+    # via
+    #   httpcore
+    #   httpx
+    #   requests
+cffi==2.1.0
+    # via cryptography
+charset-normalizer==3.4.9
+    # via requests
+click==8.4.2
+    # via uvicorn
+colorlog==6.10.1
+    # via optuna
+cryptography==49.0.0
+    # via pyjwt
+exceptiongroup==1.3.1
+    # via
+    #   anyio
+    #   pytest
+fastapi==0.139.0
+    # via -r requirements.txt
+h11==0.16.0
+    # via
+    #   httpcore
+    #   uvicorn
+httpcore==1.0.9
+    # via httpx
+httpx==0.28.1
+    # via mcp
+httpx-sse==0.4.3
+    # via mcp
+idna==3.18
+    # via
+    #   anyio
+    #   httpx
+    #   requests
+iniconfig==2.3.0
+    # via pytest
+jsonschema==4.26.0
+    # via mcp
+jsonschema-specifications==2025.9.1
+    # via jsonschema
+mako==1.3.12
+    # via alembic
+markupsafe==3.0.3
+    # via mako
+mcp==1.28.1
+    # via -r requirements.txt
+numpy==2.2.6
+    # via
+    #   -r requirements.txt
+    #   optuna
+optuna==4.9.0
+    # via -r requirements.txt
+packaging==26.2
+    # via
+    #   optuna
+    #   pytest
+pluggy==1.6.0
+    # via pytest
+pycparser==3.0
+    # via cffi
+pydantic==2.13.4
+    # via
+    #   -r requirements.txt
+    #   fastapi
+    #   mcp
+    #   pydantic-settings
+pydantic-core==2.46.4
+    # via pydantic
+pydantic-settings==2.14.2
+    # via mcp
+pygments==2.20.0
+    # via pytest
+pyjwt==2.13.0
+    # via mcp
+pytest==9.1.1
+    # via -r requirements-dev.txt
+python-dotenv==1.2.2
+    # via pydantic-settings
+python-multipart==0.0.32
+    # via mcp
+pyyaml==6.0.3
+    # via
+    #   -r requirements.txt
+    #   optuna
+referencing==0.37.0
+    # via
+    #   jsonschema
+    #   jsonschema-specifications
+requests==2.34.2
+    # via -r requirements.txt
+rpds-py==0.30.0
+    # via
+    #   jsonschema
+    #   referencing
+ruff==0.15.20
+    # via -r requirements-dev.txt
+sqlalchemy==2.0.51
+    # via
+    #   -r requirements.txt
+    #   alembic
+    #   optuna
+sse-starlette==3.4.5
+    # via mcp
+starlette==1.3.1
+    # via
+    #   fastapi
+    #   mcp
+    #   sse-starlette
+tomli==2.4.1
+    # via
+    #   alembic
+    #   pytest
+tqdm==4.68.4
+    # via optuna
+typing-extensions==4.16.0
+    # via
+    #   alembic
+    #   anyio
+    #   cryptography
+    #   exceptiongroup
+    #   fastapi
+    #   mcp
+    #   pydantic
+    #   pydantic-core
+    #   pyjwt
+    #   referencing
+    #   sqlalchemy
+    #   starlette
+    #   typing-inspection
+    #   uvicorn
+typing-inspection==0.4.2
+    # via
+    #   fastapi
+    #   mcp
+    #   pydantic
+    #   pydantic-settings
+urllib3==2.7.0
+    # via requests
+uvicorn==0.51.0
+    # via
+    #   -r requirements.txt
+    #   mcp

requirements-dev.txt

@@ -2,3 +2,8 @@
 -r requirements.txt
 pytest>=8.0
 ruff>=0.4
+
+# CI installs the lock (pinned). Users can use this file or requirements.txt with ranges.
+# Regenerate on the oldest supported Python so the matrix stays installable:
+#   python3.10 -m pip install pip-tools
+#   python3.10 -m piptools compile requirements-dev.txt -o requirements-dev.lock --strip-extras

requirements.txt

@@ -1,11 +1,13 @@
-# Runtime dependencies. Bounds cap the next major version to avoid surprise breaking changes
-# while still allowing security/patch updates. Dev/test extras live in requirements-dev.txt.
+# Editable bounds. CI pins via requirements-dev.lock (compile on the oldest supported Python).
+#   python3.10 -m pip install pip-tools
+#   python3.10 -m piptools compile requirements-dev.txt -o requirements-dev.lock --strip-extras
 mcp>=1.1,<2
 optuna>=3.6,<5
 sqlalchemy>=2.0,<3
 
 pydantic>=2.0,<3
-numpy>=1.20,<3
+# NumPy 2.3+ dropped 3.10; keep the ceiling so the 3.10 CI matrix stays installable.
+numpy>=1.20,<2.3
 fastapi>=0.110,<1
 uvicorn>=0.23,<1
 requests>=2.28,<3

simulators/training_worker.py

@@ -215,7 +215,7 @@ def run_training_worker(
 if __name__ == "__main__":
     import argparse
     parser = argparse.ArgumentParser(description="Simulated Pathfinder Training Worker")
-    parser.add_argument("--study_name", default="unet_crack_segmentation", help="Study name")
+    parser.add_argument("--study_name", default="demo_study", help="Study name")
     parser.add_argument("--max_trials", type=int, default=5, help="Number of trials to run")
     parser.add_argument("--epochs_per_trial", type=int, default=10, help="Epochs per trial")
     parser.add_argument("--broker_url", default=None, help="Broker URL")

src/leases.py

@@ -71,7 +71,7 @@ def _try_claim_lease(session, study_name: str, trial_id: int, worker_id: str,
     already held by this worker. If no row matched (no lease yet), INSERT and win on the
     trial_id primary key; a losing racer hits IntegrityError and returns False. This is the
     sole mechanism preventing two workers from being handed the same trial. Safe under
-    concurrency: SQLite (WAL + busy_timeout) serializes writers and Postgres locks the row.
+    concurrency: SQLite (WAL + busy_timeout) serializes writers.
     """
     now = datetime.now(timezone.utc).replace(tzinfo=None)
     new_expiry = now + timedelta(seconds=ttl_seconds)

src/reporting.py

@@ -451,7 +451,7 @@ def handle_api_complete_trial(req: CompleteTrialRequest):
 
             write_ide_status_file(req.study_name, health_tier, health_reason, study)
         except Exception as err:
-            print(f"Error updating coordinator health status: {err}")
+            print(f"Error updating study health status: {err}")
 
         # Fetch completed scores for sparkline
         completed_scores = []

src/routers/dashboard.py

@@ -279,7 +279,7 @@ def api_fanova(study_name: str):
 
 @router.get("/study_packet")
 def api_study_packet(study_name: str):
-    """Read-only context for the IDE coordinator: Pareto, fANOVA, eval insights, drift reasons."""
+    """Read-only context for the IDE agent: Pareto, fANOVA, eval insights, drift reasons."""
     try:
         return build_study_packet(study_name)
     except Exception as e:

src/search_space.py

@@ -214,7 +214,7 @@ def suggest_params_from_space(study, trial, space: Dict[str, Any]) -> Dict[str,
 
 
 def _apply_search_space_patch(patch: Dict[str, Any], space: Dict[str, Any], study_name: str) -> str:
-    """Validate + persist active-bound narrowing from a coordinator review."""
+    """Validate + persist active-bound narrowing from a search-space patch."""
     for param, new_val in patch.items():
         if param not in space:
             return f"Unknown parameter '{param}'."