diff --git a/.cursor/rules/hpo-coordinator.mdc b/.cursor/rules/hpo-coordinator.mdc deleted file mode 100644 index 54bfe8a..0000000 --- a/.cursor/rules/hpo-coordinator.mdc +++ /dev/null @@ -1,42 +0,0 @@ ---- -description: HPO coordinator review procedure (episodic, read-heavy, write-once per trial window) -globs: broker.py,hpo_mcp_server.py,hpo_coordinator.py,colab_worker.py,hpo_config.json,active_search_space.json,index.html,.hpo_status.json -alwaysApply: false ---- - -# Pathfinder Coordinator Review - -Optuna (TPE) in `broker.py` is the hot path. The Colab worker must never wait on a language model. You are an **episodic coordinator**: you interpret results, flag bad policy, and apply bounded changes through the `@pathfinder` MCP server. Do not turn the per-trial suggest into an LLM call. - -## Active Divergence Warning Trigger - -If the workspace status file `.hpo_status.json` has `health_tier` set to `"watch"` or `"intervene"`: -1. Immediately notify the user in your response that the Pathfinder study is in a warning state (`Watch` or `Intervene`) and explain the `health_reason` cited in the status file. -2. Proactively offer to execute the **Pathfinder Coordinator Review** (the 7-step procedure below). Do not wait for the user to ask. -3. Recommend using the `/goal` slash command to resolve the stagnation or suggesting specific active bounds changes. - -## When to run a review - -Only on an explicit user request, or when one of these is true. Do **not** auto-run from a hook. - -- The `.hpo_status.json` indicates a warning state (`Watch` or `Intervene`), or -- The dashboard shows "Coordinator review suggested", or -- ~5 completed/pruned trials have elapsed since the last review, or -- The user is about to change the search space or eval protocol, or -- The user starts editing tuning code (the files this rule is scoped to). - -Run a review **at most once per trial window** unless `review_recommended` is set or the user passes `force`. - -## Seven-step procedure - -1. Call `get_study_data(study_name)` to retrieve the compacted review packet (which contains active search space, config, and study telemetry). -2. Read dynamic metric labels from the packet's `project_context`. -3. Rate search health 1–5. Cite trials by number. Prefer fixed-eval (deploy) Dice over train Dice. -4. Perform safety review of VRAM predictions and check coordinator accuracy. -5. Pick exactly one policy action: `no_change`, `update_search_space` (propose), or `enqueue_one_manual_trial`. -6. If proposing active search space changes, call `update_search_space(study_name, space_config, apply=False)`. If enqueuing a manual trial, pass the parameter dictionary as the `manual_trial` argument when calling `submit_agent_review`. -7. Call `submit_agent_review(study_name, summary, health_rating, policy_action, reasons=...)` to persist the audit trail. - -## Multi-IDE safety - -`submit_agent_review` is idempotent per trial window. If Cursor and Antigravity are both open, a duplicate review for the same number of finished trials returns the existing review (`duplicate: true`); do not retry with `force` to override it unless the user asks. Coordinator work is read-heavy and write-once per cycle. diff --git a/.cursor/rules/hpo-onboarding.mdc b/.cursor/rules/hpo-onboarding.mdc deleted file mode 100644 index d15c7ac..0000000 --- a/.cursor/rules/hpo-onboarding.mdc +++ /dev/null @@ -1,16 +0,0 @@ ---- -description: HPO cloner onboarding - scaffold a thin worker against the broker without forking colab_worker.py -globs: templates/**,hpo_client.py,docs/INTEGRATION.md,AGENTS.md -alwaysApply: false ---- - -# Pathfinder Onboarding - -When the user asks to integrate HPO / onboard a training script, or edits these template/client files, follow the onboarding section of [AGENTS.md](../../AGENTS.md). - -- Offer a one-line "Run Pathfinder onboarding?" - do NOT write files until the user confirms. -- Scaffold from `templates/` using `hpo_client.TrialSession` (suggest / report_epoch / complete). Do NOT fork the root `colab_worker.py`. -- Use the `@pathfinder` MCP tools: `validate_manifest`, `init_from_manifest`, `validate_integration`. -- Never block the GPU worker on an LLM; never auto-run the coordinator. - -Scope note: this rule is for onboarding only. Production tuning files (`broker.py`, `hpo_mcp_server.py`, `colab_worker.py`, configs, `index.html`) are governed by `hpo-coordinator.mdc`, not this rule. diff --git a/.cursor/skills/hpo-onboard/SKILL.md b/.cursor/skills/hpo-onboard/SKILL.md deleted file mode 100644 index 3b7626f..0000000 --- a/.cursor/skills/hpo-onboard/SKILL.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -name: hpo-onboard -description: Onboard a training pipeline to Pathfinder. Use when the user wants to integrate HPO, onboard a training pipeline, set up a fresh clone, or wire hyperparameter tuning into their training script. ---- - -# Pathfinder Onboarding - -Follow AGENTS.md onboarding section; use @pathfinder MCP tools; do not modify root colab_worker.py unless user owns bridge-crack project. - -## Steps - -1. Read [AGENTS.md](../../../AGENTS.md) "Onboarding procedure" and follow it. -2. Load the MCP resource `hpo://prompts/grill` for the canonical onboarding checklist. -3. Offer a one-line "Run Pathfinder onboarding?" and do not write files until the user confirms. -4. Scaffold using templates/manifest.template.yaml and worker_minimal.py. Do not write json space config files. Validate and register the study using the MCP tools `validate_manifest` and `init_from_manifest`. -5. Finish by running the MCP tool `validate_integration(study_name)` to verify. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f12bad..dc4eea8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,5 +26,8 @@ jobs: python -m pip install --upgrade pip pip install -r requirements-dev.txt + - name: Lint + run: ruff check src/ hpo_cli.py hpo_mcp_server.py broker.py --select F + - name: Run tests (SQLite) run: pytest tests/ -q diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 216df8b..bcdc2a6 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -1,32 +1,23 @@ name: Integration Test -on: [push, pull_request] +on: + push: + branches: [main, master] + pull_request: jobs: test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.11 - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - - name: Run Unit Tests - run: | - pip install pytest httpx - pytest tests/ -v - - name: Generate Quickstart Study - run: | - printf '\n\n\n\n\n' | python hpo_cli.py quickstart - - name: Start Broker - run: | - python broker.py --daemon & - - name: Wait for Broker - run: sleep 2 - - name: Run Dummy Worker - run: | - python quickstart_worker.py + - uses: actions/checkout@v4 + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + - name: Run tests + run: pytest tests/ -q diff --git a/.gitignore b/.gitignore index 3e55830..e0371f5 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,9 @@ pending_changes.json # Build artifacts *.egg-info/ +# Generated model cards +studies/ + +# Default database location +.data/ + diff --git a/AGENTS.md b/AGENTS.md index ecc8c49..e088e90 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,7 +21,7 @@ coordinator without an explicit user request.** ## MCP setup (one time, shared across IDEs) All three IDEs use the same server `pathfinder` (`python hpo_mcp_server.py`) and the -same database. See the README "Exposing Pathfinder to AI Agents" section for the exact config block. +same database. See the README "IDE Setup (Agent-Driven Onboarding & Inspection)" section for the exact config block. Capability does not differ between Cursor, Antigravity, and Claude Code. Use one IDE at a time for writes; reviews are idempotent per trial window, so a second client will not double-write. @@ -51,7 +51,7 @@ Trigger this when the user says "integrate HPO", "onboard my training script", " 7. **Document the GPU side.** Tell the user to set `HPO_BROKER_URL` and `HPO_STUDY_NAME` on the training machine. The study name must match the manifest. Enable `HPO_SPARKLINES=1` if they want a Unicode performance curve printed on trial completion. -The worker contract is exactly three calls; full reference in `docs/INTEGRATION.md`. Load the grill checklist from `hpo://prompts/grill` (not a separate integration-guide tool). +The worker contract is exactly three calls; full reference in `docs/INTEGRATION.md`. ### Statistical confidence (caveat, not a gate) @@ -59,34 +59,19 @@ The worker contract is exactly three calls; full reference in `docs/INTEGRATION. | Tier | Completed trials | Agent behavior | |------|------------------|----------------| -| `low` | < 10 | Treat fANOVA/Spearman as noisy; use `estimated_score_improvement=-1.0` when uncertain | +| `low` | < 10 | Treat fANOVA/Spearman as noisy; be cautious with interpretation | | `medium` | 10–19 | Signals stabilizing; stay cautious on large bound shifts | | `high` | ≥ 20 | Standard interpretation | Reviews are never hard-blocked at low confidence — the dashboard shows a banner only. -## Coordinator procedure (episodic review) +## Inspection flow -When the dashboard shows a health warning (Watch or Intervene) and the brand-mark double pings (white for Watch, red for Intervene), run the 7-step review. The review prompt can be loaded from the MCP resource `hpo://prompts/review`. +When the user asks about study progress, trial results, or health: -1. Call `get_study_data(study_name)` to retrieve the compacted statistical and telemetry packet (`statistical_confidence`, `coordinator_accuracy`, `past_reviews`). -2. Read dynamic metrics from `project_context` and evaluate fANOVA importances and Spearman correlations; heed `statistical_confidence` when low/medium. -3. Perform a safety review of VRAM predictions (`bounds_oom_risk`) and review the last 3 `past_reviews` — **ignore reviews where `quality_flagged` is true**. -4. **Coordinator accuracy self-regulation:** `coordinator_accuracy` tracks review forecasts vs. measured best-score deltas (not trial-suggest logs). If `insufficient_data` is true (`n_scored_reviews` < 3), do not self-regulate yet. If `mean_absolute_error` > 0.05 with `n_scored_reviews` ≥ 3, propose smaller bound shifts. -5. Propose active search space adjustments via `update_search_space(study_name, space_config, apply=False)`. -6. Submit with `submit_agent_review` — **required:** `estimated_score_improvement` (float) and `cited_best_trial` (int). Use `-1.0` when < 5 completed trials. Human approves via dashboard **Apply Proposal** or `update_search_space(apply=True)`. -7. Call `generate_model_card(study_name)` to write the model card and index it in the database. - -### Coordinator accuracy semantics (honest correlational accountability) - -At review time the system snapshots `baseline_best_score`. After the human applies a bounds patch and **≥ 5 new COMPLETE trials** finish, `backfill_review_outcomes` sets `actual_score_improvement = new_best - baseline`. MAE uses only `outcome_status=measured` reviews (excludes sentinel `-1.0`, `inconclusive` prune-storm windows, and `quality_flagged` rows). This does **not** prove causation — it measures whether the coordinator's numeric forecast matched the best-score delta N trials later. Flag bad reviews: `python hpo_cli.py flag-review --id N` or the dashboard **Flag** button. - -## Guardrails (always) - -- Never block the GPU worker on an LLM; the suggest path stays TPE. -- Never auto-invoke the coordinator from the broker, dashboard, or a hook - only on user request. -- Do not write/modify local JSON files for search space or configuration (e.g. `active_search_space.json`, `hpo_config.json`); state must reside in SQLite. -- Do not modify the root `colab_worker.py` unless the user owns the bridge-crack project. Cloners use `templates/`. +1. Call `get_study_data(study_name)` to retrieve trial telemetry, health tier, fANOVA importances, best trials, and OOM patterns. +2. Summarize: current best score, health status, stagnation indicators, and any OOM warnings. +3. If health is `watch` or `intervene`, explain the reason and offer to explore the data further. ## IDE triggers & status polling (.hpo_status.json) @@ -94,6 +79,13 @@ When a completed trial is reported or the background daemon polls health, the sy **Agent Action (Cursor, Antigravity, and Claude Code)**: * Eagerly check for the existence of `.hpo_status.json` in the project root. -* If `health_tier` is `"watch"` or `"intervene"`, immediately notify the user in your first response, explain the `health_reason`, and proactively offer to run the **Coordinator procedure (episodic review)** described above. +* If `health_tier` is `"watch"` or `"intervene"`, immediately notify the user in your first response, explain the `health_reason`, and proactively offer to inspect the study data. * Offer to use the `/goal` slash command to help coordinate and automate resolution of search space stagnation or failures. +## Guardrails (always) + +- Never block the GPU worker on an LLM; the suggest path stays TPE. +- Never auto-invoke the coordinator from the broker, dashboard, or a hook - only on user request. +- Do not write/modify local JSON files for search space or configuration (e.g. `active_search_space.json`, `hpo_config.json`); state must reside in SQLite. +- Do not modify the root `colab_worker.py` unless the user owns the bridge-crack project. Cloners use `templates/`. +- Never modify `archive/` files — they are historical reference only. diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index bb0a244..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,80 +0,0 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## Development & Test Commands - -- **Environment Setup**: - ```bash - python3 -m venv .venv - source .venv/bin/activate - pip install -r requirements.txt - ``` -- **Run FastAPI Broker Server (serves Custom Dashboard on http://127.0.0.1:8000)**: - ```bash - python3 broker.py --daemon # local only - export HPO_SECRET_TOKEN="$(python -c 'import secrets; print(secrets.token_urlsafe(32))')" - python3 broker.py --daemon --tunnel # Colab / remote GPU (auth required) - ``` -- **Run Simulated Training Worker / GPU Runner**: - ```bash - # Standard worker simulation: - python3 simulators/training_worker.py - - # Or run a specific study: - HPO_BROKER_URL=http://localhost:8000 HPO_STUDY_NAME=your_study_name python3 simulators/training_worker.py - ``` -- **Launch Fallback Optuna Dashboard**: - ```bash - optuna-dashboard sqlite:///hpo_studies.db - ``` -- **Run Integration Tests**: - ```bash - .venv/bin/python3 tests/test_integration.py - ``` - ---- - -## High-Level Architecture - -This project implements a decentralized **Worker-Broker-Registry** pattern for hyperparameter tuning, specifically optimized for a **U-Net crack segmentation project** (and adaptable via templates). It avoids blocking training workers on LLM calls. - -``` -+---------------------------+ MCP +---------------------------+ -| AI Assistant (IDE) | <-----------------> | FastMCP Server (Local) | -+---------------------------+ +---------------------------+ - | | - Reads Stats Read / Write - v v -+-----------------------------------------------------------------------------+ -| Database (Local SQLite / Hosted Cloud PostgreSQL) | -+-----------------------------------------------------------------------------+ - ^ - Read / Write - | - +---------------------------+ - | Training Worker (Any Box)| ---> [ U-Net Pipeline ] - +---------------------------+ -``` - -### Architecture Components - -1. **SQLite Database (`hpo_studies.db`)**: The **single source of truth** for all persistence. Holds both Optuna's trial states and custom metadata tables (`trial_results`, `system_configuration`, `study_reviews`, `study_status`, `agent_reasoning_logs`, `invalid_proposals`). -2. **FastAPI Broker (`broker.py`)**: Thin HTTP broker API exposing endpoints for workers (`/api/suggest_trial`, `/api/report_epoch`, `/api/complete_trial`) and the dashboard web interface (`/api/hpo_config`, `/api/review_packet`). -3. **MCP Server (`hpo_mcp_server.py`)**: FastMCP server exposing tools for human-in-the-loop coordination, onboarding, and reviews to IDE agents. -4. **Decoupled Worker (`src/hpo_client.py`, `colab_worker.py`, `simulators/training_worker.py`)**: Interacts exclusively via HTTP using the 3-step life cycle (`suggest` -> `report_epoch` -> `complete`). Colab reference: `train_colab_trial` (one trial) and `train_colab_trial_loop` (repeated session). -5. **Decoupled Evaluator (Interactive Dashboard)**: Custom dashboard calling the FastAPI routes to view trials, Pareto fonts, fANOVA parameter importances, and toggle coordinator reviews. - ---- - -## Code Style & Development Guidelines - -1. **State & DB Isolation**: State MUST live in SQLite. Do **not** write or persist temporary configurations to files on disk like `active_search_space.json` or `hpo_config.json`. Always load/save via the `SystemConfiguration` ORM table. -2. **Database Resilience**: Column additions or schema model changes should be registered in `src/db_manager.py:__ADDITIVE_COLUMNS` to handle additive, idempotent migrations on runtime initialization instead of dropping tables. -3. **Coordinator Reviews (Episodic LLM)**: IDE agents act as **episodic coordinators**. Optuna (TPE) is the hot sampling path; workers NEVER block on language models. -4. **Review Procedure**: - - Retrieve compacted statistical packet with `get_study_data()`. - - Perform a safety review of VRAM and evaluate health alerts/triggers. - - Adjust active bounds or propose changes with `update_search_space()`. - - Submit review idempotently via `submit_agent_review()`. -5. **Worker Integration Contract**: Ensure newly integrated training scripts use `TrialSession` client rather than direct SQL queries or custom database drivers. diff --git a/Dockerfile b/Dockerfile index 727b1c0..6acbed4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,5 +12,9 @@ COPY . . # Expose broker port EXPOSE 8000 +# Health check +HEALTHCHECK --interval=30s --timeout=3s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')" || exit 1 + # Run uvicorn on 0.0.0.0 CMD ["python", "broker.py", "--daemon", "--host", "0.0.0.0", "--port", "8000"] diff --git a/README.md b/README.md index c260339..853a647 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,9 @@ # Pathfinder -[![Python 3.10+](https://img.shields.io/badge/python-3.10+-3776AB?style=flat-square&logo=python&logoColor=white)](https://www.python.org/downloads/) -[![FastAPI](https://img.shields.io/badge/FastAPI-009688?style=flat-square&logo=fastapi&logoColor=white)](https://fastapi.tiangolo.com/) -[![Optuna](https://img.shields.io/badge/Optuna-Tuning-1E90FF?style=flat-square)](https://optuna.org/) -[![SQLite](https://img.shields.io/badge/SQLite-003B57?style=flat-square&logo=sqlite&logoColor=white)](https://www.sqlite.org/) -[![MCP](https://img.shields.io/badge/MCP-Model_Context_Protocol-orange?style=flat-square)](https://modelcontextprotocol.io/) [![Build Status](https://github.com/Ishaan1402/pathfinder/actions/workflows/integration.yml/badge.svg)](https://github.com/Ishaan1402/pathfinder/actions) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg?style=flat-square)](LICENSE) -A layered hyperparameter optimization (HPO) framework that keeps quick Optuna suggestions seperate from episodic AI reviews. Workers run training script loops autonomously updating with optimized HPs without being blocked by LLM evaluation. This leaves you (or agents in your IDE) to review results periodically and decide when to adjust the search space or overarching strategy. +Pathfinder is an MCP-integrated hyperparameter optimization dashboard that lets AI coding agents onboard your training script and inspect running experiments. It wraps Optuna's TPE sampler in a FastAPI broker with SQLite persistence and a vanilla JS dashboard, while exposing structured study data through Model Context Protocol tools so your IDE agent can meaningfully participate in the tuning loop. @@ -17,32 +12,30 @@ A layered hyperparameter optimization (HPO) framework that keeps quick Optuna su
Hyperparameter Pathways Plot - ASHA Pruning Timeline + Pruning Timeline
- **Designed for:** ML researchers and students tuning deep learning models on their own infrastructure (local GPU, Colab, cloud VMs). Connect your existing training loop in just 4 lines of code — see [For Your Own Project](#onboarding-your-own-project) below. +**Designed for:** ML researchers tuning deep learning models on their own hardware (local GPU, Colab, cloud VMs). Connect your training loop in ~60 lines of code. ## Why Pathfinder? -**Problem:** Traditional HPO frameworks execute fast, but they typically operate within fixed boundaries. If your initial search space is poorly posed or if specific hyperparameter combinations trigger hardware failures (like CUDA OOMs or gradient explosions), a traditional optimizer will blindly burn through your GPU budget until it hits its limit. Fixing this requires the researcher to manually monitor charts, context-switch out of the IDE, and rewrite configuration files by hand. +ML practitioners waste GPU hours on poorly-bounded search spaces and have to manually inspect trial data by grepping logs or refreshing notebooks. Pathfinder gives you a live monitoring dashboard plus an MCP server so your IDE agent can read study state and help onboard new studies. It does not compete with W&B Sweeps or Ray Tune — it's a demonstration of agent-assisted HPO workflows. -**Solution:** Three independent layers: +Three independent layers: -- **Broker (Optuna TPE)**: Quick, deterministic suggestion engine. Hyperparameter suggestions and pruning happen in <10ms. Workers access this endpoint and continue training. -- **Worker**: Train autonomously in loops. Report metrics incrementally. Handles pruning (early stoppage), OOM, checkpointing. -- **Coordinator (You + Optional LLM)**: Run episodic reviews when *you* decide. Inspect trial history, check search health, propose bounds changes. AI agents (Claude, Cursor) can run reviews via MCP tools. +- **Broker (Optuna TPE)**: Fast, deterministic suggestion engine. Suggestions and pruning happen in <10ms. Workers hit the broker and continue training immediately. +- **Worker**: Trains autonomously in a loop. Reports metrics per epoch, handles pruning, OOM detection, and checkpointing. +- **Coordinator (you + optional LLM)**: Run episodic reviews when you decide. Inspect trial history, check search health, propose bounds changes. AI agents (Cursor, Claude Code) can assist via MCP tools. -All state lives in **SQLite** making it easy to resume reviews, audit decisions, and sync across machines. +All state lives in **SQLite** — resumable, auditable, portable. ## Quick Start ### Step 1: Start the Broker -The broker manages the study state and serves the dashboard. You can run it via Docker or Python. - -**Option A: Docker (Zero-Install)** +**Option A: Docker (zero-install)** ```bash docker-compose up -d @@ -61,50 +54,56 @@ python broker.py --daemon ### Step 2: Connect Your Workers -Workers run your training loops. They can be on the same machine or remote. - -**Local Workers on the same machine** +**Local worker (same machine)** ```bash -# Replace 'train.py' with your own training script HPO_BROKER_URL=http://localhost:8000 HPO_STUDY_NAME=my_study python train.py ``` -**Remote Workers (Colab / Cloud GPU)** -To connect remote workers to your local broker, use a tunnel: +**Remote worker (Colab / cloud GPU)** + +To expose your local broker to remote workers, use a tunnel: ```bash -# Local Terminal: Start broker with tunnel + auth +# Generate a token export HPO_SECRET_TOKEN="$(python -c 'import secrets; print(secrets.token_urlsafe(32))')" -# ngrok (auto-generates URL) +# Start broker with tunnel (ngrok auto-generates a URL) python broker.py --daemon --tunnel -# OR Cloudflare (bring your own domain) +# Or with Cloudflare (bring your own domain): python broker.py --daemon --tunnel-provider cloudflare --tunnel-url https://your-domain.com +``` -# Prints: 🔥 Remote broker URL established: https://... +Set the printed URL and token on your remote machine: -# Remote Server Terminal: Set environment and run your worker +```bash export HPO_BROKER_URL="https://..." export HPO_SECRET_TOKEN="" python train.py ``` -### Step 3: Agent Integration (Optional) +See [docs/INTEGRATION.md](docs/INTEGRATION.md) for more tunneling and auth options. -Point Claude Code, Cursor, Antigravity, etc to the MCP server for agent-driven onboarding and reviews. See [IDE Setup](#ide-setup-agent-driven-onboarding--reviews) below. +### Step 3: Agent Integration (optional) -### Environment Variables Reference +Point your IDE at the MCP server for agent-driven onboarding and inspection. See [IDE Setup](#ide-setup-agent-driven-onboarding--inspection). -Pathfinder supports the following optional environment variables for users: +### Environment Variables Reference -- `HPO_DATABASE_URL`: SQLite connection string (default: `sqlite:///hpo_studies.db`). -- `HPO_BROKER_URL`: The URL where the broker is running (e.g. `http://localhost:8000`). Required by workers. -- `HPO_STUDY_NAME`: The active study name. Overrides what is passed in code. -- `HPO_SECRET_TOKEN`: Bearer token for securing broker endpoints in remote deployments. -- `HPO_DEBUG`: Set to `1` to enable verbose debug logging in the broker. -- `HPO_SPARKLINES`: Set to `1` in the worker to print a unicode performance curve on trial completion. +| Variable | Default | Description | +|---|---|---| +| `HPO_DATABASE_URL` | `sqlite:///hpo_studies.db` | SQLite connection string. | +| `HPO_BROKER_URL` | `http://localhost:8000` | URL where the broker is running. Required by workers. | +| `HPO_STUDY_NAME` | *(none)* | Default study name when not passed explicitly. | +| `HPO_SECRET_TOKEN` | *(none)* | Bearer token for securing broker endpoints in remote deployments. | +| `HPO_DEBUG` | `0` | Set to `1` to enable verbose debug logging in the broker. | +| `HPO_SPARKLINES` | `0` | Set to `1` to print a Unicode performance curve on trial completion. | +| `HPO_BACKUP_ON_START` | `0` | Set to `1` to run a database backup when the broker starts. | +| `HPO_CAPTURE_FULL_ENV` | `0` | Set to `1` to capture full `pip freeze` output (default: ML whitelist only). | +| `HPO_TUNNEL_PROVIDER` | *(none)* | Tunnel provider for remote access: `ngrok` or `cloudflare`. | +| `HPO_TUNNEL_URL` | *(none)* | Static tunnel URL when using `cloudflare` provider. | +| `HPO_ALLOWED_ORIGINS` | *(none)* | Additional CORS origins (comma-separated) for the dashboard. | --- @@ -112,109 +111,140 @@ Pathfinder supports the following optional environment variables for users: ### Optuna Engine -- **Tree-structured Parzen Estimator Sampler**: Probability-based hyperparameter suggestions (beats grid search) -- **ASHA Pruning**: Cuts underperforming trials early to save GPU time -- **Single or Dual-Objective**: Optimize a single target, or map a Pareto front between one 'maximize' and one 'minimize' metric (e.g., accuracy vs. latency) - - *(Example: An [image segmentation model](https://github.com/Ishaan1402/crack-seg#crack-seg) could map a Pareto front to maximize Dice Score while minimizing BCE Loss).* -- **fANOVA Importances**: Identifies which hyperparams actually matter to further guide your strategy +- **TPE Sampler**: Tree-structured Parzen Estimator — probability-based hyperparameter suggestions that beat grid and random search +- **Median Pruning**: Cuts underperforming trials early to save GPU time +- **Single or Dual-Objective**: Optimize one target, or map a Pareto front between a maximize and a minimize metric (e.g., accuracy vs. loss) +- **fANOVA Importances**: Identifies which hyperparameters actually matter -### Tuning Coordinator +### Study Health Monitoring -- Dashboard shows health warnings (nudges to review, never auto-reviews) -- 7-step review procedure: retrieve telemetry → evaluate fANOVA → safety/OOM check → accuracy self-regulation → propose bound adjustments → submit audit trail → generate model card -- Search space proposals are staged, requiring your explicit approval before taking effect -- Coordinator accuracy tracks your reviews' forecasted score improvements vs. measured deltas -- Optional LLM integration (Claude, Gemini, OpenAI) for automatic reviews +The dashboard and `.hpo_status.json` show a health tier: -### Persistent Study State using SQLite +| Tier | Meaning | +|------|---------| +| `healthy` | Trials are completing, metrics are improving | +| `watch` | Stagnation or early warning signs | +| `intervene` | High OOM rate, prolonged stagnation, or 100% prune rate | -All configuration, trials, reviews, and metadata live in `hpo_studies.db`: +Health checks detect stagnation (best score flat-lining) and hardware failure patterns (CUDA OOM on specific batch sizes). -- Active search space -- Trial results + VRAM telemetry +### Persistent SQLite State + +All configuration, trials, reviews, and metadata live in `hpo_studies.db`: +- Active search space and HPO config +- Trial results with VRAM telemetry - Coordinator review history -- Study health tier - Generated model cards +### MCP Server + +An MCP server (`hpo_mcp_server.py`) exposes structured study data through Model Context Protocol tools so your IDE agent can read study state, validate manifests, and register new studies. + +--- + +## Agent Integration + +Pathfinder exposes MCP tools that let your IDE agent (Cursor, Claude Code, Antigravity) participate in two workflows: + +### Onboarding Flow + +1. Agent reads your training script, identifies tunable hyperparameters and metrics +2. Agent drafts a `train.hpo.yaml` manifest +3. Agent calls `validate_manifest` to check for errors +4. Agent calls `init_from_manifest` to register the study in Optuna and SQLite +5. Agent writes a minimal worker script from `templates/worker_minimal.py` + +### Inspection Flow + +1. Agent calls `get_study_data` to retrieve trial telemetry, health tier, fANOVA importances, and best trials +2. Agent summarizes: current best score, health status, OOM rate, stagnation warnings +3. Search space adjustments happen through the dashboard Settings UI or `hpo_cli.py` + +Key MCP tools: `validate_manifest`, `init_from_manifest`, `get_study_data`, `get_study_cards`, `export_manifest`. + +Trigger phrases: say **"integrate HPO"** or **"wire hyperparameter tuning"** and your agent will walk through the onboarding flow. For inspection, say **"show study health"** or **"check HPO progress."** + +See [AGENTS.md](AGENTS.md) for the full agent procedure. + --- ## Onboarding Your Own Project -If you cloned this to tune your own model, the easiest way to start is by having an agent (via Cursor, Claude Code, Antigravity, etc) write the manifest for you. - -After setting up the Pathfinder MCP server, simply open your training script and tell your agent something like **"help me wire this training script up to Pathfinder."**. The agent will read your script, identify tunable hyperparameters, and automatically draft the manifest. - -Otherwise, you can onboard manually: - -1. **Write a manifest** (`train.hpo.yaml`): - ```yaml - study_name: my_study - metrics: - objectives: - - name: loss - direction: minimize - - name: accuracy - direction: maximize - params: - - name: learning_rate - type: float_log - min: 1e-5 - max: 1e-2 - - name: batch_size - type: categorical - options: [4, 8, 16, 32] - worker: - entrypoint: python train.py - ``` -2. **Register the study**: - ```bash - python hpo_cli.py validate train.hpo.yaml - python hpo_cli.py init train.hpo.yaml - ``` -3. **Update your training script** (`train.py`): - Instead of hardcoding your hyperparameters, ask the Pathfinder broker for them at the start of your script, and report your loss at the end of each epoch. FastAPI endpoints will facilitate communication between Optuna and your training loop to auto-update inputs based on each trial's iterative output. - ```python - from src.hpo_client import TrialSession - - # 1. Connect to broker and get parameters - session = TrialSession(broker_url="http://localhost:8000", study_name="my_study") - trial = session.suggest() - learning_rate = trial["params"]["learning_rate"] - - for epoch in range(epochs): - loss = train_one_epoch(lr=learning_rate) - - # 2. Report metrics (Pathfinder handles pruning automatically) - if session.report_epoch(epoch, loss=loss): - break # Trial was pruned - - # 3. Mark completion - session.complete(epoch, loss=loss, state="COMPLETE") - ``` -4. **Run on your GPU** (set env vars first): - ```bash - export HPO_BROKER_URL=http://localhost:8000 - export HPO_STUDY_NAME=my_study - python train.py - ``` - -Full integration walkthrough: [docs/INTEGRATION.md](docs/INTEGRATION.md) +### 1. Write a manifest (`train.hpo.yaml`) + +```yaml +study_name: my_study +metrics: + primary_score: accuracy + objectives: + - name: accuracy + direction: maximize + label: "Accuracy" + - name: loss + direction: minimize + label: "Loss" +params: + - name: learning_rate + type: float_log + min: 1e-5 + max: 1e-2 + - name: batch_size + type: categorical + options: [4, 8, 16, 32] +worker: + entrypoint: python train.py +``` + +The `primary_score` field tells the dashboard which objective to highlight. + +### 2. Register the study + +```bash +python hpo_cli.py validate train.hpo.yaml +python hpo_cli.py init train.hpo.yaml +``` + +### 3. Update your training script + +```python +from src.hpo_client import TrialSession + +session = TrialSession() # reads HPO_BROKER_URL / HPO_STUDY_NAME +trial = session.suggest() +params = trial["params"] + +for epoch in range(epochs): + accuracy, loss = train_one_epoch(params, epoch) + + if session.report_epoch(epoch, score=accuracy, loss=loss): + # Trial was pruned by the broker + session.complete(epoch, score=accuracy, loss=loss, state="PRUNED") + break + +session.complete(epoch, score=accuracy, loss=loss, state="COMPLETE") +``` + +The worker contract is three calls: `suggest()`, `report_epoch()`, `complete()`. Map your higher-is-better metric to `score` and your lower-is-better metric to `loss`. Full details: [docs/INTEGRATION.md](docs/INTEGRATION.md). + +### 4. Run on your GPU + +```bash +export HPO_BROKER_URL=http://localhost:8000 +export HPO_STUDY_NAME=my_study +python train.py +``` --- -## IDE Setup (Agent-Driven Onboarding & Reviews) +## IDE Setup (Agent-Driven Onboarding & Inspection) ### Cursor -1. **Settings → Features → MCP** -2. **+ Add New MCP Server** -3. Name: `pathfinder` - Type: `command` - Command: `source .venv/bin/activate && python3 hpo_mcp_server.py` +**Settings → Features → MCP → + Add New MCP Server** -### Claude Code / Antigravity +Name: `pathfinder`, Type: `command`, Command: `source .venv/bin/activate && python3 hpo_mcp_server.py` -Add to your MCP config (`~/.config/claudecode/mcp_config.json` or similar): +### Claude Code / Antigravity ```json { @@ -230,16 +260,7 @@ Add to your MCP config (`~/.config/claudecode/mcp_config.json` or similar): } ``` -### Other MCP Clients (OpenCode, etc.) - -Pathfinder is compliant with the Model Context Protocol standard. You can integrate it with any other MCP-compatible IDE or agent using its standard configuration method, pointing it to `python3 hpo_mcp_server.py`. - -Then tell your agent: - -- **"integrate HPO"** → agent drafts manifest, validates, registers study -- **"run a coordinator review"** → agent fetches study data, rates health, proposes bounds changes - -See [AGENTS.md](AGENTS.md) for the full procedure. +Pathfinder is compliant with the Model Context Protocol standard — it works with any MCP-compatible IDE. --- @@ -249,21 +270,25 @@ See [AGENTS.md](AGENTS.md) for the full procedure. # Start broker + dashboard python broker.py --daemon -# Validate & initialize a study from manifest +# Validate and initialize a study from manifest python hpo_cli.py validate train.hpo.yaml python hpo_cli.py init train.hpo.yaml # Check study health python hpo_cli.py status -# Run a manual coordinator review (or prints prompt for copy-paste) -python hpo_cli.py review - # Export study config back to YAML python hpo_cli.py manifest my_study -# Commit pending search space changes -python hpo_cli.py apply +# Export/import study data +python hpo_cli.py export my_study --output my_study.json +python hpo_cli.py import my_study.json + +# Generate a model card +python hpo_cli.py modelcard my_study + +# Delete a study +python hpo_cli.py delete my_study # Run tests pytest tests/ -q @@ -271,23 +296,29 @@ pytest tests/ -q --- -## Reference: crack-seg +## Limitations + +This is not a production HPO framework. It runs on a single machine with SQLite. It does not support distributed studies, Postgres backends, or advanced samplers like MOTPE or CMA-ES. Use Optuna's native dashboard or W&B Sweeps for production workloads. Pathfinder is a demonstration of MCP/agent integration for ML experiment workflows. + +## What I Learned + +Building Pathfinder taught me the MCP architecture: how to expose structured tool surfaces so an IDE agent can participate in a tuning loop without blocking the hot path. I learned the lease/reap concurrency pattern for worker lifecycle management — detecting dead workers and reclaiming their trials without false positives. I also gained respect for SQLite as an application database; with WAL mode and careful connection pooling, it handled concurrent broker + dashboard + MCP reads without ever becoming the bottleneck. -This Pathfinder instance was initially tuned for [crack-seg](https://github.com/Ishaan1402/crack-seg#crack-seg), a **U-Net pixel-level segmentation model** trained on high-res UAV bridge imagery. See [colab_worker.py](colab_worker.py) for the full reference implementation (dataset download, model setup, training loop). +--- + +## Reference: crack-seg -**Don't modify `colab_worker.py`** unless you're maintaining the bridge-crack project. Cloners should use `templates/worker_minimal.py` instead. +Pathfinder was initially built to tune [crack-seg](https://github.com/Ishaan1402/crack-seg#crack-seg), a U-Net pixel-level segmentation model trained on UAV bridge imagery. The reference implementation (`colab_worker.py`) is preserved in [archive/](archive/). **Do not fork it** for new studies — use `templates/worker_minimal.py` instead. --- ## Docs -- **[AGENTS.md](AGENTS.md)** — Guide for AI agents (Claude, Cursor, Antigravity) -- **[CLAUDE.md](CLAUDE.md)** — Development commands and architecture for Claude Code -- **[examples/onboarding/](examples/onboarding/)** — Step-by-step walkthrough for a new project +- **[AGENTS.md](AGENTS.md)** — Guide for AI agents (Cursor, Claude Code, Antigravity) - **[docs/INTEGRATION.md](docs/INTEGRATION.md)** — Worker integration contract details --- ## License -MIT License - see the [LICENSE](LICENSE) file for details. \ No newline at end of file +MIT License - see the [LICENSE](LICENSE) file for details. diff --git a/archive/README.md b/archive/README.md new file mode 100644 index 0000000..1540e3f --- /dev/null +++ b/archive/README.md @@ -0,0 +1,11 @@ +# Archive + +This directory contains code that was part of Pathfinder's development history but is not part of the current API surface. + +## colab_worker.py + +The original bridge-crack U-Net reference implementation. This was the study Pathfinder was initially built to tune — a pixel-level crack segmentation model trained on high-res UAV bridge imagery. + +It still works as a standalone Colab worker, but uses domain-specific naming (dice/bce) that predates Pathfinder's generic score/loss metric abstraction. It is kept here for historical context and as evidence of the project's origins. + +**Do not use this as a template for new studies.** Use `templates/worker_minimal.py` instead. diff --git a/archive/bridge_crack_500px_plots.png b/archive/bridge_crack_500px_plots.png new file mode 100644 index 0000000..3c3e667 Binary files /dev/null and b/archive/bridge_crack_500px_plots.png differ diff --git a/archive/bridge_crack_study_500px.csv b/archive/bridge_crack_study_500px.csv new file mode 100644 index 0000000..00cf235 --- /dev/null +++ b/archive/bridge_crack_study_500px.csv @@ -0,0 +1,8 @@ +trial_number,trial_id,state,score,loss,duration_seconds,epoch_reached,learning_rate,batch_size,resolution,encoder_name,loss_weight_ratio,model_capacity,gpu_model,max_vram_gb +0,10,COMPLETE,0.858018802569735,3.420776633773675,1666.775988,15,1.3000740917849293e-05,8,512,efficientnet-b0,0.4569677019352898,N/A,, +1,11,COMPLETE,0.8570024923984412,3.167852641157963,2674.331473,15,0.009016956432821428,4,1024,resnet34,0.8759492571891342,N/A,, +25,35,COMPLETE,0.857793011512137,3.20810041447732,339.773174,25,0.0016227567399957443,64,512,,0.9312941039111402,wide,, +26,36,COMPLETE,0.8579996422931605,3.450359302231028,140.290804,25,1.36045241634878e-05,16,512,,0.7556562618734615,narrow,, +27,93,COMPLETE,0.858148133915902,3.164196773923399,142.660719,25,0.009599869896398518,16,512,,0.6784632970852156,narrow,, +40,178,COMPLETE,0.8553575962463495,3.1704550543917884,564.41964,15,0.0011162195120225547,8,1024,,0.8482662431928073,narrow,NVIDIA A100-SXM4-40GB,39.4935302734375 +66,204,COMPLETE,0.8557610089292585,3.171118880123026,1338.960238,15,0.0034464570787237354,4,1024,,0.6326231660481053,narrow,NVIDIA L4,22.0343017578125 diff --git a/archive/bridge_crack_study_trials.json b/archive/bridge_crack_study_trials.json new file mode 100644 index 0000000..bcc234a --- /dev/null +++ b/archive/bridge_crack_study_trials.json @@ -0,0 +1,2858 @@ +[ + { + "trial_number": 0, + "trial_id": 10, + "state": "COMPLETE", + "params": { + "learning_rate": 1.3000740917849293e-05, + "batch_size": 8, + "resolution": 512, + "encoder_name": "efficientnet-b0", + "loss_weight_ratio": 0.4569677019352898 + }, + "datetime_start": "2026-06-03T13:43:13.054883", + "datetime_complete": "2026-06-03T14:10:59.830871", + "duration_seconds": 1666.775988, + "optuna_loss": 3.420776633773675, + "optuna_score": 0.858018802569735, + "epoch_reached": 15, + "primary_score": 0.858018802569735, + "primary_loss": 3.420776633773675, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 1, + "trial_id": 11, + "state": "COMPLETE", + "params": { + "learning_rate": 0.009016956432821428, + "batch_size": 4, + "resolution": 1024, + "encoder_name": "resnet34", + "loss_weight_ratio": 0.8759492571891342 + }, + "datetime_start": "2026-06-03T14:24:34.312874", + "datetime_complete": "2026-06-03T15:09:08.644347", + "duration_seconds": 2674.331473, + "optuna_loss": 3.167852641157963, + "optuna_score": 0.8570024923984412, + "epoch_reached": 15, + "primary_score": 0.8570024923984412, + "primary_loss": 3.167852641157963, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 2, + "trial_id": 12, + "state": "COMPLETE", + "params": { + "learning_rate": 0.0004925474076170948, + "batch_size": 2, + "resolution": 256, + "encoder_name": "resnet34", + "loss_weight_ratio": 0.31234114076019825 + }, + "datetime_start": "2026-06-03T15:13:22.895474", + "datetime_complete": "2026-06-03T15:16:10.457808", + "duration_seconds": 167.562334, + "optuna_loss": 0.05479208334007218, + "optuna_score": 0.9898519602789447, + "epoch_reached": 15, + "primary_score": 0.9898519602789447, + "primary_loss": 0.05479208334007218, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 3, + "trial_id": 13, + "state": "PRUNED", + "params": { + "learning_rate": 0.0014412064723689244, + "batch_size": 32, + "resolution": 512, + "encoder_name": "resnet50", + "loss_weight_ratio": 0.43849321487231385 + }, + "datetime_start": "2026-06-03T15:58:28.404445", + "datetime_complete": "2026-06-03T16:01:28.153573", + "duration_seconds": 179.749128, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 4, + "trial_id": 14, + "state": "PRUNED", + "params": { + "learning_rate": 0.000592418490813305, + "batch_size": 16, + "resolution": 512, + "encoder_name": "efficientnet-b0", + "loss_weight_ratio": 0.5883136720418263 + }, + "datetime_start": "2026-06-03T16:02:12.182605", + "datetime_complete": "2026-06-03T16:02:29.436115", + "duration_seconds": 17.25351, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 5, + "trial_id": 15, + "state": "PRUNED", + "params": { + "learning_rate": 0.00866044884543897, + "batch_size": 2, + "resolution": 512, + "encoder_name": "resnet34", + "loss_weight_ratio": 0.1362870798746597 + }, + "datetime_start": "2026-06-03T16:10:37.069107", + "datetime_complete": "2026-06-03T16:10:54.118335", + "duration_seconds": 17.049228, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 6, + "trial_id": 16, + "state": "COMPLETE", + "params": { + "learning_rate": 2.6272157099819104e-05, + "batch_size": 32, + "resolution": 256, + "encoder_name": "resnet50", + "loss_weight_ratio": 0.6982048083978067 + }, + "datetime_start": "2026-06-03T16:14:55.109066", + "datetime_complete": "2026-06-03T16:15:48.273632", + "duration_seconds": 53.164566, + "optuna_loss": 0.34094087169643195, + "optuna_score": 0.9900689937983594, + "epoch_reached": 15, + "primary_score": 0.9900689937983594, + "primary_loss": 0.34094087169643195, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 7, + "trial_id": 17, + "state": "PRUNED", + "params": { + "learning_rate": 1.5280813833109962e-05, + "batch_size": 32, + "resolution": 256, + "encoder_name": "efficientnet-b0", + "loss_weight_ratio": 0.5494909479212793 + }, + "datetime_start": "2026-06-03T16:20:02.454960", + "datetime_complete": "2026-06-03T16:20:10.660498", + "duration_seconds": 8.205538, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 8, + "trial_id": 18, + "state": "PRUNED", + "params": { + "learning_rate": 0.0015560408056926069, + "batch_size": 2, + "resolution": 512, + "model_capacity": "wide", + "loss_weight_ratio": 0.6541735359413245 + }, + "datetime_start": "2026-06-03T16:36:07.874740", + "datetime_complete": "2026-06-03T16:36:24.756243", + "duration_seconds": 16.881503, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 9, + "trial_id": 19, + "state": "PRUNED", + "params": { + "learning_rate": 1.3128228293775808e-05, + "batch_size": 8, + "resolution": 1024, + "model_capacity": "wide", + "loss_weight_ratio": 0.8645862853362126 + }, + "datetime_start": "2026-06-03T16:42:10.936967", + "datetime_complete": "2026-06-03T16:43:46.471404", + "duration_seconds": 95.534437, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 10, + "trial_id": 20, + "state": "PRUNED", + "params": { + "learning_rate": 0.004406605185153241, + "batch_size": 2 + }, + "datetime_start": "2026-06-03T16:44:51.217700", + "datetime_complete": "2026-06-03T16:45:22.544899", + "duration_seconds": 31.327199, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 11, + "trial_id": 21, + "state": "PRUNED", + "params": { + "learning_rate": 0.004606786973558291, + "batch_size": 64 + }, + "datetime_start": "2026-06-03T16:45:43.747381", + "datetime_complete": "2026-06-03T16:46:13.669698", + "duration_seconds": 29.922317, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 12, + "trial_id": 22, + "state": "FAIL", + "params": { + "learning_rate": 0.009238447012502312, + "batch_size": 8 + }, + "datetime_start": "2026-06-03T16:48:01.525344", + "datetime_complete": "2026-06-03T16:48:01.552523", + "duration_seconds": 0.027179, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 13, + "trial_id": 23, + "state": "FAIL", + "params": { + "learning_rate": 0.0006548097272413485, + "batch_size": 32 + }, + "datetime_start": "2026-06-03T16:48:21.372164", + "datetime_complete": "2026-06-03T16:48:21.385289", + "duration_seconds": 0.013125, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 14, + "trial_id": 24, + "state": "FAIL", + "params": { + "learning_rate": 2.4349414095370824e-05, + "batch_size": 8, + "model_capacity": "narrow", + "loss_weight_ratio": 0.9316343050565551 + }, + "datetime_start": "2026-06-03T16:49:13.338941", + "datetime_complete": "2026-06-03T16:49:28.821244", + "duration_seconds": 15.482303, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 15, + "trial_id": 25, + "state": "FAIL", + "params": { + "learning_rate": 0.00014020437791801606, + "batch_size": 4, + "model_capacity": "narrow", + "loss_weight_ratio": 0.1742679306216336 + }, + "datetime_start": "2026-06-03T16:49:28.830983", + "datetime_complete": "2026-06-03T16:49:36.991214", + "duration_seconds": 8.160231, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 16, + "trial_id": 26, + "state": "FAIL", + "params": { + "learning_rate": 0.0016347032047742205, + "batch_size": 4, + "model_capacity": "narrow", + "loss_weight_ratio": 0.03717144506513459 + }, + "datetime_start": "2026-06-03T16:49:37.001315", + "datetime_complete": "2026-06-03T16:51:21.026869", + "duration_seconds": 104.025554, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 17, + "trial_id": 27, + "state": "FAIL", + "params": { + "learning_rate": 0.001424331563772705, + "batch_size": 8, + "model_capacity": "narrow", + "loss_weight_ratio": 0.38679370235988164 + }, + "datetime_start": "2026-06-03T16:51:21.047576", + "datetime_complete": "2026-06-03T16:52:06.743811", + "duration_seconds": 45.696235, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 18, + "trial_id": 28, + "state": "PRUNED", + "params": { + "learning_rate": 0.0006710326147742782, + "batch_size": 16, + "model_capacity": "narrow", + "loss_weight_ratio": 0.002323245785979422 + }, + "datetime_start": "2026-06-03T16:52:06.749228", + "datetime_complete": "2026-06-03T16:52:06.767498", + "duration_seconds": 0.01827, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 19, + "trial_id": 29, + "state": "FAIL", + "params": {}, + "datetime_start": "2026-06-03T16:52:13.957698", + "datetime_complete": "2026-06-03T16:52:28.811225", + "duration_seconds": 14.853527, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 20, + "trial_id": 30, + "state": "PRUNED", + "params": { + "learning_rate": 0.005569662100056995, + "batch_size": 2, + "model_capacity": "wide", + "loss_weight_ratio": 0.9243719613528233 + }, + "datetime_start": "2026-06-03T16:52:28.818704", + "datetime_complete": "2026-06-03T16:52:28.836298", + "duration_seconds": 0.017594, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 21, + "trial_id": 31, + "state": "PRUNED", + "params": { + "learning_rate": 8.635129594504407e-05, + "batch_size": 8, + "model_capacity": "wide", + "loss_weight_ratio": 0.2159129532506059 + }, + "datetime_start": "2026-06-03T16:53:10.479054", + "datetime_complete": "2026-06-03T16:53:27.942296", + "duration_seconds": 17.463242, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 22, + "trial_id": 32, + "state": "PRUNED", + "params": { + "learning_rate": 1.2671531921681287e-05, + "batch_size": 32, + "model_capacity": "wide", + "loss_weight_ratio": 0.8084256426884586 + }, + "datetime_start": "2026-06-03T16:56:05.615491", + "datetime_complete": "2026-06-03T16:56:34.427798", + "duration_seconds": 28.812307, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 23, + "trial_id": 33, + "state": "PRUNED", + "params": { + "learning_rate": 2.5298748345870258e-05, + "batch_size": 32, + "model_capacity": "narrow", + "loss_weight_ratio": 0.8252409941205194 + }, + "datetime_start": "2026-06-03T17:00:35.578893", + "datetime_complete": "2026-06-03T17:01:04.174560", + "duration_seconds": 28.595667, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 24, + "trial_id": 34, + "state": "PRUNED", + "params": { + "learning_rate": 2.223369003885487e-05, + "batch_size": 2, + "model_capacity": "wide", + "loss_weight_ratio": 0.5713714827302104 + }, + "datetime_start": "2026-06-03T17:02:56.369030", + "datetime_complete": "2026-06-03T17:03:24.500350", + "duration_seconds": 28.13132, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 25, + "trial_id": 35, + "state": "COMPLETE", + "params": { + "learning_rate": 0.0016227567399957443, + "batch_size": 64, + "model_capacity": "wide", + "loss_weight_ratio": 0.9312941039111402, + "resolution": 512 + }, + "datetime_start": "2026-06-03T17:15:55.022375", + "datetime_complete": "2026-06-03T17:21:34.795549", + "duration_seconds": 339.773174, + "optuna_loss": 3.20810041447732, + "optuna_score": 0.857793011512137, + "epoch_reached": 25, + "primary_score": 0.857793011512137, + "primary_loss": 3.20810041447732, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 26, + "trial_id": 36, + "state": "COMPLETE", + "params": { + "learning_rate": 1.36045241634878e-05, + "batch_size": 16, + "model_capacity": "narrow", + "loss_weight_ratio": 0.7556562618734615, + "resolution": 512 + }, + "datetime_start": "2026-06-03T17:22:26.684040", + "datetime_complete": "2026-06-03T17:24:46.974844", + "duration_seconds": 140.290804, + "optuna_loss": 3.450359302231028, + "optuna_score": 0.8579996422931605, + "epoch_reached": 25, + "primary_score": 0.8579996422931605, + "primary_loss": 3.450359302231028, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 27, + "trial_id": 93, + "state": "COMPLETE", + "params": { + "learning_rate": 0.009599869896398518, + "batch_size": 16, + "model_capacity": "narrow", + "loss_weight_ratio": 0.6784632970852156, + "resolution": 512 + }, + "datetime_start": "2026-06-03T19:53:51.414244", + "datetime_complete": "2026-06-03T19:56:14.074963", + "duration_seconds": 142.660719, + "optuna_loss": 3.164196773923399, + "optuna_score": 0.858148133915902, + "epoch_reached": 25, + "primary_score": 0.858148133915902, + "primary_loss": 3.164196773923399, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 28, + "trial_id": 130, + "state": "PRUNED", + "params": { + "learning_rate": 3.126406973760604e-05, + "loss_weight_ratio": 0.11912673266270057, + "batch_size": 8, + "resolution": 512, + "model_capacity": "wide" + }, + "datetime_start": "2026-06-03T20:41:25.704340", + "datetime_complete": "2026-06-03T20:41:59.084033", + "duration_seconds": 33.379693, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 29, + "trial_id": 167, + "state": "FAIL", + "params": { + "learning_rate": 0.0003129825951856815, + "batch_size": 64, + "resolution": 1024, + "model_capacity": "wide", + "loss_weight_ratio": 0.6505910029778382 + }, + "datetime_start": "2026-06-05T13:58:19.470631", + "datetime_complete": "2026-06-05T13:59:43.751567", + "duration_seconds": 84.280936, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 30, + "trial_id": 168, + "state": "COMPLETE", + "params": { + "learning_rate": 1.2908087536051017e-05, + "batch_size": 32, + "resolution": 256, + "model_capacity": "narrow", + "loss_weight_ratio": 0.0336207768294422 + }, + "datetime_start": "2026-06-07T13:59:34.848713", + "datetime_complete": "2026-06-07T14:01:51.748743", + "duration_seconds": 136.90003, + "optuna_loss": 0.6156984318660784, + "optuna_score": 0.8673244251587711, + "epoch_reached": 15, + "primary_score": 0.8673244251587711, + "primary_loss": 0.6156984318660784, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA A100-SXM4-40GB", + "max_vram_gb": 39.4935302734375, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 31, + "trial_id": 169, + "state": "FAIL", + "params": { + "learning_rate": 0.005529602980992804, + "batch_size": 16, + "resolution": 1024, + "model_capacity": "narrow", + "loss_weight_ratio": 0.5764868199904598 + }, + "datetime_start": "2026-06-07T14:01:52.028617", + "datetime_complete": "2026-06-07T14:01:54.489417", + "duration_seconds": 2.4608, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 32, + "trial_id": 170, + "state": "PRUNED", + "params": { + "learning_rate": 5.619733318203318e-05, + "batch_size": 32, + "resolution": 256, + "model_capacity": "wide", + "loss_weight_ratio": 0.2684312186462182 + }, + "datetime_start": "2026-06-07T14:01:54.754181", + "datetime_complete": "2026-06-07T14:03:10.632234", + "duration_seconds": 75.878053, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 33, + "trial_id": 171, + "state": "PRUNED", + "params": { + "learning_rate": 0.0023088874045366766, + "batch_size": 2, + "resolution": 256, + "model_capacity": "narrow", + "loss_weight_ratio": 0.6236688985236228 + }, + "datetime_start": "2026-06-07T14:03:10.901884", + "datetime_complete": "2026-06-07T14:04:00.814544", + "duration_seconds": 49.91266, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 34, + "trial_id": 172, + "state": "FAIL", + "params": { + "learning_rate": 0.00965489857451385, + "batch_size": 8, + "resolution": 1024, + "model_capacity": "wide", + "loss_weight_ratio": 0.5779408885200207 + }, + "datetime_start": "2026-06-07T14:04:01.090942", + "datetime_complete": "2026-06-07T14:23:47.673944", + "duration_seconds": 1186.583002, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA A100-SXM4-40GB", + "max_vram_gb": 39.4935302734375, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 35, + "trial_id": 173, + "state": "FAIL", + "params": { + "learning_rate": 0.008667747925003964, + "batch_size": 8, + "resolution": 512, + "model_capacity": "wide", + "loss_weight_ratio": 0.0854227054576977 + }, + "datetime_start": "2026-06-07T14:23:48.011546", + "datetime_complete": "2026-06-07T14:26:55.783299", + "duration_seconds": 187.771753, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 36, + "trial_id": 174, + "state": "PRUNED", + "params": { + "learning_rate": 0.003964260072586964, + "batch_size": 32, + "resolution": 512, + "model_capacity": "narrow", + "loss_weight_ratio": 0.35151491618316477 + }, + "datetime_start": "2026-06-07T14:28:32.046807", + "datetime_complete": "2026-06-07T14:29:22.126494", + "duration_seconds": 50.079687, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 5, + "primary_score": 0.8560799390346749, + "primary_loss": 3.2290616176299407, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA A100-SXM4-40GB", + "max_vram_gb": 39.4935302734375, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 37, + "trial_id": 175, + "state": "PRUNED", + "params": { + "learning_rate": 6.190175730149249e-05, + "batch_size": 8, + "resolution": 256, + "model_capacity": "narrow", + "loss_weight_ratio": 0.8010454023210211 + }, + "datetime_start": "2026-06-07T14:29:23.171195", + "datetime_complete": "2026-06-07T14:29:50.114575", + "duration_seconds": 26.94338, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 5, + "primary_score": 0.9895626247940745, + "primary_loss": 0.4081219550426499, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA A100-SXM4-40GB", + "max_vram_gb": 39.4935302734375, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 38, + "trial_id": 176, + "state": "FAIL", + "params": { + "learning_rate": 6.88390653876011e-05, + "batch_size": 64, + "resolution": 1024, + "model_capacity": "narrow", + "loss_weight_ratio": 0.7564837061499687 + }, + "datetime_start": "2026-06-07T14:29:50.971427", + "datetime_complete": "2026-06-07T14:29:53.344969", + "duration_seconds": 2.373542, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA A100-SXM4-40GB", + "max_vram_gb": 39.4935302734375, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 39, + "trial_id": 177, + "state": "PRUNED", + "params": { + "learning_rate": 3.193828708214736e-05, + "batch_size": 8, + "resolution": 512, + "model_capacity": "wide", + "loss_weight_ratio": 0.8196726856529919 + }, + "datetime_start": "2026-06-07T14:29:53.663673", + "datetime_complete": "2026-06-07T14:31:27.262921", + "duration_seconds": 93.599248, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 5, + "primary_score": 0.855850139872812, + "primary_loss": 3.454779870399443, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA A100-SXM4-40GB", + "max_vram_gb": 39.4935302734375, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 40, + "trial_id": 178, + "state": "COMPLETE", + "params": { + "learning_rate": 0.0011162195120225547, + "batch_size": 8, + "resolution": 1024, + "model_capacity": "narrow", + "loss_weight_ratio": 0.8482662431928073 + }, + "datetime_start": "2026-06-07T14:31:28.087930", + "datetime_complete": "2026-06-07T14:40:52.507570", + "duration_seconds": 564.41964, + "optuna_loss": 3.1704550543917884, + "optuna_score": 0.8553575962463495, + "epoch_reached": 15, + "primary_score": 0.8553575962463495, + "primary_loss": 3.1704550543917884, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA A100-SXM4-40GB", + "max_vram_gb": 39.4935302734375, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 41, + "trial_id": 179, + "state": "PRUNED", + "params": { + "learning_rate": 8.376984597427127e-05, + "batch_size": 32, + "resolution": 256, + "model_capacity": "narrow", + "loss_weight_ratio": 0.08460947756974468 + }, + "datetime_start": "2026-06-07T14:40:52.830605", + "datetime_complete": "2026-06-07T14:41:18.493565", + "duration_seconds": 25.66296, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 5, + "primary_score": 0.97001750168377, + "primary_loss": 0.4865190936291771, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA A100-SXM4-40GB", + "max_vram_gb": 39.4935302734375, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 42, + "trial_id": 180, + "state": "PRUNED", + "params": { + "learning_rate": 0.006814745505395586, + "batch_size": 4, + "resolution": 256, + "model_capacity": "narrow", + "loss_weight_ratio": 0.08170498386801406 + }, + "datetime_start": "2026-06-07T14:41:19.613878", + "datetime_complete": "2026-06-07T14:41:47.336577", + "duration_seconds": 27.722699, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 5, + "primary_score": 0.984673943843873, + "primary_loss": 0.11464002246689445, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA A100-SXM4-40GB", + "max_vram_gb": 39.4935302734375, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 43, + "trial_id": 181, + "state": "FAIL", + "params": { + "learning_rate": 6.435284774708397e-05, + "batch_size": 32, + "resolution": 512, + "model_capacity": "wide", + "loss_weight_ratio": 0.5264320365100166 + }, + "datetime_start": "2026-06-07T14:41:48.511069", + "datetime_complete": "2026-06-07T14:41:50.863072", + "duration_seconds": 2.352003, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA A100-SXM4-40GB", + "max_vram_gb": 39.4935302734375, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 44, + "trial_id": 182, + "state": "PRUNED", + "params": { + "learning_rate": 1.1932677758682133e-05, + "batch_size": 16, + "resolution": 512, + "model_capacity": "wide", + "loss_weight_ratio": 0.7292200426298769 + }, + "datetime_start": "2026-06-07T14:41:51.173891", + "datetime_complete": "2026-06-07T14:43:22.767690", + "duration_seconds": 91.593799, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 5, + "primary_score": 0.7932973447561847, + "primary_loss": 3.5894378851234663, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA A100-SXM4-40GB", + "max_vram_gb": 39.4935302734375, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 45, + "trial_id": 183, + "state": "FAIL", + "params": { + "learning_rate": 0.0003957933664968164, + "batch_size": 32, + "resolution": 512, + "model_capacity": "wide", + "loss_weight_ratio": 0.3679429819753126 + }, + "datetime_start": "2026-06-07T14:43:23.404815", + "datetime_complete": "2026-06-07T14:43:25.751226", + "duration_seconds": 2.346411, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA A100-SXM4-40GB", + "max_vram_gb": 39.4935302734375, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 46, + "trial_id": 184, + "state": "PRUNED", + "params": { + "learning_rate": 0.00011093794054144854, + "batch_size": 32, + "resolution": 512, + "model_capacity": "narrow", + "loss_weight_ratio": 0.8695930976572441 + }, + "datetime_start": "2026-06-07T14:47:45.801056", + "datetime_complete": "2026-06-07T14:48:35.169769", + "duration_seconds": 49.368713, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 5, + "primary_score": 0.847776949003498, + "primary_loss": 3.545791560587501, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA A100-SXM4-40GB", + "max_vram_gb": 39.4935302734375, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 47, + "trial_id": 185, + "state": "FAIL", + "params": { + "learning_rate": 0.0016064112990469893, + "batch_size": 64, + "resolution": 1024, + "model_capacity": "narrow", + "loss_weight_ratio": 0.912719707686549 + }, + "datetime_start": "2026-06-07T14:48:35.876373", + "datetime_complete": "2026-06-07T14:48:38.240824", + "duration_seconds": 2.364451, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA A100-SXM4-40GB", + "max_vram_gb": 39.4935302734375, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 48, + "trial_id": 186, + "state": "PRUNED", + "params": { + "learning_rate": 1.866719221020291e-05, + "batch_size": 4, + "resolution": 512, + "model_capacity": "narrow", + "loss_weight_ratio": 0.4256559084249397 + }, + "datetime_start": "2026-06-07T14:48:38.561165", + "datetime_complete": "2026-06-07T14:49:30.149907", + "duration_seconds": 51.588742, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 5, + "primary_score": 0.8444927109183233, + "primary_loss": 3.504164194758934, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA A100-SXM4-40GB", + "max_vram_gb": 39.4935302734375, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 49, + "trial_id": 187, + "state": "FAIL", + "params": { + "learning_rate": 7.550066161936539e-05, + "batch_size": 32, + "resolution": 1024, + "model_capacity": "wide", + "loss_weight_ratio": 0.022409146295909954 + }, + "datetime_start": "2026-06-07T14:49:31.635068", + "datetime_complete": "2026-06-07T14:49:34.006746", + "duration_seconds": 2.371678, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA A100-SXM4-40GB", + "max_vram_gb": 39.4935302734375, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 50, + "trial_id": 188, + "state": "FAIL", + "params": { + "learning_rate": 0.0053350126131481235, + "batch_size": 4, + "resolution": 1024, + "model_capacity": "wide", + "loss_weight_ratio": 0.5049364450109906 + }, + "datetime_start": "2026-06-07T14:49:34.285916", + "datetime_complete": "2026-06-07T15:12:57.731695", + "duration_seconds": 1403.445779, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 51, + "trial_id": 189, + "state": "FAIL", + "params": { + "learning_rate": 0.000777600200657905, + "batch_size": 64, + "resolution": 512, + "model_capacity": "narrow", + "loss_weight_ratio": 0.5033823758278679 + }, + "datetime_start": "2026-06-09T12:27:32.048350", + "datetime_complete": "2026-06-09T12:27:42.606962", + "duration_seconds": 10.558612, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 52, + "trial_id": 190, + "state": "FAIL", + "params": { + "learning_rate": 0.0003510810257947738, + "batch_size": 16, + "resolution": 1024, + "model_capacity": "narrow", + "loss_weight_ratio": 0.3055155227639622 + }, + "datetime_start": "2026-06-09T12:27:43.506462", + "datetime_complete": "2026-06-09T12:27:46.401631", + "duration_seconds": 2.895169, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 53, + "trial_id": 191, + "state": "FAIL", + "params": { + "learning_rate": 0.00029149980386619276, + "batch_size": 8, + "resolution": 1024, + "model_capacity": "narrow", + "loss_weight_ratio": 0.9024384903900268 + }, + "datetime_start": "2026-06-09T12:27:47.394528", + "datetime_complete": "2026-06-09T12:27:52.254843", + "duration_seconds": 4.860315, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 54, + "trial_id": 192, + "state": "FAIL", + "params": { + "learning_rate": 0.006280369628664032, + "batch_size": 32, + "resolution": 1024, + "model_capacity": "narrow", + "loss_weight_ratio": 0.7468149969124053 + }, + "datetime_start": "2026-06-09T12:27:53.012884", + "datetime_complete": "2026-06-09T12:27:56.525848", + "duration_seconds": 3.512964, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 55, + "trial_id": 193, + "state": "FAIL", + "params": { + "learning_rate": 2.1608825057250263e-05, + "batch_size": 16, + "resolution": 512, + "model_capacity": "wide", + "loss_weight_ratio": 0.8276164452067589 + }, + "datetime_start": "2026-06-09T12:27:57.553135", + "datetime_complete": "2026-06-09T12:28:01.985592", + "duration_seconds": 4.432457, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 56, + "trial_id": 194, + "state": "PRUNED", + "params": { + "learning_rate": 0.006946225010911096, + "batch_size": 2, + "resolution": 512, + "model_capacity": "wide", + "loss_weight_ratio": 0.34343077115154985 + }, + "datetime_start": "2026-06-09T12:28:03.278519", + "datetime_complete": "2026-06-09T12:31:35.254234", + "duration_seconds": 211.975715, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 5, + "primary_score": 0.8567204705534222, + "primary_loss": 3.1828140226597523, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 57, + "trial_id": 195, + "state": "FAIL", + "params": { + "learning_rate": 0.0005722650461041644, + "batch_size": 32, + "resolution": 1024, + "model_capacity": "narrow", + "loss_weight_ratio": 0.5121602362719397 + }, + "datetime_start": "2026-06-09T12:31:37.404846", + "datetime_complete": "2026-06-09T12:31:41.226259", + "duration_seconds": 3.821413, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 58, + "trial_id": 196, + "state": "PRUNED", + "params": { + "learning_rate": 0.000421080095182812, + "batch_size": 2, + "resolution": 256, + "model_capacity": "wide", + "loss_weight_ratio": 0.3569257328985008 + }, + "datetime_start": "2026-06-09T12:31:41.992999", + "datetime_complete": "2026-06-09T12:33:28.062304", + "duration_seconds": 106.069305, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 5, + "primary_score": 0.9897473019936772, + "primary_loss": 0.05817848326096052, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 59, + "trial_id": 197, + "state": "FAIL", + "params": { + "learning_rate": 0.0006999779382349623, + "batch_size": 64, + "resolution": 1024, + "model_capacity": "wide", + "loss_weight_ratio": 0.32626756206545826 + }, + "datetime_start": "2026-06-09T12:33:29.930419", + "datetime_complete": "2026-06-09T12:33:32.766389", + "duration_seconds": 2.83597, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 60, + "trial_id": 198, + "state": "FAIL", + "params": { + "learning_rate": 1.660719226277921e-05, + "batch_size": 16, + "resolution": 512, + "model_capacity": "wide", + "loss_weight_ratio": 0.7440646190555177 + }, + "datetime_start": "2026-06-09T12:33:33.487153", + "datetime_complete": "2026-06-09T12:33:37.287858", + "duration_seconds": 3.800705, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 61, + "trial_id": 199, + "state": "FAIL", + "params": { + "learning_rate": 1.6136763657195547e-05, + "batch_size": 16, + "resolution": 1024, + "model_capacity": "narrow", + "loss_weight_ratio": 0.7194931228626711 + }, + "datetime_start": "2026-06-09T12:33:38.041676", + "datetime_complete": "2026-06-09T12:33:40.859672", + "duration_seconds": 2.817996, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 62, + "trial_id": 200, + "state": "FAIL", + "params": { + "learning_rate": 0.00644351011993277, + "batch_size": 64, + "resolution": 512, + "model_capacity": "narrow", + "loss_weight_ratio": 0.9831121443518326 + }, + "datetime_start": "2026-06-09T12:33:41.664483", + "datetime_complete": "2026-06-09T12:33:45.502201", + "duration_seconds": 3.837718, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 63, + "trial_id": 201, + "state": "FAIL", + "params": { + "learning_rate": 8.015086163668646e-05, + "batch_size": 16, + "resolution": 512, + "model_capacity": "narrow", + "loss_weight_ratio": 0.6247208153386133 + }, + "datetime_start": "2026-06-09T12:39:52.680426", + "datetime_complete": "2026-06-09T12:42:09.573070", + "duration_seconds": 136.892644, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 64, + "trial_id": 202, + "state": "FAIL", + "params": { + "learning_rate": 0.00010798109091729242, + "batch_size": 4, + "resolution": 1024, + "model_capacity": "narrow", + "loss_weight_ratio": 0.020768120149031843 + }, + "datetime_start": "2026-06-09T12:42:09.579066", + "datetime_complete": "2026-06-09T12:44:14.995999", + "duration_seconds": 125.416933, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 65, + "trial_id": 203, + "state": "PRUNED", + "params": { + "learning_rate": 1.0514744422959588e-05, + "batch_size": 2, + "resolution": 512, + "model_capacity": "narrow", + "loss_weight_ratio": 0.11626061471968552 + }, + "datetime_start": "2026-06-09T12:58:08.785871", + "datetime_complete": "2026-06-09T12:59:45.650142", + "duration_seconds": 96.864271, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 5, + "primary_score": 0.8570720076452774, + "primary_loss": 3.444088376524076, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 66, + "trial_id": 204, + "state": "COMPLETE", + "params": { + "learning_rate": 0.0034464570787237354, + "batch_size": 4, + "resolution": 1024, + "model_capacity": "narrow", + "loss_weight_ratio": 0.6326231660481053 + }, + "datetime_start": "2026-06-09T12:59:48.671857", + "datetime_complete": "2026-06-09T13:22:07.632095", + "duration_seconds": 1338.960238, + "optuna_loss": 3.171118880123026, + "optuna_score": 0.8557610089292585, + "epoch_reached": 15, + "primary_score": 0.8557610089292585, + "primary_loss": 3.171118880123026, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 67, + "trial_id": 205, + "state": "PRUNED", + "params": { + "learning_rate": 0.000655602631496065, + "batch_size": 2, + "resolution": 512, + "model_capacity": "narrow", + "loss_weight_ratio": 0.6165970770863961 + }, + "datetime_start": "2026-06-09T13:22:09.164919", + "datetime_complete": "2026-06-09T13:23:44.873486", + "duration_seconds": 95.708567, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 5, + "primary_score": 0.8502720038631986, + "primary_loss": 3.1791283390190026, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 68, + "trial_id": 206, + "state": "PRUNED", + "params": { + "learning_rate": 0.004488241212661547, + "batch_size": 2, + "resolution": 1024, + "model_capacity": "narrow", + "loss_weight_ratio": 0.6947912315068989 + }, + "datetime_start": "2026-06-09T13:23:47.124323", + "datetime_complete": "2026-06-09T13:30:50.154021", + "duration_seconds": 423.029698, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 5, + "primary_score": 0.856514060787977, + "primary_loss": 3.1795108579885105, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 69, + "trial_id": 207, + "state": "PRUNED", + "params": { + "learning_rate": 0.001526069664046176, + "batch_size": 4, + "resolution": 512, + "model_capacity": "wide", + "loss_weight_ratio": 0.33976780125584527 + }, + "datetime_start": "2026-06-09T13:30:57.204234", + "datetime_complete": "2026-06-09T13:34:34.567629", + "duration_seconds": 217.363395, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 5, + "primary_score": 0.8513724619433332, + "primary_loss": 3.180013144569558, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 70, + "trial_id": 208, + "state": "FAIL", + "params": { + "learning_rate": 8.036481706507076e-05, + "batch_size": 8, + "resolution": 1024, + "model_capacity": "wide", + "loss_weight_ratio": 0.3172719209651951 + }, + "datetime_start": "2026-06-09T13:34:37.873728", + "datetime_complete": "2026-06-09T13:34:42.152489", + "duration_seconds": 4.278761, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 71, + "trial_id": 209, + "state": "PRUNED", + "params": { + "learning_rate": 2.0588378463008213e-05, + "batch_size": 2, + "resolution": 256, + "model_capacity": "narrow", + "loss_weight_ratio": 0.5418362964684905 + }, + "datetime_start": "2026-06-09T13:34:43.516880", + "datetime_complete": "2026-06-09T13:35:45.401603", + "duration_seconds": 61.884723, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 5, + "primary_score": 0.9735566368410664, + "primary_loss": 0.41913800601717793, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 72, + "trial_id": 210, + "state": "PRUNED", + "params": { + "learning_rate": 2.5824788703582884e-05, + "batch_size": 32, + "resolution": 256, + "model_capacity": "wide", + "loss_weight_ratio": 0.1001732434526551 + }, + "datetime_start": "2026-06-09T13:35:48.655548", + "datetime_complete": "2026-06-09T13:37:33.419525", + "duration_seconds": 104.763977, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 5, + "primary_score": 0.9432637325874148, + "primary_loss": 0.5283325079372664, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 73, + "trial_id": 211, + "state": "FAIL", + "params": { + "learning_rate": 0.0012503262438584753, + "batch_size": 64, + "resolution": 256, + "model_capacity": "wide", + "loss_weight_ratio": 0.36453885264546326 + }, + "datetime_start": "2026-06-09T13:37:36.288939", + "datetime_complete": "2026-06-09T13:37:42.360303", + "duration_seconds": 6.071364, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 74, + "trial_id": 212, + "state": "FAIL", + "params": { + "learning_rate": 0.00015582382174458584, + "batch_size": 8, + "resolution": 1024, + "model_capacity": "wide", + "loss_weight_ratio": 0.5578577099740122 + }, + "datetime_start": "2026-06-09T13:37:43.645381", + "datetime_complete": "2026-06-09T13:37:47.480475", + "duration_seconds": 3.835094, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 75, + "trial_id": 213, + "state": "FAIL", + "params": { + "learning_rate": 0.00044334124591300603, + "batch_size": 32, + "resolution": 512, + "model_capacity": "wide", + "loss_weight_ratio": 0.5292311853371128 + }, + "datetime_start": "2026-06-09T13:37:49.004989", + "datetime_complete": "2026-06-09T13:37:52.819199", + "duration_seconds": 3.81421, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 76, + "trial_id": 214, + "state": "FAIL", + "params": { + "learning_rate": 1.223245529632777e-05, + "batch_size": 4, + "resolution": 1024, + "model_capacity": "wide", + "loss_weight_ratio": 0.3226911430366707 + }, + "datetime_start": "2026-06-09T13:37:53.719425", + "datetime_complete": "2026-06-09T13:37:57.511614", + "duration_seconds": 3.792189, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 77, + "trial_id": 215, + "state": "FAIL", + "params": { + "learning_rate": 0.004344956027911343, + "batch_size": 16, + "resolution": 256, + "model_capacity": "narrow", + "loss_weight_ratio": 0.1358767915932685 + }, + "datetime_start": "2026-06-11T13:55:37.316306", + "datetime_complete": "2026-06-11T13:56:00.102813", + "duration_seconds": 22.786507, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": false, + "failure_tag": "DIVERGED", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 78, + "trial_id": 216, + "state": "FAIL", + "params": { + "learning_rate": 3.2979458231192225e-05, + "batch_size": 64, + "resolution": 1024, + "model_capacity": "wide", + "loss_weight_ratio": 0.6172976418982675 + }, + "datetime_start": "2026-06-11T13:56:01.980573", + "datetime_complete": "2026-06-11T13:56:04.807001", + "duration_seconds": 2.826428, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 79, + "trial_id": 217, + "state": "FAIL", + "params": { + "learning_rate": 0.0011426478938092145, + "batch_size": 8, + "resolution": 512, + "model_capacity": "narrow", + "loss_weight_ratio": 0.28562228694664504 + }, + "datetime_start": "2026-06-11T13:56:05.919390", + "datetime_complete": "2026-06-11T13:56:29.007012", + "duration_seconds": 23.087622, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": false, + "failure_tag": "DIVERGED", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 80, + "trial_id": 218, + "state": "FAIL", + "params": { + "learning_rate": 0.007871157298782676, + "batch_size": 16, + "resolution": 256, + "model_capacity": "narrow", + "loss_weight_ratio": 0.8290396014049095 + }, + "datetime_start": "2026-06-11T13:56:29.728292", + "datetime_complete": "2026-06-11T13:56:43.485919", + "duration_seconds": 13.757627, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": false, + "failure_tag": "DIVERGED", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 81, + "trial_id": 219, + "state": "FAIL", + "params": { + "learning_rate": 0.0002372784275397014, + "batch_size": 2, + "resolution": 1024, + "model_capacity": "narrow", + "loss_weight_ratio": 0.23184013756127309 + }, + "datetime_start": "2026-06-11T13:56:44.586990", + "datetime_complete": "2026-06-11T14:11:41.460615", + "duration_seconds": 896.873625, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 82, + "trial_id": 220, + "state": "FAIL", + "params": { + "learning_rate": 2.922133857069148e-05, + "batch_size": 8, + "resolution": 1024, + "model_capacity": "narrow", + "loss_weight_ratio": 0.23999846195349261 + }, + "datetime_start": "2026-06-11T13:58:13.608825", + "datetime_complete": "2026-06-11T13:58:17.422688", + "duration_seconds": 3.813863, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 83, + "trial_id": 221, + "state": "FAIL", + "params": { + "learning_rate": 1.0827083945741795e-05, + "batch_size": 32, + "resolution": 256, + "model_capacity": "wide", + "loss_weight_ratio": 0.9421656316488691 + }, + "datetime_start": "2026-06-11T13:58:20.165570", + "datetime_complete": "2026-06-11T13:58:45.518922", + "duration_seconds": 25.353352, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": false, + "failure_tag": "DIVERGED", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 84, + "trial_id": 222, + "state": "FAIL", + "params": { + "learning_rate": 0.0013651455341181428, + "batch_size": 8, + "resolution": 1024, + "model_capacity": "narrow", + "loss_weight_ratio": 0.04946229416405579 + }, + "datetime_start": "2026-06-11T13:58:46.714834", + "datetime_complete": "2026-06-11T13:58:52.615385", + "duration_seconds": 5.900551, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 85, + "trial_id": 223, + "state": "FAIL", + "params": { + "learning_rate": 0.0014649725391093422, + "batch_size": 2, + "resolution": 256, + "model_capacity": "narrow", + "loss_weight_ratio": 0.37873331580397984 + }, + "datetime_start": "2026-06-11T13:58:53.699844", + "datetime_complete": "2026-06-11T13:59:07.562627", + "duration_seconds": 13.862783, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": false, + "failure_tag": "DIVERGED", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 86, + "trial_id": 224, + "state": "FAIL", + "params": { + "learning_rate": 0.003206537784664529, + "batch_size": 8, + "resolution": 256, + "model_capacity": "narrow", + "loss_weight_ratio": 0.3470651170360838 + }, + "datetime_start": "2026-06-11T13:59:10.192695", + "datetime_complete": "2026-06-11T13:59:27.999212", + "duration_seconds": 17.806517, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": false, + "failure_tag": "DIVERGED", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 87, + "trial_id": 225, + "state": "FAIL", + "params": { + "learning_rate": 5.02454787833256e-05, + "batch_size": 64, + "resolution": 1024, + "model_capacity": "narrow", + "loss_weight_ratio": 0.6253427203524324 + }, + "datetime_start": "2026-06-11T13:59:30.460971", + "datetime_complete": "2026-06-11T13:59:35.019583", + "duration_seconds": 4.558612, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 88, + "trial_id": 226, + "state": "FAIL", + "params": { + "learning_rate": 0.001075351250772297, + "batch_size": 32, + "resolution": 512, + "model_capacity": "wide", + "loss_weight_ratio": 0.5672267142760922 + }, + "datetime_start": "2026-06-11T13:59:35.874493", + "datetime_complete": "2026-06-11T13:59:39.953464", + "duration_seconds": 4.078971, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 89, + "trial_id": 227, + "state": "FAIL", + "params": { + "learning_rate": 0.001655021881946475, + "batch_size": 16, + "resolution": 1024, + "model_capacity": "narrow", + "loss_weight_ratio": 0.7220736783251582 + }, + "datetime_start": "2026-06-11T13:59:41.214776", + "datetime_complete": "2026-06-11T13:59:45.710518", + "duration_seconds": 4.495742, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 90, + "trial_id": 228, + "state": "FAIL", + "params": { + "learning_rate": 2.4095794895938903e-05, + "batch_size": 4, + "resolution": 1024, + "model_capacity": "narrow", + "loss_weight_ratio": 0.9722243737456475 + }, + "datetime_start": "2026-06-11T13:59:48.610190", + "datetime_complete": "2026-06-11T14:11:41.465213", + "duration_seconds": 712.855023, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": null, + "primary_score": null, + "primary_loss": null, + "oom_triggered": null, + "failure_tag": null, + "gpu_model": null, + "max_vram_gb": null, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 91, + "trial_id": 229, + "state": "FAIL", + "params": { + "learning_rate": 1.2631887352189931e-05, + "batch_size": 64, + "resolution": 1024, + "model_capacity": "wide", + "loss_weight_ratio": 0.6756570515061726 + }, + "datetime_start": "2026-06-11T14:11:17.852541", + "datetime_complete": "2026-06-11T14:11:19.258056", + "duration_seconds": 1.405515, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 92, + "trial_id": 230, + "state": "PRUNED", + "params": { + "learning_rate": 0.0005469877422305655, + "batch_size": 32, + "resolution": 256, + "model_capacity": "narrow", + "loss_weight_ratio": 0.04014802323930022 + }, + "datetime_start": "2026-06-11T14:11:20.269944", + "datetime_complete": "2026-06-11T14:12:23.208958", + "duration_seconds": 62.939014, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 5, + "primary_score": 0.9669699647951362, + "primary_loss": 0.6271721638959168, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 93, + "trial_id": 231, + "state": "PRUNED", + "params": { + "learning_rate": 2.456301968536136e-05, + "batch_size": 64, + "resolution": 256, + "model_capacity": "narrow", + "loss_weight_ratio": 0.518993915804978 + }, + "datetime_start": "2026-06-11T14:12:25.707012", + "datetime_complete": "2026-06-11T14:13:24.007531", + "duration_seconds": 58.300519, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 5, + "primary_score": 0.6370085282990985, + "primary_loss": 0.6926188302945487, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 94, + "trial_id": 232, + "state": "PRUNED", + "params": { + "learning_rate": 0.0016827718590978113, + "batch_size": 2, + "resolution": 512, + "model_capacity": "narrow", + "loss_weight_ratio": 0.3901213232480468 + }, + "datetime_start": "2026-06-11T14:13:27.009462", + "datetime_complete": "2026-06-11T14:15:15.158395", + "duration_seconds": 108.148933, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 5, + "primary_score": 0.8561622114054467, + "primary_loss": 3.172988403698563, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 95, + "trial_id": 233, + "state": "FAIL", + "params": { + "learning_rate": 0.00042245361689695283, + "batch_size": 64, + "resolution": 512, + "model_capacity": "narrow", + "loss_weight_ratio": 0.09698161427820617 + }, + "datetime_start": "2026-06-11T14:15:19.053044", + "datetime_complete": "2026-06-11T14:15:21.502024", + "duration_seconds": 2.44898, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 96, + "trial_id": 234, + "state": "PRUNED", + "params": { + "learning_rate": 0.0010085301314243155, + "batch_size": 4, + "resolution": 256, + "model_capacity": "wide", + "loss_weight_ratio": 0.6825227303515402 + }, + "datetime_start": "2026-06-11T14:15:22.979716", + "datetime_complete": "2026-06-11T14:17:09.701119", + "duration_seconds": 106.721403, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 5, + "primary_score": 0.9848687559506278, + "primary_loss": 0.07858677170699156, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 97, + "trial_id": 235, + "state": "FAIL", + "params": { + "learning_rate": 2.7283826334314898e-05, + "batch_size": 64, + "resolution": 256, + "model_capacity": "wide", + "loss_weight_ratio": 0.29036664563960757 + }, + "datetime_start": "2026-06-11T14:17:12.341718", + "datetime_complete": "2026-06-11T14:17:16.485723", + "duration_seconds": 4.144005, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 98, + "trial_id": 236, + "state": "PRUNED", + "params": { + "learning_rate": 0.0005765481949314836, + "batch_size": 8, + "resolution": 512, + "model_capacity": "narrow", + "loss_weight_ratio": 0.7465559395680328 + }, + "datetime_start": "2026-06-11T14:17:17.619303", + "datetime_complete": "2026-06-11T14:18:59.380148", + "duration_seconds": 101.760845, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 5, + "primary_score": 0.8532715976286795, + "primary_loss": 3.296048546642191, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 99, + "trial_id": 237, + "state": "PRUNED", + "params": { + "learning_rate": 0.0013883083869791772, + "batch_size": 64, + "resolution": 256, + "model_capacity": "narrow", + "loss_weight_ratio": 0.9210913786629441 + }, + "datetime_start": "2026-06-11T14:19:03.614194", + "datetime_complete": "2026-06-11T14:20:03.443717", + "duration_seconds": 59.829523, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 5, + "primary_score": 0.8453963661432513, + "primary_loss": 1.3442585114939807, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 100, + "trial_id": 238, + "state": "FAIL", + "params": { + "learning_rate": 0.00013018682232310546, + "batch_size": 32, + "resolution": 512, + "model_capacity": "narrow", + "loss_weight_ratio": 0.9347399927933793 + }, + "datetime_start": "2026-06-11T14:20:06.042537", + "datetime_complete": "2026-06-11T14:20:08.893357", + "duration_seconds": 2.85082, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 101, + "trial_id": 239, + "state": "PRUNED", + "params": { + "learning_rate": 1.0172045518922102e-05, + "batch_size": 8, + "resolution": 512, + "model_capacity": "wide", + "loss_weight_ratio": 0.005662841475867597 + }, + "datetime_start": "2026-06-11T14:20:09.560118", + "datetime_complete": "2026-06-11T14:23:51.931933", + "duration_seconds": 222.371815, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 5, + "primary_score": 0.8545924746829634, + "primary_loss": 3.441398318809799, + "oom_triggered": false, + "failure_tag": null, + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + }, + { + "trial_number": 102, + "trial_id": 240, + "state": "FAIL", + "params": { + "learning_rate": 0.00010302359944750134, + "batch_size": 4, + "resolution": 1024, + "model_capacity": "wide", + "loss_weight_ratio": 0.7977654326079116 + }, + "datetime_start": "2026-06-11T14:23:53.527265", + "datetime_complete": "2026-06-11T14:23:55.897358", + "duration_seconds": 2.370093, + "optuna_loss": null, + "optuna_score": null, + "epoch_reached": 0, + "primary_score": 0.0, + "primary_loss": 999.0, + "oom_triggered": true, + "failure_tag": "OOM", + "gpu_model": "NVIDIA L4", + "max_vram_gb": 22.0343017578125, + "health_tier": null, + "health_reason": null, + "git_commit": null, + "dataset_version": null + } +] \ No newline at end of file diff --git a/colab_worker.py b/archive/colab_worker.py similarity index 100% rename from colab_worker.py rename to archive/colab_worker.py diff --git a/archive/extract_bridge_crack_study.py b/archive/extract_bridge_crack_study.py new file mode 100644 index 0000000..251f23f --- /dev/null +++ b/archive/extract_bridge_crack_study.py @@ -0,0 +1,117 @@ +import optuna +import json +import os +import datetime +from sqlalchemy.orm import Session +from src.db_manager import engine, SessionLocal +from src.schema import TrialResult + +def extract_study_data(): + study_name = "bridge_crack_study" + print(f"Loading Optuna study '{study_name}'...") + + # 1. Load the Optuna study + db_url = os.getenv("HPO_DATABASE_URL", "sqlite:///hpo_studies.db") + try: + study = optuna.load_study(study_name=study_name, storage=db_url) + except KeyError: + print(f"Error: Study '{study_name}' not found in database.") + return + + print(f"Loading Pathfinder-specific trial results for '{study_name}'...") + # 2. Query Pathfinder TrialResult metadata from the DB + pathfinder_results = {} + with SessionLocal() as session: + results = session.query(TrialResult).filter_by(study_name=study_name).all() + for r in results: + pathfinder_results[r.trial_id] = r.to_dict() + + print(f"Aggregating {len(study.trials)} trials...") + # 3. Combine Optuna trials with Pathfinder trial results + all_trials_data = [] + + for t in study.trials: + # Get basic Optuna trial info + trial_number = t.number + trial_id = t._trial_id + state = t.state.name + params = t.params + datetime_start = t.datetime_start.isoformat() if t.datetime_start else None + datetime_complete = t.datetime_complete.isoformat() if t.datetime_complete else None + duration = t.duration.total_seconds() if t.duration else None + values = t.values # This is a list/tuple of objective values + + # Primary score and loss from values or parameters + # In multi-objective: direction minimize (loss/obj 0), maximize (score/obj 1) + optuna_loss = values[0] if values and len(values) > 0 else None + optuna_score = values[1] if values and len(values) > 1 else None + + # Get Pathfinder metadata if available using _trial_id + pf_info = pathfinder_results.get(trial_id, {}) + + # Merge data + trial_record = { + "trial_number": trial_number, + "trial_id": trial_id, + "state": state, + "params": params, + "datetime_start": datetime_start, + "datetime_complete": datetime_complete, + "duration_seconds": duration, + "optuna_loss": optuna_loss, + "optuna_score": optuna_score, + "epoch_reached": pf_info.get("epoch_reached"), + "primary_score": pf_info.get("primary_score"), + "primary_loss": pf_info.get("primary_loss"), + "oom_triggered": pf_info.get("oom_triggered"), + "failure_tag": pf_info.get("failure_tag"), + "gpu_model": pf_info.get("gpu_model"), + "max_vram_gb": pf_info.get("max_vram_gb"), + "health_tier": pf_info.get("health_tier"), + "health_reason": pf_info.get("health_reason"), + "git_commit": pf_info.get("git_commit"), + "dataset_version": pf_info.get("dataset_version"), + } + all_trials_data.append(trial_record) + + # Sort by trial_number + all_trials_data.sort(key=lambda x: x["trial_number"]) + + # Save to JSON + output_path = "bridge_crack_study_trials.json" + with open(output_path, "w") as f: + json.dump(all_trials_data, f, indent=2) + print(f"Successfully saved all trials data to '{output_path}'.") + + # Filter completed trials to show progress + completed_trials = [t for t in all_trials_data if t["state"] == "COMPLETE"] + print(f"Total trials: {len(all_trials_data)}") + print(f"Completed trials: {len(completed_trials)}") + + if not completed_trials: + print("No completed trials found to evaluate improvement.") + return + + # Find start trials vs best trials + first_completed = completed_trials[:3] + best_completed = sorted(completed_trials, key=lambda x: x["primary_score"] or 0, reverse=True)[:3] + + print("\n--- FIRST COMPLETED TRIALS ---") + for t in first_completed: + print(f"Trial #{t['trial_number']} (ID={t['trial_id']}): Score={t['primary_score']}, Loss={t['primary_loss']}, Params={t['params']}") + + print("\n--- BEST COMPLETED TRIALS ---") + for t in best_completed: + print(f"Trial #{t['trial_number']} (ID={t['trial_id']}): Score={t['primary_score']}, Loss={t['primary_loss']}, Params={t['params']}") + + initial_score = first_completed[0]['primary_score'] if first_completed else None + best_score = best_completed[0]['primary_score'] if best_completed else None + + if initial_score is not None and best_score is not None: + diff = best_score - initial_score + print(f"\nImprovement in Best Score: {initial_score} -> {best_score} (Gain: +{diff:.6f})") + else: + print("\nCould not calculate improvement due to missing scores.") + +if __name__ == "__main__": + extract_study_data() diff --git a/archive/extract_to_csv.py b/archive/extract_to_csv.py new file mode 100644 index 0000000..47c194f --- /dev/null +++ b/archive/extract_to_csv.py @@ -0,0 +1,99 @@ +import optuna +import csv +import os +from sqlalchemy.orm import Session +from src.db_manager import SessionLocal +from src.schema import TrialResult + +def extract_filtered_csv(): + study_name = "bridge_crack_study" + print(f"Loading study '{study_name}'...") + + db_url = os.getenv("HPO_DATABASE_URL", "sqlite:///hpo_studies.db") + try: + study = optuna.load_study(study_name=study_name, storage=db_url) + except KeyError: + print(f"Error: Study '{study_name}' not found.") + return + + # Load Pathfinder metadata + pathfinder_results = {} + with SessionLocal() as session: + results = session.query(TrialResult).filter_by(study_name=study_name).all() + for r in results: + pathfinder_results[r.trial_id] = r.to_dict() + + # Filter and flatten completed trials with resolution >= 500 + rows = [] + headers = [ + "trial_number", "trial_id", "state", "score", "loss", + "duration_seconds", "epoch_reached", "learning_rate", + "batch_size", "resolution", "encoder_name", "loss_weight_ratio", + "model_capacity", "gpu_model", "max_vram_gb" + ] + + for t in study.trials: + if t.state.name != "COMPLETE": + continue + + params = t.params + res = params.get("resolution") + if res is None or res < 500: + continue + + trial_id = t._trial_id + pf_info = pathfinder_results.get(trial_id, {}) + + # Primary score and loss from Optuna values or Pathfinder + values = t.values + optuna_loss = values[0] if values and len(values) > 0 else None + optuna_score = values[1] if values and len(values) > 1 else None + + score = pf_info.get("primary_score") or optuna_score + loss = pf_info.get("primary_loss") or optuna_loss + + row = { + "trial_number": t.number, + "trial_id": trial_id, + "state": t.state.name, + "score": score, + "loss": loss, + "duration_seconds": t.duration.total_seconds() if t.duration else None, + "epoch_reached": pf_info.get("epoch_reached"), + "learning_rate": params.get("learning_rate"), + "batch_size": params.get("batch_size"), + "resolution": res, + "encoder_name": params.get("encoder_name"), + "loss_weight_ratio": params.get("loss_weight_ratio"), + "model_capacity": params.get("model_capacity", "N/A"), + "gpu_model": pf_info.get("gpu_model"), + "max_vram_gb": pf_info.get("max_vram_gb") + } + rows.append(row) + + # Sort by trial number + rows.sort(key=lambda x: x["trial_number"]) + + # Save to CSV + csv_file = "bridge_crack_study_500px.csv" + with open(csv_file, mode="w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=headers) + writer.writeheader() + for r in rows: + writer.writerow(r) + + print(f"Successfully extracted {len(rows)} trials to '{csv_file}'.") + + if len(rows) > 0: + first = rows[0] + best = max(rows, key=lambda x: x["score"] if x["score"] is not None else 0) + print("\n--- RESULTS OVERVIEW (Resolution >= 500px) ---") + print(f"First Trial #{first['trial_number']}: Score={first['score']}, Loss={first['loss']}, Params={first['learning_rate'], first['batch_size'], first['resolution']}") + print(f"Best Trial #{best['trial_number']}: Score={best['score']}, Loss={best['loss']}, Params={best['learning_rate'], best['batch_size'], best['resolution']}") + if first['score'] is not None and best['score'] is not None: + print(f"Improvement: {first['score']} -> {best['score']} (Gain: +{best['score'] - first['score']:.6f})") + else: + print("No completed trials matching criteria found.") + +if __name__ == "__main__": + extract_filtered_csv() diff --git a/archive/plot_results.py b/archive/plot_results.py new file mode 100644 index 0000000..efa3168 --- /dev/null +++ b/archive/plot_results.py @@ -0,0 +1,93 @@ +import os +import sys + +# Ensure pandas and matplotlib are installed +try: + import pandas as pd + import matplotlib.pyplot as plt +except ImportError: + print("Required packages (pandas, matplotlib) are missing.") + print("Installing packages...") + import subprocess + subprocess.check_call([sys.executable, "-m", "pip", "install", "pandas", "matplotlib"]) + import pandas as pd + import matplotlib.pyplot as plt + +def plot_study_results(): + csv_file = "bridge_crack_study_500px.csv" + if not os.path.exists(csv_file): + print(f"Error: {csv_file} not found. Run extract_to_csv.py first.") + return + + # Load data + df = pd.read_csv(csv_file) + print("Loaded data:") + print(df[["trial_number", "score", "loss", "learning_rate", "batch_size", "resolution"]]) + + if len(df) == 0: + print("No completed trials to plot.") + return + + # Modern styling + plt.style.use("seaborn-v0_8-whitegrid" if "seaborn-v0_8-whitegrid" in plt.style.available else "default") + fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 6)) + + # Color definitions + accent_color = "#3B82F6" # Premium Indigo/Blue + highlight_color = "#10B981" # Emerald Green for best + neutral_dark = "#1F2937" + + # Plot 1: Score Progression over completed trials + ax1.plot(df["trial_number"], df["score"], marker="o", color=accent_color, linewidth=2, markersize=8, label="Dice Score") + + # Highlight best trial + best_idx = df["score"].idxmax() + best_trial = df.loc[best_idx] + ax1.scatter(best_trial["trial_number"], best_trial["score"], color=highlight_color, s=200, zorder=5, label=f"Best (Trial #{int(best_trial['trial_number'])}: {best_trial['score']:.4f})") + + # Labels and Titles + ax1.set_title("Dice Score Progression (Resolution >= 500px)", fontsize=14, fontweight="bold", pad=15, color=neutral_dark) + ax1.set_xlabel("Optuna Trial Number", fontsize=12, labelpad=10) + ax1.set_ylabel("Dice Score (Higher is Better)", fontsize=12, labelpad=10) + ax1.legend(loc="lower right", frameon=True, facecolor="white", edgecolor="#E5E7EB") + ax1.set_ylim(df["score"].min() - 0.005, df["score"].max() + 0.005) + + # Plot 2: Learning Rate vs Score colored by Batch Size + scatter = ax2.scatter( + df["learning_rate"], + df["score"], + c=df["batch_size"], + cmap="viridis", + s=120, + edgecolors="none", + alpha=0.85 + ) + # Highlight best trial in scatter + ax2.scatter( + best_trial["learning_rate"], + best_trial["score"], + color=highlight_color, + edgecolors="black", + s=250, + zorder=5, + label="Best Model" + ) + + ax2.set_xscale("log") + ax2.set_title("Learning Rate vs. Score (Size: Batch Size)", fontsize=14, fontweight="bold", pad=15, color=neutral_dark) + ax2.set_xlabel("Learning Rate (Log Scale)", fontsize=12, labelpad=10) + ax2.set_ylabel("Dice Score", fontsize=12, labelpad=10) + + # Colorbar for Batch Size + cbar = plt.colorbar(scatter, ax=ax2) + cbar.set_label("Batch Size", fontsize=11, rotation=270, labelpad=15) + ax2.legend(loc="lower left", frameon=True, facecolor="white", edgecolor="#E5E7EB") + + plt.tight_layout() + plot_path = "bridge_crack_500px_plots.png" + plt.savefig(plot_path, dpi=300, facecolor="white") + print(f"Successfully generated and saved plots to '{plot_path}'.") + plt.close() + +if __name__ == "__main__": + plot_study_results() diff --git a/broker.py b/broker.py index 22208de..95a99a3 100644 --- a/broker.py +++ b/broker.py @@ -1,7 +1,6 @@ import hmac import os from contextlib import asynccontextmanager -from typing import List from fastapi import FastAPI, Request from fastapi.responses import JSONResponse from fastapi.middleware.cors import CORSMiddleware diff --git a/docker-compose.yml b/docker-compose.yml index 1b295a8..20dfa05 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,10 +2,13 @@ services: broker: build: . + restart: unless-stopped ports: - "8000:8000" - volumes: - - ./hpo_studies.db:/app/hpo_studies.db environment: - HPO_DATABASE_URL=sqlite:///hpo_studies.db - HPO_DEBUG=0 + # NOTE: the database is ephemeral by default (inside the container). + # For persistent storage, mount a volume at /app and pre-create hpo_studies.db: + # volumes: + # - ./data:/app diff --git a/docs/INTEGRATION.md b/docs/INTEGRATION.md index 76e9405..9bc2171 100644 --- a/docs/INTEGRATION.md +++ b/docs/INTEGRATION.md @@ -3,9 +3,9 @@ A human quickstart for wiring your own model to the broker. For the agent-driven version of this (have your IDE assistant do it), see [`AGENTS.md`](../AGENTS.md). -The root `colab_worker.py` is the full bridge-crack U-Net reference implementation. You do -**not** need to fork it. Cloners start from `templates/worker_minimal.py` and the 3-call -client in `hpo_client.py`. +The reference crack-seg implementation (`colab_worker.py`) lives in [`archive/`](../archive/). You +do **not** need to fork it. Cloners start from `templates/worker_minimal.py` and the 3-call +client in `src/hpo_client.py`. ## 1. Install @@ -17,7 +17,7 @@ pip install -r requirements.txt ## 2. Run the broker -**Local only** (simulator on the same machine — Colab cannot reach this): +**Local only** (worker on the same machine — Colab cannot reach this): ```bash source .venv/bin/activate @@ -37,24 +37,19 @@ Save that token and use the **same** value in three places: | Where | How | |-------|-----| | Colab / worker | `os.environ["HPO_SECRET_TOKEN"] = "…"` — `TrialSession` sends `X-HPO-Token` | -| Dashboard | First visit to the ngrok URL prompts once; stored in a session cookie | +| Dashboard | First visit to the tunnel URL prompts once; stored in a session cookie | | CLI / MCP | `export HPO_SECRET_TOKEN=…` when tools hit the tunneled broker | -Worker downloads (`/colab_worker.py`, `/hpo_client.py`) also require the token header when -auth is on. The dashboard **Worker Setup** tab explains this and generates copy-paste snippets. +Worker downloads also require the token header when auth is on. The dashboard **Worker Setup** tab +generates copy-paste snippets. -## 3. Configure MCP (optional, for agent assistance) +## 3. Define your search space via a Manifest -Point your IDE's MCP config at `pathfinder` (`python hpo_mcp_server.py`). The same -config works in Cursor, Antigravity, and Claude Code - see the README -"Exposing Pathfinder to AI Agents" section. Set `HPO_BROKER_URL` in the MCP env if you want -`validate_integration` to check the live broker. - -## 4. Define your search space via a Manifest - -Pathfinder uses a manifest-based onboarding system. You define your study config, search space parameters, objectives, and training command in a single YAML file (e.g. `train.hpo.yaml`). +Pathfinder uses a manifest-based onboarding system. You define your study config, search space +parameters, objectives, and training command in a single YAML file (e.g. `train.hpo.yaml`). To set up a study: + 1. **Create the manifest file**: Write a YAML file based on `templates/manifest.template.yaml`. 2. **Validate the manifest**: - **CLI**: `python hpo_cli.py validate train.hpo.yaml` @@ -65,9 +60,10 @@ To set up a study: - **MCP**: Use the `init_from_manifest` tool. - **Dashboard**: Click **Initialize Study** after validating. -This stores configuration in the database and creates the Optuna study. The database configuration keeps all studies isolated. +This stores configuration in the database and creates the Optuna study. The database +configuration keeps all studies isolated. -## 5. The worker contract (three calls) +## 4. The worker contract (three calls) Set environment variables on the machine that trains: @@ -77,10 +73,10 @@ export HPO_STUDY_NAME="my_study" export HPO_SPARKLINES=1 # Optional: prints a Unicode curve on completion ``` -Then use `hpo_client.TrialSession`: +Then use `src.hpo_client.TrialSession`: ```python -from hpo_client import TrialSession +from src.hpo_client import TrialSession import sys # 1. Detect GPU telemetry @@ -100,24 +96,25 @@ trial = session.suggest() # -> {trial_id, trial_number, params} pruned = False oom_triggered = False last_epoch = 0 -dice, bce = 0.0, 0.0 +score, loss = 0.0, 0.0 try: for epoch in range(num_epochs): last_epoch = epoch - dice, bce = train_one_epoch(trial["params"]) # your training step - if session.report_epoch(epoch, dice, bce): # True => broker says prune + score, loss = train_one_epoch(trial["params"]) # your training step + if session.report_epoch(epoch, score=score, loss=loss): # True => broker says prune session.complete( - epoch, dice, bce, state="PRUNED", + epoch, score=score, loss=loss, state="PRUNED", gpu_model=gpu_model, max_vram_gb=max_vram_gb, oom_triggered=False ) pruned = True break except Exception as exc: # 2. Catch and report Out Of Memory (OOM) failures - if "out of memory" in str(exc).lower(): + exc_str = str(exc).lower() + if "out of memory" in exc_str or "oom" in exc_str: session.complete( - last_epoch, dice, bce, state="FAIL", + last_epoch, score=score, loss=loss, state="FAIL", gpu_model=gpu_model, max_vram_gb=max_vram_gb, oom_triggered=True ) print("Trial failed due to GPU OOM.") @@ -127,7 +124,7 @@ except Exception as exc: if not pruned: session.complete( - last_epoch, dice, bce, weights_path="model.pt", state="COMPLETE", + last_epoch, score=score, loss=loss, weights_path="model.pt", state="COMPLETE", gpu_model=gpu_model, max_vram_gb=max_vram_gb, oom_triggered=False ) ``` @@ -137,54 +134,31 @@ That is the entire contract: | Call | Endpoint | Purpose | |------|----------|---------| | `session.suggest()` | `POST /api/suggest_trial` | Get the next trial's hyperparameters | -| `session.report_epoch(epoch, dice, bce, ...)` | `POST /api/report_epoch` | Log an epoch; returns `should_prune` | -| `session.complete(epoch, dice, bce, ..., gpu_model, max_vram_gb, oom_triggered)` | `POST /api/complete_trial` | Finalize (COMPLETE / PRUNED / FAIL) with hardware telemetry | - -`templates/worker_minimal.py` is a ~65-line starting point - fill in `train_one_epoch`. - -### Google Colab (bridge-crack reference) - -The root `colab_worker.py` is **only** for the bridge-crack U-Net study. Cloners use -`templates/worker_minimal.py` above — do not fork `colab_worker.py`. - -| Entrypoint | When to use | -|------------|-------------| -| `train_colab_trial(study_name, epochs=15)` | One trial (smoke test or debugging) | -| `train_colab_trial_loop(study_name, n_trials=12, epochs=15)` | Normal Colab session (default) | - -`train_colab_trial_loop` repeatedly calls `train_colab_trial`. Each iteration: - -- Reports guardrail skips and caught OOM/crashes as `FAIL` to the broker (loop continues). -- Clears the CUDA cache between trials. -- Retries transient `suggest_trial` errors (via `TrialSession` backoff). - -If the Colab kernel hard-crashes without running Python cleanup, the broker marks the trial -`FAIL` after the worker lease expires (~45s without a heartbeat) or on the next dashboard poll. - -```python -import os -os.environ["HPO_BROKER_URL"] = "https://" -os.environ["HPO_SECRET_TOKEN"] = "" # required when --tunnel is on - -from colab_worker import train_colab_trial_loop -train_colab_trial_loop("bridge_crack_study", n_trials=12, epochs=15) -``` +| `session.report_epoch(epoch, score=score, loss=loss, ...)` | `POST /api/report_epoch` | Log an epoch; returns `should_prune` | +| `session.complete(epoch, score=score, loss=loss, ..., gpu_model, max_vram_gb, oom_triggered)` | `POST /api/complete_trial` | Finalize (COMPLETE / PRUNED / FAIL) with hardware telemetry | -The dashboard **Worker Setup → Google Colab Integration** tab generates the full -download-and-run snippet (including authenticated fetches of `colab_worker.py`). +`templates/worker_minimal.py` is a ~60-line starting point — fill in `train_one_epoch`. > [!NOTE] > **Abstracting Metrics (Non-CV Tasks)** -> The parameter names `dice` (higher-is-better) and `bce` (lower-is-better) are abstract placeholders in the API. If you are doing NLP (e.g. Perplexity and BLEU), RL (e.g. Reward and Episode Length), or Tabular tasks: -> - Pass your higher-is-better metric (e.g. Accuracy, BLEU, F1, Reward) as `dice`. -> - Pass your lower-is-better metric (e.g. Loss, Perplexity, MAE) as `bce`. -> - You can customize their display names on the UI dashboard under **Settings > Eval protocol** by setting "Loss metric display name" and "Score metric display name" (they default to BCE and Dice). +> The parameter names `score` (higher-is-better) and `loss` (lower-is-better) are generic slots in the API. If you are doing NLP (e.g. Perplexity and BLEU), RL (e.g. Reward and Episode Length), or Tabular tasks: +> - Pass your higher-is-better metric (e.g. Accuracy, BLEU, F1, Reward) as `score`. +> - Pass your lower-is-better metric (e.g. Cross-Entropy, Perplexity, MAE) as `loss`. +> - You can customize their display names on the UI dashboard under **Settings > Eval protocol** by setting "Loss metric display name" and "Score metric display name". + +### Google Colab (reference only) -## 6. Create the study and validate +The historical bridge-crack U-Net reference implementation is preserved in +[`archive/colab_worker.py`](../archive/colab_worker.py). **Do not use it for new studies.** +Cloners should use `templates/worker_minimal.py`. -Call the MCP `init_from_manifest` tool (or CLI `init`) to create the Optuna study and seed configuration options from your manifest file. Use the `/health` broker endpoint to verify connectivity. +## 5. Create the study and validate -## 7. CLI operations +Call the MCP `init_from_manifest` tool (or CLI `init`) to create the Optuna study and seed +configuration options from your manifest file. Use the `/health` broker endpoint to verify +connectivity. + +## 6. CLI operations Pathfinder ships a command-line interface (`hpo_cli.py`) for database operations. @@ -194,7 +168,8 @@ Pathfinder ships a command-line interface (`hpo_cli.py`) for database operations python hpo_cli.py validate train.hpo.yaml ``` -Parses and validates the YAML manifest against the Pathfinder schema. Reports errors and warnings without touching the database. +Parses and validates the YAML manifest against the Pathfinder schema. Reports errors and +warnings without touching the database. ### Initialize a study from manifest @@ -210,7 +185,8 @@ python hpo_cli.py export my_study --output my_study.json python hpo_cli.py export my_study --format csv --output my_study.csv ``` -Exports the full study — Optuna trials, trial results, reviews, agent logs — into a portable JSON or CSV file. Useful for archiving, sharing, or migrating between machines. +Exports the full study — Optuna trials, trial results, reviews, agent logs — into a portable +JSON or CSV file. Useful for archiving, sharing, or migrating between machines. ### Import a study @@ -220,7 +196,26 @@ python hpo_cli.py import my_study.json --rename new_study_name python hpo_cli.py import my_study.json --rename new_study_name --force # overwrite if exists ``` -Imports a study from a previously exported JSON file. Any trials that were `RUNNING` at export time are automatically converted to `FAIL` so they do not appear as zombie trials. If the import fails partway through (e.g. corrupted data), the entire import is rolled back atomically — no orphan rows are left. +Imports a study from a previously exported JSON file. Any trials that were `RUNNING` at export +time are automatically converted to `FAIL` so they do not appear as zombie trials. If the import +fails partway through, the entire import is rolled back atomically. + +### Generate a model card + +```bash +python hpo_cli.py modelcard my_study +``` + +Writes `MODEL_CARD.md` to disk with a synthesis of the study's results, best hyperparameters, +and importance rankings. + +### Delete a study + +```bash +python hpo_cli.py delete my_study +``` + +Permanently removes a study and all its data from the database. Requires confirmation. ### Backup the database @@ -228,22 +223,29 @@ Imports a study from a previously exported JSON file. Any trials that were `RUNN python hpo_cli.py backup --output backup.db ``` -Creates a point-in-time snapshot of the full SQLite database (`hpo_studies.db`) using SQLite's online backup API. Safe to run while the broker is running. +Creates a point-in-time snapshot of the full SQLite database using SQLite's online backup API. +Safe to run while the broker is running. -## 8. Environment variables +## 7. Environment variables | Variable | Default | Description | -|---|---|---| +|---|---|---|---| +| `HPO_DATABASE_URL` | `sqlite:///hpo_studies.db` | SQLite connection string. | | `HPO_BROKER_URL` | `http://localhost:8000` | URL the worker uses to reach the broker. | | `HPO_STUDY_NAME` | *(none)* | Default study name when not passed explicitly. | | `HPO_SECRET_TOKEN` | *(none)* | Bearer token required when `--tunnel` auth is enabled. | +| `HPO_DEBUG` | `0` | Set to `1` to enable verbose debug logging in the broker. | | `HPO_SPARKLINES` | `0` | Set to `1` to print a Unicode training curve on trial completion. | -| `HPO_BACKUP_ON_START` | `0` | Set to `1` to run an automatic database backup when the broker starts. Equivalent to `--backup-on-start` CLI flag. | -| `HPO_CAPTURE_FULL_ENV` | `0` | Set to `1` to capture the full `pip freeze` output rather than the default ML-library whitelist. Useful for exact reproducibility audits. | +| `HPO_BACKUP_ON_START` | `0` | Set to `1` to run an automatic database backup when the broker starts. | +| `HPO_CAPTURE_FULL_ENV` | `0` | Set to `1` to capture the full `pip freeze` output rather than the default ML-library whitelist. | +| `HPO_TUNNEL_PROVIDER` | *(none)* | Tunnel provider for remote access: `ngrok` or `cloudflare`. | +| `HPO_TUNNEL_URL` | *(none)* | Static tunnel URL when using `cloudflare` provider. | +| `HPO_ALLOWED_ORIGINS` | *(none)* | Additional CORS origins (comma-separated) for the dashboard. | -## 9. Validation guardrails schema +## 8. Validation guardrails schema -`validation_rules` can be set in the manifest YAML or via **Settings → Eval protocol → Metric guardrails** in the dashboard. +`validation_rules` can be set in the manifest YAML or via **Settings > Eval protocol > +Metric guardrails** in the dashboard. ```yaml validation_rules: @@ -253,11 +255,15 @@ validation_rules: max_epoch_jump: 0.5 # warn when score changes by more than this fraction between consecutive epochs ``` -When `enabled: false` (the default for new studies), no metric warnings are ever generated. Set `enabled: true` only when you have domain knowledge about valid metric ranges for your task. +When `enabled: false` (the default for new studies), no metric warnings are ever generated. +Set `enabled: true` only when you have domain knowledge about valid metric ranges for your task. -When a trial triggers a guardrail it is flagged as **Watch** in the study health, but the trial is still recorded — guardrails are advisory, not blocking (the only hard rejection is when *both* metrics are exactly `0.0`, history is empty, and `epoch ≤ 0` on a multi-objective study, which strongly indicates training never ran). +When a trial triggers a guardrail it is flagged as **Watch** in the study health, but the +trial is still recorded — guardrails are advisory, not blocking (the only hard rejection is +when *both* metrics are exactly `0.0`, history is empty, and `epoch ≤ 0` on a multi-objective +study, which strongly indicates training never ran). ## Next steps - Open the dashboard (`index.html` served via the broker root) to watch trials, the Pareto front, and fANOVA importance. -- Use the episodic coordinator review (see [`AGENTS.md`](../AGENTS.md)) to interpret results and adjust the search space with evidence. +- Use the inspection flow (see [`AGENTS.md`](../AGENTS.md)) to interpret results with your IDE agent. diff --git a/hpo_cli.py b/hpo_cli.py index 08017b1..5a0104f 100644 --- a/hpo_cli.py +++ b/hpo_cli.py @@ -1,17 +1,17 @@ #!/usr/bin/env python3 """Decoupled Control CLI for Pathfinder. -Provides standalone commands to check status, run reviews, and manage pending search space patches. +Provides standalone commands to check status, manage pending search space patches, +export/import studies, generate model cards, and delete studies. """ import os import sys import json import argparse -import requests import csv import sqlite3 import datetime -from typing import Dict, Any, Optional +from datetime import timezone import optuna # Make sure we can import from workspace root and src @@ -21,298 +21,46 @@ from src.schema import ( StudyStatus, SystemConfiguration, - StudyReview, TrialResult, TrialMetadata, CompactedPacket, StudyCard, - AgentReasoningLog, - InvalidProposal, TrialLease, - CoordinatorMetric, - SuggestMetric, ) -from src.hpo_coordinator import ( - compute_review_heuristics, - build_review_prompt, - save_study_review, - count_evaluated_trials, - validate_review_fields, - mark_review_applied, - flag_study_review, -) -from src.hpo_config import load_hpo_config -from src.suggest import get_or_create_study, load_study, _enqueue_manual_trial -from src.search_space import load_search_space, _apply_search_space_patch +from src.health import compute_health_tier, count_evaluated_trials +from src.analytics import build_study_packet +from src.suggest import load_study -DEFAULT_STUDY = "bridge_crack_study" +DEFAULT_STUDY = None def get_study_name(args) -> str: """Resolve study name from args, env, or default.""" - return args.study or os.getenv("HPO_STUDY_NAME") or DEFAULT_STUDY - -def call_llm(prompt: str) -> str: - """Call local LLM APIs directly using requests to avoid heavy client dependencies.""" - gemini_key = os.getenv("GEMINI_API_KEY") - anthropic_key = os.getenv("ANTHROPIC_API_KEY") - openai_key = os.getenv("OPENAI_API_KEY") - - if gemini_key: - url = f"https://generativelanguage.googleapis.com/v1beta/models/gemini-1.5-pro:generateContent?key={gemini_key}" - headers = {"Content-Type": "application/json"} - payload = { - "contents": [{"parts": [{"text": prompt}]}], - "generationConfig": { - "responseMimeType": "application/json" - } - } - res = requests.post(url, json=payload, headers=headers, timeout=120) - res.raise_for_status() - data = res.json() - return data["candidates"][0]["content"]["parts"][0]["text"] - - elif anthropic_key: - url = "https://api.anthropic.com/v1/messages" - headers = { - "x-api-key": anthropic_key, - "anthropic-version": "2023-06-01", - "content-type": "application/json" - } - payload = { - "model": "claude-3-5-sonnet-20241022", - "max_tokens": 4096, - "messages": [{"role": "user", "content": prompt}], - "system": "You are a professional ML experiment optimization coordinator. You MUST return JSON only." - } - res = requests.post(url, json=payload, headers=headers, timeout=120) - res.raise_for_status() - data = res.json() - return data["content"][0]["text"] - - elif openai_key: - url = "https://api.openai.com/v1/chat/completions" - headers = { - "Authorization": f"Bearer {openai_key}", - "Content-Type": "application/json" - } - payload = { - "model": "gpt-4o", - "response_format": {"type": "json_object"}, - "messages": [ - {"role": "system", "content": "You are a professional ML experiment optimization coordinator. You MUST return JSON only."}, - {"role": "user", "content": prompt} - ] - } - res = requests.post(url, json=payload, headers=headers, timeout=120) - res.raise_for_status() - data = res.json() - return data["choices"][0]["message"]["content"] - - else: - raise ValueError("No API keys found for GEMINI_API_KEY, ANTHROPIC_API_KEY, or OPENAI_API_KEY.") + name = args.study or os.getenv("HPO_STUDY_NAME") + if not name: + print("No study specified. Set HPO_STUDY_NAME or pass --study.") + sys.exit(1) + return name def cmd_status(args): init_db() study_name = get_study_name(args) try: - study = get_or_create_study(study_name) + study = load_study(study_name) except Exception as e: print(f"Error loading study '{study_name}': {e}") sys.exit(1) - from src.hpo_coordinator import study_eval_insights - hpo_config = load_hpo_config(study_name) - insights = study_eval_insights(study, hpo_config) - heuristics = compute_review_heuristics(study, insights, hpo_config, study_name) + health_tier, health_reason = compute_health_tier(study, study_name) - print(f"\n==================================================") + print("\n==================================================") print(f"📊 STUDY STATUS: {study_name}") - print(f"==================================================") + print("==================================================") print(f"Total Trials: {len(study.trials)}") - print(f"Evaluated: {heuristics['trials_evaluated']}") - print(f"Health Tier: {heuristics['health_tier'].upper()}") - print(f"Health Reason: {heuristics['health_reason']}") - print(f"Review Recommended: {heuristics['review_recommended']}") - print(f"Already Dismissed: {heuristics.get('already_dismissed', False)}") - - # Check pending changes - with get_db_session() as session: - pending_row = session.query(SystemConfiguration).filter_by( - study_name=study_name, config_key="pending_search_space" - ).first() - if pending_row: - print(f"Pending Changes: YES (use 'python hpo_cli.py apply' to commit)") - else: - print(f"Pending Changes: NO") - print(f"==================================================\n") - -def cmd_review(args): - init_db() - study_name = get_study_name(args) - study = get_or_create_study(study_name) - - hpo_config = load_hpo_config(study_name) - from src.hpo_coordinator import study_eval_insights - insights = study_eval_insights(study, hpo_config) - heuristics = compute_review_heuristics(study, insights, hpo_config, study_name) - - # Check if review already completed - n_eval = heuristics["trials_evaluated"] - if not args.force and heuristics["already_reviewed"]: - print(f"Info: Study has already been reviewed for trial count {n_eval}. Use --force to override.") - return - - # Check if API keys are set - has_keys = any(os.getenv(k) for k in ("GEMINI_API_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY")) - - if not has_keys: - # Just print the prompt - print(f"No LLM API keys found (GEMINI_API_KEY, ANTHROPIC_API_KEY, or OPENAI_API_KEY).") - print(f"Printing coordinator review prompt below for manual copy-paste:\n") - print("----------------------------------------------------------------------") - print(build_review_prompt(study_name)) - print("----------------------------------------------------------------------") - return - - print("Running background LLM coordinator review...") - prompt = build_review_prompt(study_name) - - try: - response_text = call_llm(prompt) - # Parse JSON output from LLM - review_data = json.loads(response_text) - except Exception as e: - print(f"Failed to generate or parse LLM review: {e}") - sys.exit(1) + print(f"Evaluated: {count_evaluated_trials(study)}") + print(f"Health Tier: {health_tier.upper()}") + print(f"Health Reason: {health_reason}") + print("==================================================\n") - summary = review_data.get("summary", "LLM Generated review") - health_rating = review_data.get("health_rating", 3) - policy_action = review_data.get("policy_action", "no_change") - reasons = review_data.get("reasons", []) - est_imp = review_data.get("estimated_score_improvement") or review_data.get("estimated_dice_improvement") - cited_best = review_data.get("cited_best_trial") - patch = review_data.get("search_space_patch") - manual_trial = review_data.get("manual_trial") - - validation = validate_review_fields(est_imp, cited_best) - if not validation["ok"]: - print(f"Review JSON contract error: {'; '.join(validation['errors'])}") - sys.exit(1) - - print(f"\n==================================================") - print(f"🤖 LLM COORDINATOR REVIEW COMPLETED") - print(f"==================================================") - print(f"Health Rating: {health_rating}/5") - print(f"Action: {policy_action.upper()}") - print(f"Summary: {summary}") - if patch: - print(f"Space Patch: {json.dumps(patch)}") - if manual_trial: - print(f"Manual Trial: {json.dumps(manual_trial)}") - print(f"==================================================") - - # Persist the review - try: - result = save_study_review( - study_name, - summary, - health_rating=health_rating, - policy_action=policy_action, - model_version="cli_coordinator", - reasons=reasons, - trials_evaluated=n_eval, - estimated_score_improvement=est_imp, - cited_best_trial=cited_best, - force=args.force - ) - - applied = {} - space = load_search_space(study_name) - - # Save bounds proposal to pending config or apply it - if patch: - with get_db_session() as session: - session.merge(SystemConfiguration( - study_name=study_name, - config_key="pending_search_space", - config_value=json.dumps(patch) - )) - session.commit() - print("Proposed search space patch staged in 'pending_search_space'. Approve on dashboard or run 'python hpo_cli.py apply'.") - - if manual_trial: - applied["manual_trial"] = _enqueue_manual_trial(study, manual_trial, space, summary) - print(f"Enqueued manual trial: {manual_trial}") - - print("Review successfully saved in SQLite.") - except Exception as e: - print(f"Error persisting review: {e}") - sys.exit(1) - -def cmd_apply(args): - init_db() - study_name = get_study_name(args) - - with get_db_session() as session: - pending_row = session.query(SystemConfiguration).filter_by( - study_name=study_name, config_key="pending_search_space" - ).first() - if not pending_row: - print("No pending search space changes found.") - return - - proposed = json.loads(pending_row.config_value) - space = load_search_space(study_name) - - # Merge changes into active space - for key, new_val in proposed.items(): - if key in space: - p_type = space[key].get("type") - if p_type == "categorical": - if "active" in new_val: - space[key]["active"] = new_val["active"] - else: - if "min" in new_val: - space[key]["min"] = float(new_val["min"]) - if "max" in new_val: - space[key]["max"] = float(new_val["max"]) - - session.merge(SystemConfiguration( - study_name=study_name, - config_key="active_search_space", - config_value=json.dumps(space) - )) - session.delete(pending_row) - session.commit() - - mark_review_applied(study_name) - print("Pending search space changes committed successfully.") - - -def cmd_flag_review(args): - init_db() - result = flag_study_review(args.id, flagged=not args.unflag) - if not result.get("success"): - print(f"Error: {result.get('error')}") - sys.exit(1) - state = "flagged" if not args.unflag else "unflagged" - print(f"Review #{args.id} {state}.") - -def cmd_discard(args): - init_db() - study_name = get_study_name(args) - - with get_db_session() as session: - pending_row = session.query(SystemConfiguration).filter_by( - study_name=study_name, config_key="pending_search_space" - ).first() - if not pending_row: - print("No pending search space changes found.") - return - session.delete(pending_row) - session.commit() - - print("Pending search space changes discarded.") def cmd_validate(args): import yaml @@ -408,11 +156,12 @@ def cmd_quickstart(args): f.write(yaml_content) metric_name = "loss" if direction == "minimize" else "score" + broker_url = os.getenv("HPO_BROKER_URL", "http://localhost:8000") worker_content = f"""import sys from src.hpo_client import TrialSession def main(): - session = TrialSession(broker_url="http://localhost:8000", study_name="{study_name}") + session = TrialSession(broker_url="{broker_url}", study_name="{study_name}") trial = session.suggest() {param_name} = trial["params"]["{param_name}"] @@ -448,11 +197,11 @@ def main(): result = init_study_from_manifest_dict(data, force=False) print(result) - print(f"\n==================================================") + print("\n==================================================") print("🚀 SUCCESS! Your dummy study is registered.") print("Run the following command in another terminal:") - print(f"\n python quickstart_worker.py") - print(f"==================================================\n") + print("\n python quickstart_worker.py") + print("==================================================\n") sys.exit(0) def cmd_init(args): @@ -524,30 +273,6 @@ def cmd_export(args): study_name = get_study_name(args) fmt = args.format.lower() - if fmt == "sqlite": - if not args.output: - print("✗ Error: --output file path is required for sqlite format export.") - sys.exit(1) - print("Note: SQLite export copies the entire database file, including all studies.") - db_path = DATABASE_URL.replace("sqlite:///", "") if DATABASE_URL.startswith("sqlite:///") else "hpo_studies.db" - if not os.path.exists(db_path): - print(f"✗ Error: Source database file '{db_path}' does not exist.") - sys.exit(1) - - os.makedirs(os.path.dirname(os.path.abspath(args.output)) or ".", exist_ok=True) - try: - src_conn = sqlite3.connect(db_path) - dst_conn = sqlite3.connect(args.output) - with dst_conn: - src_conn.backup(dst_conn) - dst_conn.close() - src_conn.close() - print(f"✓ Successfully exported database to '{args.output}' via SQLite online backup.") - sys.exit(0) - except Exception as e: - print(f"✗ Error exporting database: {e}") - sys.exit(1) - try: study = load_study(study_name) except Exception as e: @@ -618,12 +343,7 @@ def cmd_export(args): "system_configuration": [], "compacted_packets": [], "study_cards": [], - "agent_reasoning_logs": [], - "study_reviews": [], "study_status": [], - "invalid_proposals": [], - "coordinator_metrics": [], - "suggest_metrics": [] } from optuna.distributions import distribution_to_json @@ -670,24 +390,9 @@ def cmd_export(args): cards = session.query(StudyCard).filter_by(study_name=study_name).all() export_data["study_cards"] = [c.to_dict() for c in cards] - reasoning = session.query(AgentReasoningLog).filter_by(study_name=study_name).all() - export_data["agent_reasoning_logs"] = [ar.to_dict() for ar in reasoning] - - reviews = session.query(StudyReview).filter_by(study_name=study_name).all() - export_data["study_reviews"] = [sr.to_dict() for sr in reviews] - status = session.query(StudyStatus).filter_by(study_name=study_name).all() export_data["study_status"] = [s.to_dict() for s in status] - proposals = session.query(InvalidProposal).filter_by(study_name=study_name).all() - export_data["invalid_proposals"] = [ip.to_dict() for ip in proposals] - - c_metrics = session.query(CoordinatorMetric).filter_by(study_name=study_name).all() - export_data["coordinator_metrics"] = [cm.to_dict() for cm in c_metrics] - - s_metrics = session.query(SuggestMetric).filter_by(study_name=study_name).all() - export_data["suggest_metrics"] = [sm.to_dict() for sm in s_metrics] - if args.output: os.makedirs(os.path.dirname(os.path.abspath(args.output)) or ".", exist_ok=True) with open(args.output, "w") as f: @@ -750,7 +455,7 @@ def cmd_import(args): print(f"Deleting existing study '{new_study_name}' as --force was specified...") optuna.delete_study(study_name=new_study_name, storage=DATABASE_URL) with get_db_session() as session: - for model in [TrialResult, TrialMetadata, SystemConfiguration, CompactedPacket, StudyCard, AgentReasoningLog, StudyReview, StudyStatus, InvalidProposal, TrialLease, CoordinatorMetric, SuggestMetric]: + for model in [TrialResult, TrialMetadata, SystemConfiguration, CompactedPacket, StudyCard, StudyStatus, TrialLease]: session.query(model).filter_by(study_name=new_study_name).delete() except KeyError: pass @@ -768,7 +473,7 @@ def cmd_import(args): sys.exit(1) try: - print(f"Importing Optuna trials...") + print("Importing Optuna trials...") trial_id_mapping = {} from optuna.trial import FrozenTrial, TrialState from optuna.distributions import json_to_distribution @@ -786,7 +491,7 @@ def cmd_import(args): if t_state_name == "RUNNING": t_state_name = "FAIL" if not dt_complete: - dt_complete = datetime.datetime.utcnow() + dt_complete = datetime.datetime.now(timezone.utc).replace(tzinfo=None) frozen_trial = FrozenTrial( number=t["number"], @@ -807,7 +512,7 @@ def cmd_import(args): new_trial = study.trials[-1] trial_id_mapping[t["trial_id"]] = new_trial._trial_id - print(f"Importing custom Pathfinder tables...") + print("Importing custom Pathfinder tables...") with get_db_session() as session: # Cache invalidation: delete old compacted packets session.query(CompactedPacket).filter_by(study_name=new_study_name).delete() @@ -826,7 +531,7 @@ def cmd_import(args): if new_trial_id is None: continue - created_at = datetime.datetime.fromisoformat(r["created_at"]) if r.get("created_at") else datetime.datetime.utcnow() + created_at = datetime.datetime.fromisoformat(r["created_at"]) if r.get("created_at") else datetime.datetime.now(timezone.utc).replace(tzinfo=None) session.add(TrialResult( trial_id=new_trial_id, study_name=new_study_name, @@ -852,7 +557,7 @@ def cmd_import(args): new_trial_id = trial_id_mapping.get(orig_trial_id) if new_trial_id is None: continue - created_at = datetime.datetime.fromisoformat(m["created_at"]) if m.get("created_at") else datetime.datetime.utcnow() + created_at = datetime.datetime.fromisoformat(m["created_at"]) if m.get("created_at") else datetime.datetime.now(timezone.utc).replace(tzinfo=None) session.add(TrialMetadata( trial_id=new_trial_id, study_name=new_study_name, @@ -861,27 +566,8 @@ def cmd_import(args): created_at=created_at )) - for ar in export_data.get("agent_reasoning_logs", []): - orig_trial_id = ar["trial_id"] - new_trial_id = trial_id_mapping.get(orig_trial_id) - if new_trial_id is None: - continue - created_at = datetime.datetime.fromisoformat(ar["created_at"]) if ar.get("created_at") else datetime.datetime.utcnow() - session.add(AgentReasoningLog( - trial_id=new_trial_id, - study_name=new_study_name, - model_version=ar["model_version"], - prompt_strategy=ar["prompt_strategy"], - predicted_outcome_rationale=ar["predicted_outcome_rationale"], - estimated_score_improvement=ar["estimated_score_improvement"], - actual_score_improvement=ar.get("actual_score_improvement"), - created_at=created_at - )) - - # CompactedPackets: Omitted/cache invalidation (skip importing) - for c in export_data.get("study_cards", []): - created_at = datetime.datetime.fromisoformat(c["created_at"]) if c.get("created_at") else datetime.datetime.utcnow() + created_at = datetime.datetime.fromisoformat(c["created_at"]) if c.get("created_at") else datetime.datetime.now(timezone.utc).replace(tzinfo=None) session.add(StudyCard( study_name=new_study_name, card_type=c["card_type"], @@ -891,36 +577,8 @@ def cmd_import(args): created_at=created_at )) - for sr in export_data.get("study_reviews", []): - created_at = datetime.datetime.fromisoformat(sr["created_at"]) if sr.get("created_at") else datetime.datetime.utcnow() - applied_at = datetime.datetime.fromisoformat(sr["applied_at"]) if sr.get("applied_at") else None - outcome_measured_at = datetime.datetime.fromisoformat(sr["outcome_measured_at"]) if sr.get("outcome_measured_at") else None - - review = StudyReview( - study_name=new_study_name, - health_rating=sr.get("health_rating"), - summary=sr["summary"], - policy_action=sr.get("policy_action", "no_change"), - model_version=sr.get("model_version", "unspecified"), - prompt_strategy=sr.get("prompt_strategy", "coordinator_review"), - trials_evaluated=sr.get("trials_evaluated", 0), - estimated_score_improvement=sr.get("estimated_score_improvement"), - cited_best_trial=sr.get("cited_best_trial"), - confidence=sr.get("confidence", "high"), - baseline_best_score=sr.get("baseline_best_score"), - applied_at_completed_count=sr.get("applied_at_completed_count"), - applied_at=applied_at, - actual_score_improvement=sr.get("actual_score_improvement"), - outcome_measured_at=outcome_measured_at, - outcome_status=sr.get("outcome_status", "pending"), - quality_flagged=sr.get("quality_flagged", False), - created_at=created_at - ) - review.set_reasons(sr.get("reasons", [])) - session.add(review) - for s in export_data.get("study_status", []): - health_updated_at = datetime.datetime.fromisoformat(s["health_updated_at"]) if s.get("health_updated_at") else datetime.datetime.utcnow() + health_updated_at = datetime.datetime.fromisoformat(s["health_updated_at"]) if s.get("health_updated_at") else datetime.datetime.now(timezone.utc).replace(tzinfo=None) session.add(StudyStatus( study_name=new_study_name, health_tier=s.get("health_tier", "healthy"), @@ -928,37 +586,6 @@ def cmd_import(args): health_updated_at=health_updated_at, nudge_dismissed_trials=s.get("nudge_dismissed_trials") )) - - for ip in export_data.get("invalid_proposals", []): - created_at = datetime.datetime.fromisoformat(ip["created_at"]) if ip.get("created_at") else datetime.datetime.utcnow() - session.add(InvalidProposal( - study_name=new_study_name, - model_version=ip["model_version"], - prompt_strategy=ip["prompt_strategy"], - invalid_parameters=json.dumps(ip.get("invalid_parameters", {})), - validation_error=ip["validation_error"], - created_at=created_at - )) - - for cm in export_data.get("coordinator_metrics", []): - timestamp = datetime.datetime.fromisoformat(cm["timestamp"]) if cm.get("timestamp") else datetime.datetime.utcnow() - session.add(CoordinatorMetric( - study_name=new_study_name, - timestamp=timestamp, - model=cm["model"], - latency_ms=cm["latency_ms"], - action_taken=cm["action_taken"], - trials_at_review=cm["trials_at_review"] - )) - - for sm in export_data.get("suggest_metrics", []): - timestamp = datetime.datetime.fromisoformat(sm["timestamp"]) if sm.get("timestamp") else datetime.datetime.utcnow() - session.add(SuggestMetric( - study_name=new_study_name, - timestamp=timestamp, - latency_ms=sm["latency_ms"], - source=sm["source"] - )) except Exception as err: print(f"✗ Error during import execution: {err}") print("Rolling back database transaction and deleting half-imported study...") @@ -999,6 +626,103 @@ def cmd_backup(args): print(f"✗ Backup failed: {e}") sys.exit(1) +def cmd_modelcard(args): + import hashlib + import os as _os + + study_name = args.study_name + init_db() + + packet = build_study_packet(study_name) + + if not packet or packet.get("success") is False: + print(f"Error: Could not build study packet for '{study_name}'") + sys.exit(1) + + lines = [] + lines.append(f"# Model Card: {study_name}") + lines.append("") + lines.append(f"**Generated:** {datetime.datetime.now(timezone.utc).replace(tzinfo=None).isoformat()}") + lines.append("") + + lines.append("## Executive Summary") + counts = packet.get("counts", {}) + lines.append(f"- Total Trials: {counts.get('total', 0)}") + lines.append(f"- Completed: {counts.get('complete', 0)}") + lines.append(f"- Pruned: {counts.get('pruned', 0)}") + lines.append(f"- Failed: {counts.get('failed', 0)}") + lines.append(f"- Statistical Confidence: {packet.get('statistical_confidence', 'unknown')}") + lines.append("") + + lines.append("## Best Parameters") + trial_bins = packet.get("trial_bins", {}) + elite = trial_bins.get("elite", []) + if elite: + best = elite[0] + lines.append(f"- Best Trial: #{best.get('trial_id')}") + lines.append(f"- Best Score: {best.get('primary_score')}") + lines.append(f"- Best Loss: {best.get('primary_loss')}") + lines.append("") + lines.append("### Best Trial Parameters") + for k, v in best.get("params", {}).items(): + lines.append(f"- `{k}`: {v}") + lines.append("") + + fanova = packet.get("fanova_importances", {}) + if fanova: + lines.append("## fANOVA Importances") + for param, importance in sorted(fanova.items(), key=lambda x: x[1], reverse=True): + lines.append(f"- `{param}`: {importance:.4f}") + lines.append("") + + vram = packet.get("vram_telemetry", {}) + if vram: + lines.append("## VRAM Telemetry") + lines.append(f"- GPU Model: {vram.get('gpu_model', 'Unknown')}") + lines.append(f"- GPU Capacity: {vram.get('gpu_capacity_gb', 'N/A')} GB") + lines.append(f"- OOM Count: {vram.get('oom_count', 0)}") + oom_risk = vram.get("bounds_oom_risk") + if oom_risk: + lines.append(f"- OOM Risk Level: {oom_risk.get('risk_level', 'N/A')}") + lines.append(f"- Predicted Max VRAM: {oom_risk.get('predicted_max_vram_gb', 'N/A'):.2f} GB") + lines.append("") + + health = packet.get("health", {}) + lines.append("## Health") + lines.append(f"- Tier: {health.get('tier', 'unknown')}") + lines.append(f"- Reason: {health.get('reason', 'N/A')}") + lines.append("") + + content = "\n".join(lines) + + _os.makedirs("studies", exist_ok=True) + file_path = f"studies/{study_name}_model_card.md" + with open(file_path, "w") as f: + f.write(content) + + content_hash = hashlib.sha256(content.encode()).hexdigest() + with get_db_session() as session: + session.add(StudyCard( + study_name=study_name, + card_type="model_card", + file_path=file_path, + content_hash=content_hash, + metadata_json=json.dumps({"generated_at": datetime.datetime.now(timezone.utc).replace(tzinfo=None).isoformat()}) + )) + session.commit() + + print(f"Model card saved to {file_path} (hash: {content_hash[:12]}...)") + +def cmd_delete(args): + study_name = args.study_name + if not args.confirm: + print("Use --confirm to permanently delete the study") + sys.exit(0) + + from src.onboarding import delete_study_internal + result = delete_study_internal(study_name=study_name, confirm=True) + print(result["message"]) + def main(): parser = argparse.ArgumentParser(description="Pathfinder CLI Control") subparsers = parser.add_subparsers(dest="command", required=True) @@ -1007,30 +731,12 @@ def main(): p_status = subparsers.add_parser("status", help="Get study health, trial counts, and pending status") p_status.add_argument("--study", help="Study name") - # Review - p_review = subparsers.add_parser("review", help="Execute coordinator review or output review prompt") - p_review.add_argument("--study", help="Study name") - p_review.add_argument("--force", action="store_true", help="Force review generation even if already completed for current trials") - - # Apply - p_apply = subparsers.add_parser("apply", help="Commit pending search bounds configuration") - p_apply.add_argument("--study", help="Study name") - - # Discard - p_discard = subparsers.add_parser("discard", help="Discard pending search bounds configuration") - p_discard.add_argument("--study", help="Study name") - - # Flag review quality - p_flag = subparsers.add_parser("flag-review", help="Flag a coordinator review as low-quality (excluded from MAE)") - p_flag.add_argument("--id", type=int, required=True, help="StudyReview row id") - p_flag.add_argument("--unflag", action="store_true", help="Remove quality flag") - # Validate p_validate = subparsers.add_parser("validate", help="Check manifest for errors") p_validate.add_argument("manifest", help="Path to manifest YAML file") # Quickstart - p_quickstart = subparsers.add_parser("quickstart", help="Interactive wizard to generate and initialize a dummy study") + subparsers.add_parser("quickstart", help="Interactive wizard to generate and initialize a dummy study") # Init p_init = subparsers.add_parser("init", help="Validate + register study") @@ -1044,8 +750,8 @@ def main(): # Export p_export = subparsers.add_parser("export", help="Export HPO study trials and config") p_export.add_argument("--study", help="Study name") - p_export.add_argument("--format", choices=["json", "csv", "sqlite"], default="json", help="Export format (default: json)") - p_export.add_argument("--output", help="File path to save the export (required for csv and sqlite)") + p_export.add_argument("--format", choices=["json", "csv"], default="json", help="Export format (default: json)") + p_export.add_argument("--output", help="File path to save the export (required for csv)") # Import p_import = subparsers.add_parser("import", help="Import HPO study trials and config from JSON file") @@ -1057,18 +763,19 @@ def main(): p_backup = subparsers.add_parser("backup", help="Create a safe online backup of the SQLite database") p_backup.add_argument("--output", help="Custom backup file path") + # Modelcard + p_modelcard = subparsers.add_parser("modelcard", help="Generate a model card for a study") + p_modelcard.add_argument("study_name", help="Study name to generate model card for") + + # Delete + p_delete = subparsers.add_parser("delete", help="Permanently delete a study and all its data") + p_delete.add_argument("study_name", help="Study name to delete") + p_delete.add_argument("--confirm", action="store_true", help="Confirm permanent deletion") + args = parser.parse_args() if args.command == "status": cmd_status(args) - elif args.command == "review": - cmd_review(args) - elif args.command == "apply": - cmd_apply(args) - elif args.command == "discard": - cmd_discard(args) - elif args.command == "flag-review": - cmd_flag_review(args) elif args.command == "validate": cmd_validate(args) elif args.command == "quickstart": @@ -1083,6 +790,10 @@ def main(): cmd_import(args) elif args.command == "backup": cmd_backup(args) + elif args.command == "modelcard": + cmd_modelcard(args) + elif args.command == "delete": + cmd_delete(args) if __name__ == "__main__": main() diff --git a/hpo_mcp_server.py b/hpo_mcp_server.py index 9c560d7..bbe7f27 100644 --- a/hpo_mcp_server.py +++ b/hpo_mcp_server.py @@ -1,481 +1,55 @@ -import os -import json -import datetime -import hashlib from typing import Optional, Dict, Any, List -import requests -import optuna -from optuna.trial import TrialState from mcp.server.fastmcp import FastMCP mcp = FastMCP("Pathfinder") -from src.db_manager import init_db, get_db_session, DATABASE_URL -from src.schema import ( - TrialResult, - TrialMetadata, - SystemConfiguration, - CompactedPacket, - StudyCard, - AgentReasoningLog, - StudyReview, - StudyStatus -) -from src.hpo_config import load_hpo_config, normalize_trial_params -from src.analytics import build_compacted_packet -from src.hpo_coordinator import ( - compute_health_tier, - count_evaluated_trials, - POLICY_ACTIONS, - build_review_packet, - load_study_cards -) - -from src.search_space import ( - load_search_space, - save_search_space, - _apply_search_space_patch, -) -from src.suggest import get_or_create_study, _enqueue_manual_trial - -from src.hpo_coordinator import _validate_manual_parameters +from src.db_manager import init_db # --- MCP TOOLS --- -from src.onboarding import ( - initialize_study as core_initialize_study, - delete_study_internal, - init_study_from_manifest_dict -) - -@mcp.tool() -def initialize_study( - study_name: str, - active_search_space: Dict[str, Any], - hpo_config: Dict[str, Any], - project_context: Optional[Dict[str, Any]] = None, - source_files: Optional[Dict[str, str]] = None, - multi_objective: bool = True, - directions: Optional[List[str]] = None -) -> str: - """Initializes a new study: creates Optuna study and stores search space, config, context, and source files in DB.""" - return core_initialize_study( - study_name=study_name, - active_search_space=active_search_space, - hpo_config=hpo_config, - project_context=project_context, - source_files=source_files, multi_objective=multi_objective, - directions=directions - ) - @mcp.tool() def get_study_data(study_name: str) -> Dict[str, Any]: """Returns the compacted HPO review packet, utilizing a lazy materialization cache layer.""" - return build_review_packet(study_name) - -@mcp.tool() -def validate_search_space( - space_config: Dict[str, Any], - hpo_config: Optional[Dict[str, Any]] = None, - project_context: Optional[Dict[str, Any]] = None, - historical_fail_patterns: Optional[List[str]] = None, -) -> Dict[str, Any]: - """Validate search space bounds, tunable coverage, and metric label consistency.""" - errors = [] - warnings = [] - tunable_count = 0 - - for param_name, spec in space_config.items(): - if param_name.startswith("_") or not isinstance(spec, dict): - continue - ptype = spec.get("type") - if ptype in ("float", "float_log", "int"): - lo = spec.get("min") - hi = spec.get("max") - if lo is None or hi is None: - errors.append(f"Parameter '{param_name}' is missing required min/max bounds.") - else: - try: - lo_f = float(lo) - hi_f = float(hi) - if lo_f >= hi_f: - errors.append(f"Parameter '{param_name}': min ({lo}) must be strictly less than max ({hi}).") - if ptype == "float_log" and lo_f <= 0: - errors.append(f"Parameter '{param_name}' is log-scale and must have a min bound strictly greater than 0.") - if ptype == "float_log" and hi_f > 0 and lo_f > 0: - import math - span_orders = math.log10(hi_f) - math.log10(lo_f) - if span_orders > 6: - warnings.append( - f"Parameter '{param_name}' log-span spans {span_orders:.1f} orders of magnitude (>6); consider narrowing." - ) - if hi_f > lo_f: - tunable_count += 1 - except (ValueError, TypeError): - errors.append(f"Parameter '{param_name}' has non-numeric min/max bounds.") - elif ptype == "categorical": - options = spec.get("options", []) - active = spec.get("active", options) - if not options: - errors.append(f"Categorical parameter '{param_name}' must specify allowed options.") - elif len(active) == 0: - errors.append(f"Categorical parameter '{param_name}' must have at least one active option.") - elif len(active) == 1: - warnings.append(f"Categorical parameter '{param_name}' has only 1 active choice ({active[0]}), effectively pinning it.") - elif len(active) > 1: - tunable_count += 1 - - # Check active in options - invalid_active = [x for x in active if x not in options] - if invalid_active: - errors.append(f"Categorical parameter '{param_name}' active options {invalid_active} are not in choices: {options}") - - # Check known OOM-risk combinations - batch_size_spec = space_config.get("batch_size", {}) - resolution_spec = space_config.get("resolution", {}) - - max_bs = None - if batch_size_spec.get("type") == "categorical": - max_bs = max(batch_size_spec.get("active", [0])) - elif batch_size_spec.get("type") in ("int", "float"): - max_bs = batch_size_spec.get("max") - - max_res = None - if resolution_spec.get("type") == "categorical": - max_res = max(resolution_spec.get("active", [0])) - elif resolution_spec.get("type") in ("int", "float"): - max_res = resolution_spec.get("max") - - if max_bs is not None and max_res is not None: - try: - if float(max_bs) >= 64 and float(max_res) >= 1024: - warnings.append(f"High risk configuration: batch_size={max_bs} combined with resolution={max_res} has historically high OOM risk.") - except (ValueError, TypeError): - pass - - if tunable_count == 0: - errors.append("Search space has no tunable parameters (all bounds pinned or single-choice categoricals).") - - if project_context: - ctx = project_context if isinstance(project_context, dict) else {} - declared_score = ctx.get("metric_score_name") or ctx.get("score_metric") - declared_loss = ctx.get("metric_loss_name") or ctx.get("loss_metric") - if hpo_config and (declared_score or declared_loss): - if declared_score and not hpo_config.get("metric_score_label"): - warnings.append("project_context declares a score metric but hpo_config.metric_score_label is missing.") - if declared_loss and not hpo_config.get("metric_loss_label"): - warnings.append("project_context declares a loss metric but hpo_config.metric_loss_label is missing.") - - return { - "valid": len(errors) == 0, - "errors": errors, - "warnings": warnings - } - -@mcp.tool() -def update_search_space(study_name: str, space_config: Dict[str, Any], apply: bool = False) -> str: - """Propose or apply updates to the active search space. - - Every change is validated against the canonical search space (legacy parameter aliases - already normalized). With apply=False the change is staged as ``pending_search_space`` for - the human to approve on the dashboard; apply=True commits immediately. Unrecognized - parameters, out-of-bounds categorical choices, and no-op proposals return an explicit error - string instead of being silently dropped. - """ - current_space = load_search_space(study_name) - validated_proposals: Dict[str, Any] = {} - - for param_name, new_val in space_config.items(): - if param_name not in current_space: - return f"Error: Hyperparameter '{param_name}' is not recognized in the search space." - param_type = current_space[param_name].get("type") - proposal: Dict[str, Any] = {} - if param_type == "categorical": - if "active" in new_val: - allowed = current_space[param_name].get("options", []) - invalid_options = [x for x in new_val["active"] if x not in allowed] - if invalid_options: - return f"Error: Active choices {invalid_options} for {param_name} are not in options: {allowed}" - if len(new_val["active"]) == 0: - return f"Error: Categorical parameter {param_name} must have at least one active option." - proposal["active"] = new_val["active"] - else: - if "min" in new_val: - proposal["min"] = float(new_val["min"]) - if "max" in new_val: - proposal["max"] = float(new_val["max"]) - if not proposal: - return ( - f"Error: No valid changes for '{param_name}'. Provide 'active' for categorical " - f"parameters, or 'min'/'max' for numeric parameters." - ) - validated_proposals[param_name] = proposal - - if apply: - for key, new_val in validated_proposals.items(): - current_space[key].update(new_val) - save_search_space(current_space, study_name) - with get_db_session() as session: - pending = session.query(SystemConfiguration).filter_by( - study_name=study_name, config_key="pending_search_space" - ).first() - if pending: - session.delete(pending) - from src.hpo_coordinator import mark_review_applied - mark_review_applied(study_name) - return "Search space changes committed successfully." - - with get_db_session() as session: - session.merge(SystemConfiguration( - study_name=study_name, - config_key="pending_search_space", - config_value=json.dumps(validated_proposals), - )) - return "Search space changes proposed successfully. They must be approved via the dashboard before they take effect." - -@mcp.tool() -def delete_study(study_name: str, confirm: bool = False) -> Dict[str, Any]: - """Permanently delete a study: its Optuna trials and ALL custom metadata rows.""" - return delete_study_internal(study_name=study_name, confirm=confirm) + from src.analytics import build_study_packet + return build_study_packet(study_name) -@mcp.tool() -def generate_model_card(study_name: str) -> Dict[str, Any]: - """Generates an end-of-study synthesis, writes MODEL_CARD.md to disk, and indexes it in DB.""" - try: - # Load data - packet = get_study_data(study_name) - if "error" in packet: - return packet - - best_params = {} - best_score = 0.0 - elite = packet.get("trial_bins", {}).get("elite", []) - if elite: - best_params = elite[0].get("params", {}) - best_score = elite[0].get("primary_score", 0.0) - - # Construct beautiful model card Markdown - card_content = f"""# Study Model Card: {study_name} - -## Executive Summary -This model card synthesizes results for study `{study_name}`. - -- **Best Achieved Score ({packet.get('metric_score_label', 'Score')}):** {best_score:.4f} -- **Optimal Hyperparameters:** -{chr(10).join(f" - `{k}`: {v}" for k, v in best_params.items())} - -## Search Space Performance -- **Total Trials Evaluated:** {packet.get('counts', {}).get('total', 0)} -- **Successful Runs:** {packet.get('counts', {}).get('complete', 0)} -- **Pruned Runs:** {packet.get('counts', {}).get('pruned', 0)} -- **Failed/OOM Runs:** {packet.get('counts', {}).get('failed', 0)} - -### Key Parameter Importances (fANOVA) -{chr(10).join(f"- `{k}`: {v:.4f}" for k, v in packet.get('fanova_importances', {}).items())} - -## Telemetry Profile -- **GPU Device:** {packet.get('vram_telemetry', {}).get('gpu_model', 'Unknown')} -- **Peak VRAM Recorded:** {packet.get('vram_telemetry', {}).get('gpu_capacity_gb', 0.0):.2f} GB -- **OOM Failures:** {packet.get('vram_telemetry', {}).get('oom_count', 0)} - ---- -*Generated by Pathfinder on {datetime.datetime.utcnow().isoformat()}* -""" - - # Write to studies directory in workspace - studies_dir = os.path.join(os.path.dirname(__file__), "studies") - os.makedirs(studies_dir, exist_ok=True) - - file_path = os.path.join(studies_dir, f"{study_name}_model_card.md") - with open(file_path, "w") as f: - f.write(card_content) - - # Hash calculation - sha = hashlib.sha256(card_content.encode("utf-8")).hexdigest() - - # Save card index to database - with get_db_session() as session: - session.merge(StudyCard( - study_name=study_name, - card_type="model_card", - file_path=os.path.relpath(file_path, os.path.dirname(__file__)), - content_hash=sha, - metadata_json=json.dumps({ - "best_score": best_score, - "best_params": best_params, - "total_trials": packet.get("counts", {}).get("total", 0) - }) - )) - - return { - "success": True, - "file_path": file_path, - "content_hash": sha, - "message": f"Model card written to disk and database index updated." - } - except Exception as e: - return {"success": False, "error": f"Failed to generate model card: {str(e)}"} - -@mcp.tool() -def submit_agent_review( - study_name: str, - summary: str, - health_rating: int, - policy_action: str = "no_change", - model_version: str = "coordinator", - prompt_strategy: str = "coordinator_review", - reasons: Optional[List[Dict[str, Any]]] = None, - estimated_score_improvement: Optional[float] = None, - cited_best_trial: Optional[int] = None, - search_space_patch: Optional[Dict[str, Any]] = None, - manual_trial: Optional[Dict[str, Any]] = None, - force: bool = False, -) -> Dict[str, Any]: - """Persists a coordinator review. Idempotent per trial window. Matches the HTTP route logic.""" - try: - from src.hpo_coordinator import ( - save_study_review, - count_evaluated_trials, - POLICY_ACTIONS, - validate_review_fields, - ) - - if policy_action not in POLICY_ACTIONS: - return { - "success": False, - "error": f"Invalid policy_action '{policy_action}'. Valid options are: {', '.join(POLICY_ACTIONS)}" - } - - study = get_or_create_study(study_name) - space = load_search_space(study_name) - trials_evaluated = count_evaluated_trials(study) - - if manual_trial: - val_res = _validate_manual_parameters(manual_trial, study_name) - if not val_res["ok"]: - with get_db_session() as session: - from src.schema import InvalidProposal - session.add(InvalidProposal( - study_name=study_name, - model_version=model_version or "coordinator", - prompt_strategy=prompt_strategy or "coordinator_review", - invalid_parameters=json.dumps(manual_trial), - validation_error=val_res["error"] - )) - return {"success": False, "error": f"Invalid manual parameters: {val_res['error']}"} - - validation = validate_review_fields(estimated_score_improvement, cited_best_trial) - if not validation["ok"]: - return {"success": False, "error": "; ".join(validation["errors"])} - - result = save_study_review( - study_name, - summary, - health_rating=health_rating, - policy_action=policy_action or "no_change", - model_version=model_version or "coordinator", - prompt_strategy=prompt_strategy or "coordinator_review", - reasons=reasons, - trials_evaluated=trials_evaluated, - estimated_score_improvement=estimated_score_improvement, - cited_best_trial=cited_best_trial, - force=force, - ) - - applied = {} - if not result.get("duplicate"): - if search_space_patch: - applied["search_space"] = _apply_search_space_patch(search_space_patch, space, study_name) - if manual_trial: - applied["manual_trial"] = _enqueue_manual_trial(study, manual_trial, space, summary) - - result["applied"] = applied - return result - except Exception as e: - import traceback - traceback.print_exc() - return {"success": False, "error": str(e)} - -@mcp.tool() -def validate_integration(study_name: str) -> Dict[str, Any]: - """Validates that a study is correctly initialized and configured in SQLite.""" - try: - from src.db_manager import get_db_session - from src.schema import SystemConfiguration, StudyStatus - import optuna - - status = {} - try: - study = optuna.load_study(study_name=study_name, storage=DATABASE_URL) - status["optuna_study_exists"] = True - status["study_directions"] = [d.name for d in study.directions] - status["total_trials"] = len(study.trials) - except Exception as e: - status["optuna_study_exists"] = False - status["optuna_study_error"] = str(e) - - with get_db_session() as session: - space = session.query(SystemConfiguration).filter_by( - study_name=study_name, config_key="active_search_space" - ).first() - config = session.query(SystemConfiguration).filter_by( - study_name=study_name, config_key="hpo_config" - ).first() - status["db_search_space_configured"] = space is not None - status["db_hpo_config_configured"] = config is not None - - status_row = session.query(StudyStatus).filter_by(study_name=study_name).first() - if status_row: - status["health_tier"] = status_row.health_tier - status["health_reason"] = status_row.health_reason - else: - status["health_tier"] = "unknown" - - broker_url = os.getenv("HPO_BROKER_URL", "http://localhost:8000") - status["broker_url"] = broker_url - try: - resp = requests.get(f"{broker_url.rstrip('/')}/health", timeout=3) - status["broker_online"] = resp.status_code == 200 - except Exception as e: - status["broker_online"] = False - status["broker_error"] = str(e) - - status["success"] = status.get("optuna_study_exists", False) and status.get("db_search_space_configured", False) - return status - except Exception as e: - return {"success": False, "error": str(e)} - @mcp.tool() def get_study_cards(study_name: Optional[str] = None) -> List[Dict[str, Any]]: """Retrieves generated study cards (model cards, recaps) from the database to enable cross-study queries.""" + from src.analytics import load_study_cards return load_study_cards(study_name) + @mcp.tool() def validate_manifest(yaml_str: str) -> Dict[str, Any]: """Mechanically validate a manifest YAML string against the Pathfinder schema rules.""" import yaml from src.manifest import validate_manifest as core_validate + try: data = yaml.safe_load(yaml_str) except Exception as e: return {"success": False, "errors": [f"Invalid YAML structure: {str(e)}"], "warnings": []} - + if not isinstance(data, dict): return {"success": False, "errors": ["Manifest root must be a dictionary"], "warnings": []} errors, warnings = core_validate(data) return {"success": len(errors) == 0, "errors": errors, "warnings": warnings} + @mcp.tool() def init_from_manifest(yaml_str: str, force: bool = False) -> Dict[str, Any]: """Validate and register a new HPO study from a manifest YAML string, with deep overwrite cleanup on force=True.""" import yaml + from src.onboarding import init_study_from_manifest_dict + try: data = yaml.safe_load(yaml_str) except Exception as e: return {"success": False, "error": f"Invalid YAML structure: {str(e)}"} - + if not isinstance(data, dict): return {"success": False, "error": "Manifest root must be a dictionary"} @@ -485,12 +59,14 @@ def init_from_manifest(yaml_str: str, force: bool = False) -> Dict[str, Any]: except Exception as e: return {"success": False, "error": str(e)} + @mcp.tool() def export_manifest(study_name: str) -> str: """Export the active search space, HPO config, and context of an existing study as a valid manifest YAML string.""" from src.manifest import export_manifest_yaml return export_manifest_yaml(study_name) + # --- MCP PROMPT RESOURCES --- @mcp.resource("hpo://prompts/grill") @@ -503,26 +79,11 @@ def resource_grill() -> str: 1. Draft a YAML manifest configuration (e.g. `train.hpo.yaml`). 2. Call `validate_manifest(yaml_str)` to check for errors/warnings mechanically. 3. Call `init_from_manifest(yaml_str)` to register the study in SQLite and Optuna. -4. Call `validate_integration(study_name)` to confirm the broker is healthy and integration is ready. +4. Call `get_study_data(study_name)` to confirm the study is accessible and healthy. Worker integration reference: `docs/INTEGRATION.md`. Do not write json space config files to disk. """ -@mcp.resource("hpo://prompts/review") -def resource_review() -> str: - """7-step episodic coordinator review (human-initiated only).""" - return """# Pathfinder Coordinator Review (7 Steps) - -Follow AGENTS.md. Trigger only when the user explicitly requests a review (watch/intervene nudges are not automatic). - -1. `get_study_data(study_name)` — packet includes fANOVA, past_reviews, coordinator_accuracy, statistical_confidence. -2. Interpret metrics using dynamic labels from project_context; heed statistical_confidence caveat when low/medium. -3. VRAM safety via vram_telemetry bounds_oom_risk; check past_reviews (ignore quality_flagged). -4. Self-regulate only if coordinator_accuracy.n_scored_reviews >= 3 and MAE > 0.05. -5. `update_search_space(..., apply=False)` to stage bounds (human approves on dashboard). -6. `submit_agent_review` with required estimated_score_improvement and cited_best_trial. -7. `generate_model_card(study_name)` when wrapping up. -""" if __name__ == "__main__": init_db() diff --git a/pathfinder.egg-info/PKG-INFO b/pathfinder.egg-info/PKG-INFO deleted file mode 100644 index 96f3743..0000000 --- a/pathfinder.egg-info/PKG-INFO +++ /dev/null @@ -1,302 +0,0 @@ -Metadata-Version: 2.4 -Name: pathfinder -Version: 1.0.0 -Summary: Decoupled hyperparameter optimization (HPO) framework with periodically scheduled AI coordinator reviews -Author-email: Ishaan Patel -License: MIT -Classifier: Programming Language :: Python :: 3 -Classifier: License :: OSI Approved :: MIT License -Classifier: Operating System :: OS Independent -Requires-Python: >=3.10 -Description-Content-Type: text/markdown -License-File: LICENSE -Requires-Dist: mcp<2,>=1.1 -Requires-Dist: optuna<5,>=3.6 -Requires-Dist: optuna-dashboard<1,>=0.15 -Requires-Dist: sqlalchemy<3,>=2.0 -Requires-Dist: psycopg2-binary<3,>=2.9 -Requires-Dist: pydantic<3,>=2.0 -Requires-Dist: numpy<3,>=1.20 -Requires-Dist: scikit-learn<2,>=1.0 -Requires-Dist: fastapi<1,>=0.110 -Requires-Dist: uvicorn<1,>=0.23 -Requires-Dist: requests<3,>=2.28 -Requires-Dist: PyYAML<7,>=6.0 -Provides-Extra: dev -Requires-Dist: pytest>=8.0; extra == "dev" -Dynamic: license-file - -# Pathfinder - -[![Python 3.10+](https://img.shields.io/badge/python-3.10+-3776AB?style=flat-square&logo=python&logoColor=white)](https://www.python.org/downloads/) -[![FastAPI](https://img.shields.io/badge/FastAPI-009688?style=flat-square&logo=fastapi&logoColor=white)](https://fastapi.tiangolo.com/) -[![Optuna](https://img.shields.io/badge/Optuna-Tuning-1E90FF?style=flat-square)](https://optuna.org/) -[![SQLite](https://img.shields.io/badge/SQLite-003B57?style=flat-square&logo=sqlite&logoColor=white)](https://www.sqlite.org/) -[![MCP](https://img.shields.io/badge/MCP-Model_Context_Protocol-orange?style=flat-square)](https://modelcontextprotocol.io/) -[![Build Status](https://github.com/Ishaan1402/pathfinder/actions/workflows/ci.yml/badge.svg)](https://github.com/Ishaan1402/pathfinder/actions) -[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg?style=flat-square)](LICENSE) - - -A decoupled hyperparameter optimization (HPO) framework that separates the deterministic optimizer from episodic AI reviews. Train workers run autonomously without ever blocking on an LLM. Optimizers run fast. Humans (or AI agents in your IDE) review results periodically and decide when to adjust the search space. - -**Designed for:** ML researchers and engineers tuning deep learning models on their own infrastructure (local GPU, Colab, cloud VMs). Use it as a reference for the bridge-crack U-Net project, or adapt the templates for your own training script. - -## Why Pathfinder? - -**Problem:** Traditional HPO frameworks either require workers to wait for an optimizer, or they add LLM reasoning that introduces latency into every training loop. You end up trading off between speed and intelligence. - -**Solution:** Three independent layers: - -- **Broker (Optuna TPE)**: Fast, deterministic suggestion engine. Never calls an LLM. Workers hit this endpoint and move on. -- **Worker**: Train autonomously. Report metrics incrementally. Handles pruning, OOM, checkpointing. Never waits. -- **Coordinator (You + Optional LLM)**: Run episodic reviews when *you* decide. Inspect trial history, check search health, propose bounds changes. AI agents (Claude, Cursor) can run reviews via MCP tools. - -All state lives in **SQLite**—no config files, no in-memory state. This makes it easy to resume reviews, audit decisions, and sync across machines. - -## Quick Start - -### Option 1: Local GPU - -```bash -python3 -m venv .venv -source .venv/bin/activate -pip install -r requirements.txt - -# Terminal 1: Start broker -python broker.py --daemon -# Dashboard: http://127.0.0.1:8000 - -# Terminal 2: Run worker -HPO_BROKER_URL=http://localhost:8000 HPO_STUDY_NAME=test_study python simulators/training_worker.py -Option 2: Remote Workers (Colab / Cloud GPU) -``` - -### Option 2: Remote Workers (Colab / Cloud GPU) - -```bash -# Terminal 1: Start broker with tunnel + auth -export HPO_SECRET_TOKEN="$(python -c 'import secrets; print(secrets.token_urlsafe(32))')" - -# ngrok (auto-generates URL) -python broker.py --daemon --tunnel - -# OR Cloudflare (bring your own domain) -python broker.py --daemon --tunnel-provider cloudflare --tunnel-url https://your-domain.com - -# Prints: 🔥 Remote broker URL established: https://... - -# Terminal 2 (on remote): Set environment and run worker -export HPO_BROKER_URL="https://..." -export HPO_SECRET_TOKEN="" -python colab_worker.py -``` - -### 4. (Optional) Use IDE Integration - -Point Claude Code, Cursor, or Antigravity to the MCP server for agent-driven onboarding and reviews. See **IDE Setup** below. - ---- - -## Core Features - -### Deterministic Optimizer (Hot Path) - -- **TPE Sampler**: Probability-based hyperparameter suggestions (beats grid search) -- **ASHA Pruning**: Stop underperforming trials early to save GPU time -- **Multi-Objective Pareto**: Optimize for both Dice score *and* loss simultaneously -- **fANOVA Importances**: Which hyperparams actually matter? (→ guides your reviews) -- **No LLM calls**: Workers never block. Suggest latency is <10ms. - -### Episodic Coordinator (You Decide When) - -- Dashboard shows health warnings (nudges to review, never auto-reviews) -- 7-step review procedure: inspect data → rate health → adjust bounds if needed → submit audit trail -- Search space proposals staged for approval before taking effect -- Coordinator accuracy tracked: your reviews' forecasted score gains vs. measured deltas -- Optional LLM integration (Claude, Gemini, OpenAI) for automatic reviews - -### State Machine (SQLite) - -All configuration, trials, reviews, and metadata live in `hpo_studies.db`: - -- Active search space (not on disk) -- Trial results + VRAM telemetry -- Coordinator review history with citations -- Study health tier and dismissal states -- Generated model cards - ---- - -## For Your Own Project - -If you cloned this to tune your model (not the bridge-crack reference): - -1. **Write a manifest** (`train.hpo.yaml`): - ```yaml - study_name: my_study - metrics: - objectives: - - name: loss - direction: minimize - - name: accuracy - direction: maximize - params: - - name: learning_rate - type: float_log - min: 1e-5 - max: 1e-2 - - name: batch_size - type: categorical - options: [4, 8, 16, 32] - worker: - entrypoint: python train.py - ``` -2. **Register the study**: - ```bash - python hpo_cli.py validate train.hpo.yaml - python hpo_cli.py init train.hpo.yaml - ``` -3. **Write a worker** (use `templates/worker_minimal.py` as template): - ```python - from src.hpo_client import TrialSession - - session = TrialSession(broker_url="http://localhost:8000", study_name="my_study") - trial = session.suggest() - - for epoch in range(epochs): - accuracy, loss = train_one_epoch(trial["params"]) - should_prune = session.report_epoch(epoch, score=accuracy, loss=loss) - if should_prune: - break - - session.complete(epoch, score=accuracy, loss=loss, state="COMPLETE") - ``` -4. **Run on your GPU** (set env vars first): - ```bash - export HPO_BROKER_URL=http://localhost:8000 - export HPO_STUDY_NAME=my_study - python train.py - ``` - -Full integration walkthrough: [docs/INTEGRATION.md](docs/INTEGRATION.md) - ---- - -## IDE Setup (Agent-Driven Onboarding & Reviews) - -### Cursor - -1. **Settings → Features → MCP** -2. **+ Add New MCP Server** -3. Name: `pathfinder` - Type: `command` - Command: `source .venv/bin/activate && python3 hpo_mcp_server.py` - -### Claude Code / Antigravity - -Add to your MCP config (`~/.config/claudecode/mcp_config.json` or similar): - -```json -{ - "mcpServers": { - "pathfinder": { - "command": "python3", - "args": ["hpo_mcp_server.py"], - "env": { - "HPO_DATABASE_URL": "sqlite:///./hpo_studies.db" - } - } - } -} -``` - -Then tell any agent: - -- **"integrate HPO"** → agent drafts manifest, validates, registers study -- **"run a coordinator review"** → agent fetches study data, rates health, proposes bounds changes - -See [AGENTS.md](AGENTS.md) for the full procedure. - ---- - -## Architecture - -``` -┌─────────────────────────────────────────┐ -│ You + Optional IDE Agent │ -│ - Manual reviews or @pathfinder tools │ -│ - Dashboard inspection │ -└─────────────────────────────────────────┘ - ↕ MCP + HTTP -┌─────────────────────────────────────────┐ -│ Broker (broker.py on localhost:8000) │ -│ - Optuna TPE suggestion engine │ -│ - Trial lifecycle (/api/suggest, │ -│ /api/report_epoch, /api/complete) │ -│ - Dashboard serving │ -└─────────────────────────────────────────┘ - ↕ SQLite -┌─────────────────────────────────────────┐ -│ hpo_studies.db │ -│ - All state (search space, trials, │ -│ reviews, health, config) │ -└─────────────────────────────────────────┘ - ↕ HTTP -┌─────────────────────────────────────────┐ -│ Training Workers (Any Box) │ -│ - Colab, local GPU, cloud VM │ -│ - 3-call client API │ -│ - Never blocks on optimizer or LLM │ -└─────────────────────────────────────────┘ -``` - ---- - -## Common Commands - -```bash -# Start broker + dashboard -python broker.py --daemon - -# Validate & initialize a study from manifest -python hpo_cli.py validate train.hpo.yaml -python hpo_cli.py init train.hpo.yaml - -# Check study health -python hpo_cli.py status - -# Run a manual coordinator review (or prints prompt for copy-paste) -python hpo_cli.py review - -# Export study config back to YAML -python hpo_cli.py manifest my_study - -# Commit pending search space changes -python hpo_cli.py apply - -# Run tests -pytest tests/ -q -``` - ---- - -## Reference: Bridge Crack Segmentation (bridge-crack repo) - -This Pathfinder instance was initially tuned for [crack-seg](https://github.com/Ishaan1402/crack-seg#crack-seg), a **U-Net pixel-level crack detection model** on high-res UAV bridge imagery. See [colab_worker.py](colab_worker.py) for the full reference implementation (dataset download, model setup, training loop). - -**Don't modify `colab_worker.py`** unless you're maintaining the bridge-crack project. Cloners should use `templates/worker_minimal.py` instead. - ---- - -## Docs - -- **[AGENTS.md](AGENTS.md)** — Guide for AI agents (Claude, Cursor, Antigravity) -- **[CLAUDE.md](CLAUDE.md)** — Development commands and architecture for Claude Code -- **[examples/onboarding/](examples/onboarding/)** — Step-by-step walkthrough for a new project -- **[docs/INTEGRATION.md](docs/INTEGRATION.md)** — Worker integration contract details - ---- - -## License - -MIT License - see the [LICENSE](LICENSE) file for details. diff --git a/pathfinder.egg-info/SOURCES.txt b/pathfinder.egg-info/SOURCES.txt deleted file mode 100644 index e660f33..0000000 --- a/pathfinder.egg-info/SOURCES.txt +++ /dev/null @@ -1,45 +0,0 @@ -LICENSE -README.md -broker.py -hpo_cli.py -hpo_mcp_server.py -pyproject.toml -pathfinder.egg-info/PKG-INFO -pathfinder.egg-info/SOURCES.txt -pathfinder.egg-info/dependency_links.txt -pathfinder.egg-info/entry_points.txt -pathfinder.egg-info/requires.txt -pathfinder.egg-info/top_level.txt -src/analytics.py -src/db_manager.py -src/hpo_client.py -src/hpo_config.py -src/hpo_coordinator.py -src/hpo_daemon.py -src/leases.py -src/manifest.py -src/metrics.py -src/onboarding.py -src/pruning.py -src/reporting.py -src/schema.py -src/search_space.py -src/settings.py -src/sparklines.py -src/suggest.py -src/routers/__init__.py -src/routers/dashboard.py -src/routers/static.py -src/routers/worker.py -tests/test_concurrency.py -tests/test_coordinator_packet.py -tests/test_health_tier.py -tests/test_http_api.py -tests/test_http_auth.py -tests/test_http_concurrency.py -tests/test_integration.py -tests/test_lean_roadmap.py -tests/test_manifest.py -tests/test_metrics.py -tests/test_pruning.py -tests/test_robustness_features.py \ No newline at end of file diff --git a/pathfinder.egg-info/dependency_links.txt b/pathfinder.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789..0000000 --- a/pathfinder.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/pathfinder.egg-info/entry_points.txt b/pathfinder.egg-info/entry_points.txt deleted file mode 100644 index 13df9cf..0000000 --- a/pathfinder.egg-info/entry_points.txt +++ /dev/null @@ -1,2 +0,0 @@ -[console_scripts] -pathfinder = hpo_cli:main diff --git a/pathfinder.egg-info/requires.txt b/pathfinder.egg-info/requires.txt deleted file mode 100644 index 1e14ac7..0000000 --- a/pathfinder.egg-info/requires.txt +++ /dev/null @@ -1,15 +0,0 @@ -mcp<2,>=1.1 -optuna<5,>=3.6 -optuna-dashboard<1,>=0.15 -sqlalchemy<3,>=2.0 -psycopg2-binary<3,>=2.9 -pydantic<3,>=2.0 -numpy<3,>=1.20 -scikit-learn<2,>=1.0 -fastapi<1,>=0.110 -uvicorn<1,>=0.23 -requests<3,>=2.28 -PyYAML<7,>=6.0 - -[dev] -pytest>=8.0 diff --git a/pathfinder.egg-info/top_level.txt b/pathfinder.egg-info/top_level.txt deleted file mode 100644 index ef471f8..0000000 --- a/pathfinder.egg-info/top_level.txt +++ /dev/null @@ -1,4 +0,0 @@ -broker -hpo_cli -hpo_mcp_server -src diff --git a/pyproject.toml b/pyproject.toml index 9301dfc..5e013df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,18 +20,24 @@ classifiers = [ dependencies = [ "mcp>=1.1,<2", "optuna>=3.6,<5", - "optuna-dashboard>=0.15,<1", "sqlalchemy>=2.0,<3", "pydantic>=2.0,<3", "numpy>=1.20,<3", - "scikit-learn>=1.0,<2", "fastapi>=0.110,<1", "uvicorn>=0.23,<1", "requests>=2.28,<3", "PyYAML>=6.0,<7", ] +[tool.pytest.ini_options] +addopts = "-q" +testpaths = ["tests"] +filterwarnings = [ + "ignore::DeprecationWarning:optuna.*", + "ignore::DeprecationWarning:sqlalchemy.*", +] + [project.optional-dependencies] dev = [ "pytest>=8.0", diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 77f8fc4..0000000 --- a/pytest.ini +++ /dev/null @@ -1,7 +0,0 @@ -[pytest] -# Keep test output readable; tests are fast and DB-backed on a temp SQLite file. -addopts = -q -testpaths = tests -filterwarnings = - # Optuna + SQLAlchemy + stdlib still emit utcnow() deprecations on py3.12+; don't fail on them. - ignore::DeprecationWarning diff --git a/requirements-dev.txt b/requirements-dev.txt index 02f3772..3b41a5b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ # Development / test dependencies (not needed at runtime). -r requirements.txt pytest>=8.0 +ruff>=0.4 diff --git a/requirements.txt b/requirements.txt index 2bfe0fa..1e6aa10 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,12 +2,10 @@ # while still allowing security/patch updates. Dev/test extras live in requirements-dev.txt. mcp>=1.1,<2 optuna>=3.6,<5 -optuna-dashboard>=0.15,<1 sqlalchemy>=2.0,<3 pydantic>=2.0,<3 numpy>=1.20,<3 -scikit-learn>=1.0,<2 fastapi>=0.110,<1 uvicorn>=0.23,<1 requests>=2.28,<3 diff --git a/simulators/training_worker.py b/simulators/training_worker.py index 8b013b4..4b72681 100644 --- a/simulators/training_worker.py +++ b/simulators/training_worker.py @@ -8,18 +8,19 @@ from src.hpo_client import TrialSession -def simulate_unet_training_epoch( +def simulate_training_epoch( epoch: int, params: Dict[str, Any] ) -> tuple[float, float]: """ - Simulates a U-Net training epoch on crack segmentation. + Simulates a training epoch on a segmentation model. Defines a continuous non-linear optimization landscape: - Optimal learning rate: log10(lr) = -3 (1e-3) - Optimal BCE weight ratio: 0.3 - - Resolution 1024 captures fine details better (higher Dice), but takes longer + - Resolution 1024 captures fine details better (higher Score), but takes longer - model_capacity 'wide' performs best """ + # This simulates a BCE/Dice optimization landscape with a known optimum at lr≈1e-3, resolution=1024 # 1. LR performance (quadratic curve in log space) log_lr = np.log10(params["learning_rate"]) lr_perf = -1.5 * (log_lr - (-3.0))**2 # Peak at -3.0 (1e-3) @@ -44,21 +45,21 @@ def simulate_unet_training_epoch( loss_perf = -0.4 * (params["loss_weight_ratio"] - 0.3)**2 # Assemble base Dice score ceiling (maximum is ~0.92) - base_dice = 0.70 + lr_perf + res_perf + enc_perf + loss_perf - base_dice = max(0.15, min(0.92, base_dice)) + base_score = 0.70 + lr_perf + res_perf + enc_perf + loss_perf + base_score = max(0.15, min(0.92, base_score)) # Learning curve: approaches the ceiling asymptotically over epochs (max 10) progress = 1.0 - np.exp(-0.35 * epoch) - current_dice = base_dice * progress + current_score = base_score * progress # Add stochastic noise to simulate realistic batch training variances noise = np.random.normal(0, 0.008) - current_dice = float(max(0.01, min(0.95, current_dice + noise))) + current_score = float(max(0.01, min(0.95, current_score + noise))) # BCE Loss correlates inversely with Dice Score - current_bce = float(max(0.02, 2.5 * (1.0 - current_dice) + np.random.normal(0, 0.015))) + current_loss = float(max(0.02, 2.5 * (1.0 - current_score) + np.random.normal(0, 0.015))) - return current_dice, current_bce + return current_score, current_loss def run_training_worker( @@ -125,28 +126,28 @@ def run_training_worker( # Initialize tracking metrics val_history = [] - final_dice = 0.0 - final_bce = 999.0 + final_score = 0.0 + final_loss = 999.0 pruned = False # 2. Run Epoch training loop for epoch in range(1, epochs_per_trial + 1): # Simulate training/val forward pass - dice, bce = simulate_unet_training_epoch(epoch, params) - val_history.append({"epoch": epoch, "dice": dice, "bce": bce}) + score, loss = simulate_training_epoch(epoch, params) + val_history.append({"epoch": epoch, "score": score, "loss": loss}) - print(f" Epoch {epoch:02d}/{epochs_per_trial:02d} | Dice: {dice:.4f} | BCE: {bce:.4f}") + print(f" Epoch {epoch:02d}/{epochs_per_trial:02d} | Score: {score:.4f} | Loss: {loss:.4f}") # Record final metrics - final_dice = dice - final_bce = bce + final_score = score + final_loss = loss # 3. Intermediate epoch reporting & pruning evaluation try: should_prune = session.report_epoch( epoch=epoch, - score=dice, - loss=bce + score=score, + loss=loss ) except Exception as rep_err: print(f"Error reporting epoch: {rep_err}") @@ -159,8 +160,8 @@ def run_training_worker( try: session.complete( epoch=epoch, - score=dice, - loss=bce, + score=score, + loss=loss, state="PRUNED" ) except Exception as prune_err: @@ -176,8 +177,8 @@ def run_training_worker( try: comp_result = session.complete( epoch=epochs_per_trial, - score=final_dice, - loss=final_bce, + score=final_score, + loss=final_loss, weights_path=weights_path, history=val_history, state="COMPLETE" diff --git a/src/analytics.py b/src/analytics.py index 7d2e399..ce1b85a 100644 --- a/src/analytics.py +++ b/src/analytics.py @@ -1,25 +1,33 @@ -import math import json -from typing import List, Dict, Any, Optional +import math +import os +from typing import Any, Dict, List, Optional + +import optuna from optuna.trial import TrialState -from .metrics import get_score, get_loss + +from .db_manager import get_db_session, DATABASE_URL +from .hpo_config import load_hpo_config, param_display_name +from .metrics import get_score, get_loss, get_best_trial, score_objective_index, get_completed_trials, get_eval_attr_names +from .schema import TrialResult, SystemConfiguration, CompactedPacket, StudyCard + +logger = __import__('logging').getLogger(__name__) + def compress_loss_curve(history: List[Dict[str, Any]]) -> Dict[str, Any]: - """Compresses a raw epoch-by-epoch history curve into key indicators to save tokens.""" if not history: return {} scores = [h.get("score") for h in history if h.get("score") is not None] losses = [h.get("loss") for h in history if h.get("loss") is not None] - + res = { "initial_score": scores[0] if scores else None, "min_loss": min(losses) if losses else None, "final_score": scores[-1] if scores else None, "final_loss": losses[-1] if losses else None, - "total_epochs": len(history) + "total_epochs": len(history), } - - # Linear slope of the last 10% of epochs (convergence slope) + y = losses if losses else (scores if scores else []) if len(y) >= 2: n = max(2, int(len(y) * 0.1)) @@ -39,56 +47,54 @@ def compress_loss_curve(history: List[Dict[str, Any]]) -> Dict[str, Any]: res["convergence_slope"] = 0.0 return res + def bin_trials(study, db_metrics: Dict[int, Any], search_space: Dict[str, Any]) -> Dict[str, Any]: - """Segments trials into Elite (top 10%), Noise Floor (middle 80%), and Failure modes.""" + from .search_space import _fixed_categorical_params + fixed = _fixed_categorical_params(search_space) trials = list(study.trials) - completed_trials = [] - failed_trials = [] - + completed_trials: List[Dict[str, Any]] = [] + failed_trials: List[Dict[str, Any]] = [] + for t in trials: if t.state == TrialState.COMPLETE: - # Use generic helpers that derive indices from study directions s = get_score(t, study) score = s if s is not None else 0.0 l = get_loss(t, study) loss = l if l is not None else 0.0 - # Fetch from db_metrics if present - metric = db_metrics.get(t.number, {}) + metric = db_metrics.get(t._trial_id, {}) score = metric.get("primary_score") if metric.get("primary_score") is not None else score loss = metric.get("primary_loss") if metric.get("primary_loss") is not None else loss - + + params = {**dict(t.params), **fixed} completed_trials.append({ "trial_id": t.number, - "params": dict(t.params), + "params": params, "primary_score": score, "primary_loss": loss, - "epoch_reached": metric.get("epoch_reached", t.user_attrs.get("latest_epoch", 0)) + "epoch_reached": metric.get("epoch_reached", t.user_attrs.get("latest_epoch", 0)), }) elif t.state in (TrialState.FAIL, TrialState.PRUNED): - # Check OOM or failure status - metric = db_metrics.get(t.number, {}) + metric = db_metrics.get(t._trial_id, {}) oom = metric.get("oom_triggered") or t.user_attrs.get("oom_triggered", False) failure_tag = metric.get("failure_tag") or ("OOM" if oom else "PRUNED" if t.state == TrialState.PRUNED else "FAILED") - failed_trials.append({ "trial_id": t.number, - "params": dict(t.params), - "failure_tag": failure_tag + "params": {**dict(t.params), **fixed}, + "failure_tag": failure_tag, }) - - # Sort completed trials by score descending (higher score is better) + completed_trials.sort(key=lambda x: x["primary_score"] or 0.0, reverse=True) n_completed = len(completed_trials) - - elite = [] - noise_floor = {} - + + elite: List[Dict[str, Any]] = [] + noise_floor: Dict[str, Any] = {} + if n_completed > 0: elite_count = max(1, int(math.ceil(n_completed * 0.1))) elite = completed_trials[:elite_count] middle_trials = completed_trials[elite_count:] - + if middle_trials: scores = [t["primary_score"] for t in middle_trials if t["primary_score"] is not None] n_mid = len(scores) @@ -100,69 +106,467 @@ def bin_trials(study, db_metrics: Dict[int, Any], search_space: Dict[str, Any]) else: median_score = 0.0 variance = 0.0 - + param_ranges = {} - # Gather all param names in search space param_names = set(search_space.keys()) for t in middle_trials: param_names.update(t["params"].keys()) - for p_name in param_names: p_vals = [t["params"].get(p_name) for t in middle_trials if t["params"].get(p_name) is not None] if not p_vals: continue try: - # Numeric ranges numeric_vals = [float(v) for v in p_vals] param_ranges[p_name] = [min(numeric_vals), max(numeric_vals)] except (ValueError, TypeError): - # Categorical counts - counts = {} + counts: Dict[str, int] = {} for v in p_vals: counts[str(v)] = counts.get(str(v), 0) + 1 param_ranges[p_name] = counts - + noise_floor = { "count": len(middle_trials), "median_score": median_score, "score_variance": variance, - "param_ranges": param_ranges + "param_ranges": param_ranges, } else: - noise_floor = { - "count": 0, - "median_score": 0.0, - "score_variance": 0.0, - "param_ranges": {} - } - - # Aggregated failure/OOM modes count matrix - failure_matrix = {} + noise_floor = {"count": 0, "median_score": 0.0, "score_variance": 0.0, "param_ranges": {}} + + failure_matrix: Dict[str, int] = {} for t in failed_trials: - # Construct combinations of parameters for categorization - # Focus on key hyperparams: batch_size, resolution, lr if they exist, or just all params sorted important_params = ["batch_size", "resolution", "lr"] key_parts = [] for p in important_params: if p in t["params"]: key_parts.append(f"{p}={t['params'][p]}") - if not key_parts: - # Fallback to sorting all params key_parts = [f"{k}={v}" for k, v in sorted(t["params"].items())] - param_key = " && ".join(key_parts) if key_parts else "unknown_params" tag = t["failure_tag"] - full_key = f"{param_key} [{tag}]" failure_matrix[full_key] = failure_matrix.get(full_key, 0) + 1 - + + return {"elite": elite, "noise_floor": noise_floor, "failure_matrix": failure_matrix} + + +# --- Train-resolution helper (shared with pruning/pareto) --- + +def trial_train_resolution(trial, train_param: str) -> Optional[int]: + val = trial.params.get(train_param) + if val is None: + return None + try: + return int(val) + except (TypeError, ValueError): + return None + + +# --- fANOVA --- + +def get_fanova_importances(study, config: Dict[str, Any]) -> Dict[str, float]: + complete = get_completed_trials(study) + if len(complete) < 2: + return {} + importances: Dict[str, float] = {} + try: + if len(study.directions) > 1: + _si = score_objective_index(study) + if _si is not None: + importances = optuna.importance.get_param_importances( + study, + target=lambda t, idx=_si: t.values[idx] if (t.values and len(t.values) > idx) else None, + evaluator=optuna.importance.FanovaImportanceEvaluator(), + ) + else: + importances = optuna.importance.get_param_importances( + study, evaluator=optuna.importance.FanovaImportanceEvaluator() + ) + except Exception: + return {} + + aliases = config.get("legacy_param_aliases", {}) + display: Dict[str, float] = {} + for param, value in importances.items(): + canonical = aliases.get(param, param) + label = param_display_name(canonical, config) + display[label] = max(display.get(label, 0.0), float(value)) + return display + + +# --- Pareto --- + +def pareto_trial_numbers_deploy_aware(study, hpo_config: Dict[str, Any]) -> List[int]: + ev = hpo_config.get("eval_protocol", {}) + train_param = ev.get("train_resolution_param", "resolution") + low_warn = ev.get("low_train_res_warning") + low_warn = int(low_warn) if low_warn is not None else None + deploy_only = ev.get("pareto_deploy_resolution_only", True) + score_fixed_key = ev.get("fixed_score_attr", "score_eval_fixed") + + points: List[tuple] = [] + for t in study.trials: + if t.state != TrialState.COMPLETE: + continue + loss_val = get_loss(t, study) + score_val = get_score(t, study) + if loss_val is None or score_val is None: + continue + train_res = trial_train_resolution(t, train_param) + if deploy_only and low_warn is not None and train_res is not None and train_res < low_warn: + continue + if ev.get("enabled"): + fd = t.user_attrs.get(score_fixed_key) + if fd is not None: + score_val = float(fd) + points.append((t.number, float(loss_val), float(score_val))) + + if not points: + try: + return [t.number for t in study.best_trials] + except Exception: + return [] + + pareto: List[int] = [] + for num_i, loss_i, score_i in points: + dominated = False + for num_j, loss_j, score_j in points: + if num_i == num_j: + continue + if loss_j <= loss_i and score_j >= score_i and (loss_j < loss_i or score_j > score_i): + dominated = True + break + if not dominated: + pareto.append(num_i) + return pareto + + +# --- Boundary hits --- + +def check_boundary_hits(study, pareto_numbers: List[int], search_space: Dict[str, Any]) -> Dict[str, Any]: + hits: Dict[str, Any] = {} + pareto_trials = [t for t in study.trials if t.number in pareto_numbers and t.state == TrialState.COMPLETE] + n_pareto = len(pareto_trials) + if n_pareto == 0: + return hits + + for p_name, p_info in search_space.items(): + p_type = p_info.get("type", "") + if p_type not in ("float", "float_log", "int"): + continue + s_min = p_info.get("min") + s_max = p_info.get("max") + if s_min is None or s_max is None or s_max <= s_min: + continue + s_min, s_max = float(s_min), float(s_max) + margin = 0.1 * (s_max - s_min) + near_min_count = 0 + near_max_count = 0 + for t in pareto_trials: + val = t.params.get(p_name) + if val is not None: + val = float(val) + if val <= s_min + margin: + near_min_count += 1 + if val >= s_max - margin: + near_max_count += 1 + total_hits = near_min_count + near_max_count + ratio = total_hits / n_pareto + if ratio > 0.6: + hits[p_name] = { + "near_min_count": near_min_count, + "near_max_count": near_max_count, + "total_pareto": n_pareto, + "hit_ratio": ratio, + "bound_hit": "min" if near_min_count > near_max_count else "max" if near_max_count > near_min_count else "both", + } + return hits + + +# --- Fidelity durations --- + +def compute_fidelity_durations(study, config: Dict[str, Any]) -> Dict[str, Any]: + ev = config.get("eval_protocol", {}) + train_param = ev.get("train_resolution_param", "resolution") + groups: Dict[int, List] = {} + for t in study.trials: + if t.state != TrialState.COMPLETE: + continue + val = t.params.get(train_param) + if val is None: + continue + try: + val = int(val) + except (TypeError, ValueError): + continue + groups.setdefault(val, []).append(t) + + res_stats: Dict[int, Dict[str, Any]] = {} + for val, trials in groups.items(): + durations = [] + epoch_durations = [] + for t in trials: + if t.datetime_start and t.datetime_complete: + dur = (t.datetime_complete - t.datetime_start).total_seconds() + durations.append(dur) + history = t.user_attrs.get("history", []) + epochs = len(history) if history else t.user_attrs.get("latest_epoch") + if not epochs: + epochs = max([h.get("epoch", 1) for h in history] or [1]) + if epochs > 0: + epoch_durations.append(dur / epochs) + if durations: + res_stats[val] = { + "avg_total_duration": sum(durations) / len(durations), + "avg_epoch_duration": sum(epoch_durations) / len(epoch_durations) if epoch_durations else None, + "count": len(durations), + } + + if not res_stats: + return {} + lowest_scale = min(res_stats.keys()) + base_dur = res_stats[lowest_scale]["avg_total_duration"] + for val, stats in res_stats.items(): + if base_dur > 0: + stats["overhead_ratio"] = stats["avg_total_duration"] / base_dur + else: + stats["overhead_ratio"] = 1.0 + return {"fidelity_param": train_param, "lowest_scale": lowest_scale, "scales": res_stats} + + +# --- VRAM telemetry --- + +def fit_vram_model(trials: List, db_metrics: Dict[int, Any], train_param: str) -> Optional[Dict[str, Any]]: + points: List[tuple] = [] + for t in trials: + if t.state != TrialState.COMPLETE: + continue + metric = db_metrics.get(t._trial_id, {}) + oom = metric.get("oom_triggered") or t.user_attrs.get("oom_triggered", False) + if oom: + continue + vram = metric.get("max_vram_gb") or t.user_attrs.get("max_vram_gb") + if vram is None: + continue + bs = t.params.get("batch_size") + res = t.params.get(train_param) + if bs is not None and res is not None: + try: + points.append((float(bs), float(res), float(vram))) + except (ValueError, TypeError): + continue + + if len(points) < 6: + return None + + X = [p[0] * (p[1] ** 2) for p in points] + Y = [p[2] for p in points] + if len(set(X)) < 2: + return None + + n = len(points) + sum_x = sum(X) + sum_y = sum(Y) + sum_xx = sum(x * x for x in X) + sum_xy = sum(X[i] * Y[i] for i in range(n)) + denom = n * sum_xx - sum_x * sum_x + if abs(denom) < 1e-12: + return None + + slope = (n * sum_xy - sum_x * sum_y) / denom + intercept = (sum_y - slope * sum_x) / n + ssr = sum((Y[i] - (slope * X[i] + intercept)) ** 2 for i in range(n)) + rse = (ssr / (n - 2)) ** 0.5 if n > 2 else 0.0 + return {"slope": slope, "intercept": intercept, "n_points": n, "rse": rse} + + +def compute_vram_telemetry(study, db_metrics: Dict[int, Any], search_space: Dict[str, Any], config: Dict[str, Any]) -> Dict[str, Any]: + ev = config.get("eval_protocol", {}) + train_param = ev.get("train_resolution_param", "resolution") + trials = list(study.trials) + model = fit_vram_model(trials, db_metrics, train_param) + + gpu_capacity_gb = 0.0 + gpu_models: List[str] = [] + oom_count = 0 + + for t in trials: + metric = db_metrics.get(t._trial_id, {}) + vram = metric.get("max_vram_gb") or t.user_attrs.get("max_vram_gb") + gpu = metric.get("gpu_model") or t.user_attrs.get("gpu_model") + oom = metric.get("oom_triggered") or t.user_attrs.get("oom_triggered", False) + if vram: + gpu_capacity_gb = max(gpu_capacity_gb, float(vram)) + if gpu: + gpu_models.append(gpu) + if oom: + oom_count += 1 + + gpu_model = max(set(gpu_models), key=gpu_models.count) if gpu_models else "Unknown" + oom_risk = None + + if model and gpu_capacity_gb > 0: + max_bs = None + bs_info = search_space.get("batch_size", {}) + if bs_info.get("type") == "categorical": + active_bs = bs_info.get("active", []) + if active_bs: + max_bs = max(active_bs) + else: + max_bs = bs_info.get("max") + + max_res = None + res_info = search_space.get(train_param, {}) + if res_info.get("type") == "categorical": + active_res = res_info.get("active", []) + if active_res: + max_res = max(active_res) + else: + max_res = res_info.get("max") + + if max_bs is not None and max_res is not None: + predicted_mean_vram = model["slope"] * (float(max_bs) * (float(max_res) ** 2)) + model["intercept"] + margin = max(1.0, 1.96 * model["rse"]) + predicted_max_vram = predicted_mean_vram + margin + if predicted_max_vram > 0.9 * gpu_capacity_gb: + oom_risk = { + "max_batch_size": max_bs, + "max_resolution": max_res, + "predicted_mean_vram_gb": predicted_mean_vram, + "margin_gb": margin, + "predicted_max_vram_gb": predicted_max_vram, + "gpu_capacity_gb": gpu_capacity_gb, + "risk_level": "high" if predicted_max_vram > gpu_capacity_gb else "medium", + } + return { - "elite": elite, - "noise_floor": noise_floor, - "failure_matrix": failure_matrix + "gpu_model": gpu_model, + "gpu_capacity_gb": gpu_capacity_gb, + "oom_count": oom_count, + "vram_model": model, + "bounds_oom_risk": oom_risk, } + +# --- Eval insights --- + +def study_eval_insights(study, config: Dict[str, Any]) -> Dict[str, Any]: + ev = config.get("eval_protocol", {}) + train_param = ev.get("train_resolution_param", "resolution") + fixed_res = ev.get("fixed_resolution") + low_warn = ev.get("low_train_res_warning") + score_fixed_key, _ = get_eval_attr_names(ev) + + complete = get_completed_trials(study) + by_res: Dict[int, List] = {} + warnings: List[Dict[str, Any]] = [] + + for t in complete: + tr = t.params.get(train_param) + if tr is not None: + by_res.setdefault(int(tr), []).append(t) + + res_summary: Dict[int, Dict[str, Any]] = {} + for res, trials in sorted(by_res.items()): + scores = [get_score(t, study) for t in trials] + scores = [s for s in scores if s is not None] + fixed_scores = [t.user_attrs.get(score_fixed_key) for t in trials if t.user_attrs.get(score_fixed_key) is not None] + res_summary[res] = { + "count": len(trials), + "best_score_train": max(scores) if scores else None, + "best_score_fixed": max(fixed_scores) if fixed_scores else None, + } + + valid_deploy_exists = False + if low_warn is not None: + for t in complete: + tr = t.params.get(train_param) + fd = t.user_attrs.get(score_fixed_key) + if tr is not None and int(tr) >= int(low_warn) and fd is not None: + valid_deploy_exists = True + break + + if complete and ev.get("enabled") and fixed_res: + best_train = get_best_trial(complete, study) + if best_train is None: + best_train = complete[0] + train_res = best_train.params.get(train_param) + if train_res is not None and low_warn and int(train_res) < int(low_warn) and not valid_deploy_exists: + warnings.append({ + "code": "low_train_res_pareto", + "trial_number": best_train.number, + "message": f"Pareto-best trial #{best_train.number} trained at scale {train_res}, below warning threshold {low_warn}. Check fixed eval.", + }) + fd = best_train.user_attrs.get(score_fixed_key) + td = get_score(best_train, study) + if fd is not None and td is not None and (td - fd) > 0.08: + warnings.append({ + "code": "train_eval_gap", + "trial_number": best_train.number, + "message": f"Trial #{best_train.number}: train score {td:.3f} vs fixed-eval score {fd:.3f} — train scale/resolution may be inflating scores.", + }) + + best_deploy: Any = None + if ev.get("enabled"): + ranked = [t for t in complete if t.user_attrs.get(score_fixed_key) is not None] + if ranked: + best_deploy = max(ranked, key=lambda t: t.user_attrs.get(score_fixed_key)) + + return { + "resolution_summary": res_summary, + "warnings": warnings, + "best_deploy_trial_number": best_deploy.number if best_deploy else None, + "best_deploy_score_fixed": best_deploy.user_attrs.get(score_fixed_key) if best_deploy else None, + } + + +# --- Prune rate clusters --- + +def compute_prune_rate_clusters(study, search_space: Dict[str, Any]) -> Dict[str, Any]: + clusters: Dict[str, Any] = {} + continuous_params = [] + for p_name, p_info in search_space.items(): + if p_info.get("type", "") in ("float", "float_log", "int"): + continuous_params.append(p_name) + + for p_name in continuous_params: + vals: List[float] = [] + states: List[TrialState] = [] + for t in study.trials: + if t.state in (TrialState.COMPLETE, TrialState.PRUNED) and p_name in t.params: + val = t.params[p_name] + if val is not None: + vals.append(float(val)) + states.append(t.state) + if not vals: + continue + v_min, v_max = min(vals), max(vals) + if v_max <= v_min: + continue + w = (v_max - v_min) / 3.0 + bins = [ + {"min": v_min, "max": v_min + w, "total": 0, "pruned": 0}, + {"min": v_min + w, "max": v_min + 2 * w, "total": 0, "pruned": 0}, + {"min": v_min + 2 * w, "max": v_max, "total": 0, "pruned": 0}, + ] + for val, state in zip(vals, states): + if val <= bins[0]["max"]: + bin_idx = 0 + elif val <= bins[1]["max"]: + bin_idx = 1 + else: + bin_idx = 2 + bins[bin_idx]["total"] += 1 + if state == TrialState.PRUNED: + bins[bin_idx]["pruned"] += 1 + for b in bins: + b["prune_rate"] = b["pruned"] / b["total"] if b["total"] > 0 else 0.0 + clusters[p_name] = bins + return clusters + + +# --- Study packet assembly --- + def build_compacted_packet( study_name: str, study, @@ -171,24 +575,11 @@ def build_compacted_packet( config: Dict[str, Any], health_tier: str, health_reason: Optional[str], - past_reviews: List[Dict[str, Any]], - accuracy_stats: Dict[str, Any], project_context: Dict[str, Any], statistical_confidence: str = "low", ) -> Dict[str, Any]: - """Assembles a highly compressed, token-efficient HPO review packet.""" - from .hpo_coordinator import ( - get_fanova_importances, - pareto_trial_numbers_deploy_aware, - check_boundary_hits, - compute_fidelity_durations, - compute_vram_telemetry, - compute_spearman_rank_correlation - ) - trials = list(study.trials) - - # Study status counts + counts = { "total": len(trials), "complete": sum(1 for t in trials if t.state == TrialState.COMPLETE), @@ -196,110 +587,122 @@ def build_compacted_packet( "failed": sum(1 for t in trials if t.state == TrialState.FAIL), "running": sum(1 for t in trials if t.state == TrialState.RUNNING), } - - # 1. Compacted Trial Bins + trial_bins = bin_trials(study, db_metrics, search_space) - - # 2. fANOVA Importances (Top 5 only) + try: raw_importances = get_fanova_importances(study, config) - except Exception as e: + except Exception: raw_importances = {} sorted_importances = sorted(raw_importances.items(), key=lambda x: x[1], reverse=True)[:5] top_fanova = dict(sorted_importances) - - # 3. Spearman correlations (with confidence tags, numeric params only) - spearman_correlations = {} - complete_trials = [t for t in trials if t.state == TrialState.COMPLETE] - if complete_trials: - dice_fixed_key = config.get("eval_protocol", {}).get("fixed_dice_attr", "dice_eval_fixed") - for p_name, p_info in search_space.items(): - # Only analyze numeric parameters - if p_info.get("type") not in ("float", "float_log", "int"): - continue - - paired_x = [] - paired_y = [] - try: - for t in complete_trials: - val = t.params.get(p_name) - if val is None: - continue - - # Score lookup - fd = t.user_attrs.get(dice_fixed_key) - if fd is not None: - score_val = float(fd) - else: - s = get_score(t, study) - score_val = float(s) if s is not None else 0.0 - - paired_x.append(float(val)) - paired_y.append(score_val) - except (ValueError, TypeError): - continue - - if len(set(paired_x)) > 1 and len(paired_x) >= 3: - n_samples = len(paired_x) - confidence = "Low" if n_samples < 8 else "Moderate" if n_samples < 15 else "High" - corr_coef = compute_spearman_rank_correlation(paired_x, paired_y) - spearman_correlations[p_name] = { - "coefficient": round(corr_coef, 4), - "n_samples": n_samples, - "confidence": confidence - } - - # 4. Boundary hits (Pareto-adjacent params only) - pareto_numbers = ( - pareto_trial_numbers_deploy_aware(study, config) if len(study.directions) > 1 else [] - ) - raw_boundary_hits = check_boundary_hits(study, pareto_numbers, search_space) - # Filter to only params that actually hit boundaries (near_min or near_max) - boundary_hits = {k: v for k, v in raw_boundary_hits.items() if v.get("hit_ratio", 0) > 0.0} - # 5. Fidelity durations + pareto_numbers = pareto_trial_numbers_deploy_aware(study, config) if len(study.directions) > 1 else [] + boundary_hits = check_boundary_hits(study, pareto_numbers, search_space) fidelity_durations = compute_fidelity_durations(study, config) - - # 6. VRAM Telemetry (with RSE prediction intervals) vram_telemetry = compute_vram_telemetry(study, db_metrics, search_space, config) - # 7. Past 3 Reviews (Summary + Action + Rating only, not full text if long) - compact_reviews = [] - for r in past_reviews[:3]: - summary_lines = r.get("summary", "").split("\n") - short_summary = summary_lines[0] if summary_lines else "" - if len(r.get("summary", "")) > 150: - short_summary = r.get("summary", "")[:147] + "..." - - compact_reviews.append({ - "id": r.get("id"), - "created_at": r.get("created_at"), - "health_rating": r.get("health_rating"), - "policy_action": r.get("policy_action"), - "trials_evaluated": r.get("trials_evaluated"), - "estimated_score_improvement": r.get("estimated_score_improvement"), - "quality_flagged": r.get("quality_flagged", False), - "outcome_status": r.get("outcome_status"), - "summary": short_summary - }) - return { "study_name": study_name, "counts": counts, "project_context": project_context, - "health": { - "tier": health_tier, - "reason": health_reason - }, + "health": {"tier": health_tier, "reason": health_reason}, "trial_bins": trial_bins, "fanova_importances": top_fanova, - "spearman_correlations": spearman_correlations, "boundary_hits": boundary_hits, "fidelity_durations": fidelity_durations, "vram_telemetry": vram_telemetry, - "past_reviews": compact_reviews, - "coordinator_accuracy": accuracy_stats, "statistical_confidence": statistical_confidence, "metric_score_label": config.get("metric_score_label", "Score"), - "metric_loss_label": config.get("metric_loss_label", "Loss") + "metric_loss_label": config.get("metric_loss_label", "Loss"), } + + +def build_study_packet(study_name: str) -> Dict[str, Any]: + try: + from .health import compute_health_tier, compute_statistical_confidence, count_evaluated_trials + + study = optuna.load_study(study_name=study_name, storage=DATABASE_URL) + n_eval = count_evaluated_trials(study) + + with get_db_session() as session: + cached = session.query(CompactedPacket).filter_by( + study_name=study_name, trials_evaluated=n_eval + ).first() + if cached: + try: + packet = json.loads(cached.packet_json) + n_complete = len(get_completed_trials(study)) + packet["statistical_confidence"] = compute_statistical_confidence(n_complete) + return packet + except Exception as e: + logger.warning(f"Failed to load cached packet for {study_name}: {e}") + + with get_db_session() as session: + db_metrics: Dict[int, Any] = {} + rows = session.query(TrialResult).filter_by(study_name=study_name).all() + for r in rows: + db_metrics[r.trial_id] = r.to_dict() + + from .search_space import load_search_space + search_space = load_search_space(study_name) + config = load_hpo_config(study_name) + + project_context: Dict[str, Any] = {} + context_row = session.query(SystemConfiguration).filter_by( + study_name=study_name, config_key="project_context" + ).first() + if context_row: + try: + project_context = json.loads(context_row.config_value) + except Exception as e: + logger.warning(f"Failed to parse project_context for {study_name}: {e}") + + health_tier, health_reason = compute_health_tier(study, study_name) + + n_complete = len(get_completed_trials(study)) + statistical_confidence = compute_statistical_confidence(n_complete) + + packet = build_compacted_packet( + study_name, study, db_metrics, search_space, config, + health_tier, health_reason, project_context, statistical_confidence, + ) + + with get_db_session() as session: + session.merge(CompactedPacket( + study_name=study_name, + trials_evaluated=n_eval, + packet_json=json.dumps(packet), + )) + + packet["statistical_confidence"] = statistical_confidence + return packet + except Exception as e: + import traceback + traceback.print_exc() + return {"success": False, "error": f"Failed to build study packet: {str(e)}"} + + +# --- Study cards --- + +def load_study_cards(study_name: Optional[str] = None) -> List[Dict[str, Any]]: + with get_db_session() as session: + query = session.query(StudyCard) + if study_name: + query = query.filter_by(study_name=study_name) + cards = query.all() + + result: List[Dict[str, Any]] = [] + for c in cards: + card_dict = c.to_dict() + full_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), c.file_path) + if os.path.exists(full_path): + try: + with open(full_path, "r") as f: + card_dict["markdown_content"] = f.read() + except Exception as e: + logger.warning(f"Failed to read study card {full_path}: {e}") + else: + card_dict["markdown_content"] = "" + result.append(card_dict) + return result diff --git a/src/db_manager.py b/src/db_manager.py index fee7634..8458234 100644 --- a/src/db_manager.py +++ b/src/db_manager.py @@ -1,6 +1,7 @@ import contextlib import logging +import os from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker, Session from .schema import Base @@ -28,10 +29,17 @@ def set_sqlite_pragma(dbapi_connection, connection_record): cursor.execute("PRAGMA synchronous=NORMAL") cursor.execute("PRAGMA busy_timeout=30000") cursor.close() -SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) +SessionLocal = sessionmaker(bind=engine) def init_db(): """Initializes the database, creates tables, and runs additive migrations.""" + # Ensure parent directory exists for the default SQLite path (.data/) + if DATABASE_URL.startswith("sqlite:///"): + db_path = DATABASE_URL.replace("sqlite:///", "") + db_dir = os.path.dirname(db_path) + if db_dir and not os.path.isdir(db_dir): + os.makedirs(db_dir, exist_ok=True) + from sqlalchemy import inspect try: @@ -74,27 +82,11 @@ def init_db(): "health_tier": "VARCHAR(50)", "health_reason": "TEXT", }, - "study_reviews": { - "estimated_score_improvement": "FLOAT", - "cited_best_trial": "INTEGER", - "confidence": "VARCHAR(50) DEFAULT 'high'", - "baseline_best_score": "FLOAT", - "applied_at_completed_count": "INTEGER", - "applied_at": "DATETIME", - "actual_score_improvement": "FLOAT", - "outcome_measured_at": "DATETIME", - "outcome_status": "VARCHAR(30) DEFAULT 'pending'", - "quality_flagged": "BOOLEAN DEFAULT 0", - }, - "agent_reasoning_logs": { - "estimated_score_improvement": "FLOAT", - "actual_score_improvement": "FLOAT", - }, + "study_status": { "health_tier": "VARCHAR(50) DEFAULT 'healthy'", "health_reason": "TEXT", "health_updated_at": "DATETIME", - "nudge_dismissed_trials": "INTEGER", } } @@ -119,44 +111,12 @@ def _apply_additive_migrations(): if col_name in present: continue try: - # Check for old column name to copy data - old_name = None - if col_name == "estimated_score_improvement" and "estimated_dice_improvement" in present: - old_name = "estimated_dice_improvement" - elif col_name == "actual_score_improvement" and "actual_dice_improvement" in present: - old_name = "actual_dice_improvement" - with engine.begin() as conn: conn.execute(text(f'ALTER TABLE {table} ADD COLUMN {col_name} {col_type}')) - if old_name: - conn.execute(text(f'UPDATE {table} SET {col_name} = {old_name}')) except Exception as e: # Best-effort: a concurrent process may have added it already. print(f"Error migrating column {col_name} in {table}: {e}") - # Drop obsolete columns after confirming new columns are present and data is copied - for table, col_map in [ - ("agent_reasoning_logs", {"estimated_dice_improvement": "estimated_score_improvement", "actual_dice_improvement": "actual_score_improvement"}), - ("study_reviews", {"estimated_dice_improvement": "estimated_score_improvement", "actual_dice_improvement": "actual_score_improvement"}), - ]: - if table in existing_tables: - try: - present = {c["name"] for c in inspector.get_columns(table)} - except Exception: - continue - for old_col, new_col in col_map.items(): - if old_col in present: - try: - with engine.begin() as conn: - # If new column exists, migrate remaining NULL values if any - if new_col in present: - conn.execute(text(f"UPDATE {table} SET {new_col} = {old_col} WHERE {new_col} IS NULL")) - # Drop the obsolete column - conn.execute(text(f"ALTER TABLE {table} DROP COLUMN {old_col}")) - print(f"Migration: Dropped obsolete column '{old_col}' from table '{table}'") - except Exception as drop_err: - print(f"Migration: Error dropping obsolete column '{old_col}' from table '{table}': {drop_err}") - def _migrate_segmentation_metrics_to_trial_results(): from sqlalchemy import inspect, text @@ -212,7 +172,7 @@ def _migrate_segmentation_metrics_to_trial_results(): SELECT :study_name, {select_clause} FROM segmentation_metrics """ conn.execute(text(stmt), {"study_name": study_name}) - print(f"Successfully migrated data from segmentation_metrics to trial_results.") + print("Successfully migrated data from segmentation_metrics to trial_results.") except Exception as e: print(f"Error migrating segmentation_metrics data to trial_results: {e}") diff --git a/src/health.py b/src/health.py new file mode 100644 index 0000000..1671d2e --- /dev/null +++ b/src/health.py @@ -0,0 +1,150 @@ +"""Study health monitoring — health tier computation, statistical confidence, status file. + +This module replaces the health-monitoring portions of the deleted hpo_coordinator.py. +It contains no LLM-calling code, no review persistence, no coordinator logic. It is a +deterministic, read-only health assessment layer imported by the broker, daemon, and CLI. +""" + +import datetime +import json +import math +from typing import Dict, List, Optional, Tuple + +from optuna.trial import TrialState + +from .db_manager import get_db_session +from .metrics import get_score, get_completed_trials, TERMINAL_STATES + +_MIN_COMPLETED_FOR_FIRST_REVIEW = 5 + + +def compute_statistical_confidence(n_complete: int) -> str: + if n_complete < 10: + return "low" + if n_complete < 20: + return "medium" + return "high" + + +def count_evaluated_trials(study) -> int: + return sum(1 for t in study.trials if t.state in TERMINAL_STATES) + + +def compute_health_tier(study, study_name: str) -> Tuple[str, Optional[str]]: + + trials = list(study.trials) + finished = sorted([t for t in trials if t.state in TERMINAL_STATES], key=lambda t: t.number) + completed = sorted(get_completed_trials(study), key=lambda t: t.number) + + # === Intervene triggers === + + for t in trials: + if t.values: + for v in t.values: + if v is not None and (math.isnan(v) or math.isinf(v)): + return "intervene", f"NaN or Inf detected in reported metrics for Trial #{t.number}" + + try: + from .schema import TrialResult + with get_db_session() as session: + oom_trials = session.query(TrialResult).filter_by(study_name=study_name, oom_triggered=True).all() + if len(oom_trials) >= 2: + trial_params_map = {t._trial_id: t.params for t in study.trials} + oom_combos: Dict[tuple, int] = {} + for r in oom_trials: + params = trial_params_map.get(r.trial_id) + if params: + key = tuple(sorted((k, str(v)) for k, v in params.items())) + oom_combos[key] = oom_combos.get(key, 0) + 1 + if oom_combos[key] >= 2: + params_desc = ", ".join(f"{k}={v}" for k, v in key) + return "intervene", f"OOM cluster detected: parameter combination ({params_desc}) failed with OOM 2+ times" + except Exception as e: + import logging + logging.getLogger(__name__).warning(f"Error checking OOM clusters: {e}") + + if len(completed) >= 5: + improvements: List[int] = [] + best_so_far = -float("inf") + for i, t in enumerate(completed): + score = get_score(t, study) + if score is None: + continue + if score > best_so_far + 1e-4: + best_so_far = score + improvements.append(i) + if len(improvements) >= 2: + intervals = [improvements[j] - improvements[j - 1] for j in range(1, len(improvements))] + avg_interval = sum(intervals) / len(intervals) + trials_since = len(completed) - 1 - improvements[-1] + threshold = max(4, int(math.ceil(2 * avg_interval))) + if trials_since >= threshold: + return "intervene", f"Score stagnation: no improvement over last {trials_since} completed trials (average improvement interval is {avg_interval:.1f} trials, threshold is {threshold})" + + if len(completed) >= 4: + from .hpo_config import load_hpo_config + config = load_hpo_config(study_name) + score_fixed_key = config.get("eval_protocol", {}).get("fixed_score_attr", "score_eval_fixed") + gaps: List[float] = [] + for t in completed: + fd = t.user_attrs.get(score_fixed_key) + td = get_score(t, study) + if fd is not None and td is not None: + gaps.append(float(td) - float(fd)) + if len(gaps) >= 4: + mean_gap = sum(gaps) / len(gaps) + var_gap = sum((g - mean_gap) ** 2 for g in gaps) / len(gaps) + std_gap = var_gap ** 0.5 + if std_gap > 0 and gaps[-1] > mean_gap + 2 * std_gap: + return "intervene", f"Train-eval gap anomaly: latest trial gap ({gaps[-1]:.4f}) exceeds 2 standard deviations of historical gap distribution (mean={mean_gap:.4f}, std={std_gap:.4f}, threshold={mean_gap + 2 * std_gap:.4f})" + + # === Watch triggers === + + if len(finished) >= 5: + recent_finished = finished[-5:] + pruned_count = sum(1 for t in recent_finished if t.state == TrialState.PRUNED) + if pruned_count >= 4: + return "watch", f"High prune rate: {pruned_count}/5 ({pruned_count * 20}%) of recent trials were pruned" + + if len(completed) >= 4: + scores = [get_score(t, study) for t in completed] + scores = [s for s in scores if s is not None] + scores.sort(reverse=True) + top_count = max(1, len(scores) // 4) + top_scores = scores[:top_count] + if len(top_scores) >= 2: + mean_top = sum(top_scores) / len(top_scores) + var_top = sum((x - mean_top) ** 2 for x in top_scores) / len(top_scores) + if var_top < 1e-4: + return "watch", f"Score convergence: top quartile score variance ({var_top:.6f}) is below 1e-4" + + running_trials = [t for t in trials if t.state == TrialState.RUNNING] + latest_completed = completed[-1:] if completed else [] + for t in (running_trials + latest_completed): + t_tier = t.user_attrs.get("health_tier") + t_reason = t.user_attrs.get("health_reason") + if t_tier == "watch" and t_reason: + return "watch", f"Trial #{t.number} warning: {t_reason}" + + return "healthy", "No issues detected. Search space is healthy." + + +def write_ide_status_file(study_name: str, health_tier: str, health_reason: str, study) -> None: + trials_evaluated = count_evaluated_trials(study) + payload = { + "study_name": study_name, + "health_tier": health_tier.lower(), + "health_reason": health_reason, + "trials_evaluated": trials_evaluated, + "review_recommended": health_tier.lower() in ("watch", "intervene"), + "last_updated": datetime.datetime.now(datetime.timezone.utc).isoformat(), + } + from pathlib import Path + root_dir = Path(__file__).resolve().parent.parent + status_file_path = root_dir / ".hpo_status.json" + try: + with open(status_file_path, "w") as f: + json.dump(payload, f, indent=4) + except Exception as e: + import logging + logging.getLogger(__name__).warning(f"Error writing .hpo_status.json: {e}") diff --git a/src/hpo_client.py b/src/hpo_client.py index fa11076..e777b31 100644 --- a/src/hpo_client.py +++ b/src/hpo_client.py @@ -3,10 +3,9 @@ This is the entire contract a worker needs to participate in a study: suggest -> report_epoch (per epoch) -> complete -It has no ML / framework dependencies (no torch, cv2, DeepCrack, or UNet). Bring your own -training loop and call these three methods. The root `colab_worker.py` is the full -bridge-crack reference implementation; cloners should use this client plus -`templates/worker_minimal.py` instead of forking that file. +It has no ML / framework dependencies (no torch, cv2, or model-specific code). Bring your own +training loop and call these three methods. Use `templates/worker_minimal.py` as a starting +point for your own project. Environment: HPO_BROKER_URL Base URL of the broker (e.g. an ngrok tunnel). Required for HTTP mode. @@ -18,12 +17,12 @@ session = TrialSession() # reads HPO_BROKER_URL / HPO_STUDY_NAME trial = session.suggest() # {trial_id, trial_number, params} for epoch in range(num_epochs): - dice, bce = train_one_epoch(trial["params"]) - if session.report_epoch(epoch, dice, bce): # True => broker says prune - session.complete(epoch, dice, bce, state="PRUNED") + score, loss = train_one_epoch(trial["params"]) + if session.report_epoch(epoch, score, loss): # True => broker says prune + session.complete(epoch, score, loss, state="PRUNED") break else: - session.complete(epoch, dice, bce, weights_path="model.pt", history=session.history) + session.complete(epoch, score, loss, weights_path="model.pt", history=session.history) """ import os from typing import Any, Dict, List, Optional @@ -332,15 +331,11 @@ def report_epoch( "epoch": epoch, "score": score, "loss": loss, - # Backwards compatibility keys for UI / charts - "dice": score, - "bce": loss, } if score_eval_fixed is not None: entry["score_eval_fixed"] = score_eval_fixed - entry["dice_eval_fixed"] = score_eval_fixed + if loss_eval_fixed is not None: entry["loss_eval_fixed"] = loss_eval_fixed - entry["bce_eval_fixed"] = loss_eval_fixed self.history.append(entry) data = self._post("/api/report_epoch", payload) diff --git a/src/hpo_config.py b/src/hpo_config.py index cd5fb74..06babfd 100644 --- a/src/hpo_config.py +++ b/src/hpo_config.py @@ -17,7 +17,6 @@ "metric_loss_label": "Loss", "metric_score_label": "Score", "metric_names": {"score": "score", "loss": "loss"}, - "desktop_notifications_enabled": False, "validation_rules": { "score_min": None, "loss_min": None, @@ -28,10 +27,10 @@ "enabled": False, "fixed_resolution": None, "train_resolution_param": "resolution", - "fixed_dice_attr": "score_eval_fixed", - "fixed_bce_attr": "loss_eval_fixed", - "dice_train_label": "Score (train)", - "dice_fixed_label": "Score (eval)", + "fixed_score_attr": "score_eval_fixed", + "fixed_loss_attr": "loss_eval_fixed", + "score_train_label": "Score (train)", + "score_fixed_label": "Score (eval)", "use_fixed_metric_for_pruning": True, "prune_min_epoch": 5, "prune_compare_same_resolution_only": True, @@ -45,7 +44,6 @@ LEGACY_DEFAULT_HPO_CONFIG: Dict[str, Any] = { "metric_loss_label": "BCE", "metric_score_label": "Dice", - "desktop_notifications_enabled": False, "validation_rules": { "score_min": 0.0, "loss_min": 0.0, @@ -83,6 +81,8 @@ def load_hpo_config(study_name: Optional[str] = None) -> Dict[str, Any]: from .settings import settings if not study_name: study_name = settings.study_name + if not study_name or not study_name.strip(): + return copy.deepcopy(DEFAULT_HPO_CONFIG) # Try loading study-specific config from DB data = None @@ -103,7 +103,7 @@ def load_hpo_config(study_name: Optional[str] = None) -> Dict[str, Any]: # If we fell back to _global and the current study is NOT a legacy U-Net study, # check if the global config is legacy (version 1). If it is, ignore it to # prevent legacy poisoning of generic studies. - if row.study_name == "_global" and study_name not in ("seg_v1", "bridge_crack_study"): + if row.study_name == "_global": if loaded_data.get("config_version", 1) == 1: loaded_data = None if loaded_data: @@ -113,16 +113,14 @@ def load_hpo_config(study_name: Optional[str] = None) -> Dict[str, Any]: # Fallback to loading default template from disk if DB fails or has no config if not data: - is_legacy = study_name in ("seg_v1", "bridge_crack_study") - data = LEGACY_DEFAULT_HPO_CONFIG if is_legacy else DEFAULT_HPO_CONFIG + data = DEFAULT_HPO_CONFIG # Seed it into DB for this study so it exists in DB try: save_hpo_config(data, study_name) except Exception as e: logger.warning(f"Failed to seed hpo_config in DB: {e}") - is_legacy_name = study_name in ("seg_v1", "bridge_crack_study") - config_version = data.get("config_version", 1 if is_legacy_name else 2) + config_version = data.get("config_version", 2) defaults = LEGACY_DEFAULT_HPO_CONFIG if config_version == 1 else DEFAULT_HPO_CONFIG try: @@ -150,11 +148,12 @@ def save_hpo_config(config: Dict[str, Any], study_name: Optional[str] = None) -> from .settings import settings if not study_name: study_name = settings.study_name + if not study_name or not study_name.strip(): + raise ValueError("study_name cannot be empty.") # Enforce config_version on save to prevent client downgrades config = dict(config) - is_legacy = study_name in ("seg_v1", "bridge_crack_study") - config["config_version"] = config.get("config_version", 1 if is_legacy else 2) + config["config_version"] = config.get("config_version", 2) try: from .db_manager import get_db_session @@ -176,6 +175,15 @@ def save_hpo_config(config: Dict[str, Any], study_name: Optional[str] = None) -> except Exception as e: print(f"Error saving hpo_config to DB: {e}") + # Bust cached study packets — config changes invalidate analytics + try: + from .db_manager import get_db_session + from .schema import CompactedPacket + with get_db_session() as session: + session.query(CompactedPacket).filter_by(study_name=study_name).delete() + except Exception: + pass + def param_display_name(param: str, config: Optional[Dict[str, Any]] = None, study_name: Optional[str] = None) -> str: config = config or load_hpo_config(study_name) diff --git a/src/hpo_coordinator.py b/src/hpo_coordinator.py deleted file mode 100644 index 515c4ef..0000000 --- a/src/hpo_coordinator.py +++ /dev/null @@ -1,1305 +0,0 @@ -"""Shared coordinator logic for Pathfinder. - -This module is the single source of truth for: - - read-only review packets (Pareto, fANOVA, eval insights, recent trials), - - deterministic "review recommended" heuristics (no LLM, no network), - - study review persistence (idempotent across IDE clients). - -It is imported by both broker.py (HTTP runtime) and hpo_mcp_server.py (MCP tools) so -the two surfaces speak the same language without duplicating the suggest path or the -drift-detection logic. -""" -import datetime -import json -import logging -from typing import Any, Dict, List, Optional - -import optuna -from optuna.trial import TrialState - -from .db_manager import get_db_session, DATABASE_URL -from .schema import StudyReview, StudyStatus, TrialResult, SystemConfiguration, CompactedPacket -from .hpo_config import load_hpo_config, normalize_trial_params, param_display_name -from .metrics import get_score, get_loss, get_best_trial, get_best_score, score_objective_index, get_completed_trials, get_eval_attr_names, TERMINAL_STATES - -logger = logging.getLogger(__name__) - -# --- Tunables for drift heuristics (deterministic, no model calls) --- - -MIN_COMPLETED_FOR_FIRST_REVIEW = 5 - -POLICY_ACTIONS = ("no_change", "update_active_search_space", "enqueue_one_manual_trial") - -REVIEW_PROMPT = """You are the Pathfinder coordinator reviewing study '{study_name}'. Do NOT block the training worker. -1. Call get_study_data('{study_name}') to obtain the compacted review packet, which includes active search space constraints, HPO config, Spearman correlations, fANOVA, VRAM/OOM telemetry, boundary hits, past reviews, and prediction accuracy. -2. Analyze the packet data under the following constraints: - - COORDINATOR DECISION MEMORY: Review `past_reviews` (up to the last 3 reviews) to maintain logical consistency. Do NOT blindly reverse previous decisions or flip-flop unless new evidence warrants it. However, do NOT copy previous decisions; critically analyze fresh trials and build on previous hypotheses. - - SPEARMAN CORRELATIONS: Check the `spearman_correlations` confidence tags. Treat correlations with caution if the confidence is "Low" or "Moderate" (due to statistical noise at low sample sizes). - - VRAM & OOM FORECASTS: Examine `bounds_oom_risk` under `vram_telemetry`. Do NOT treat mean predictions as facts. Treat `predicted_mean_vram_gb + margin_gb` (the predicted max VRAM) as the safety boundary relative to `gpu_capacity_gb`. Shrink/cap search bounds if there is high OOM risk. - - ACCURACY SELF-REGULATION: Check `coordinator_accuracy` in the packet. If `insufficient_data` is true (fewer than 3 scored reviews), do not self-regulate yet. If `mean_absolute_error` > 0.05 with n_scored_reviews >= 3, be more conservative—propose smaller search space shifts. Ignore reviews where `quality_flagged` is true. - - DYNAMIC METRIC LABELS: Refer to scores and losses using the dynamic labels specified in the packet (e.g. '{metric_score_label}' and '{metric_loss_label}'). - - GLOBAL BEST TRIAL: The current global best trial is: {best_trial_info}. -3. Rate search space health 1-5 (preferring fixed-eval score if available). -4. Provide a numeric forecast for estimated score improvement. If you have thin/insufficient data (e.g. fewer than 5 trials completed), use `-1.0` as a sentinel value. -5. Identify the trial number you cite as the best trial so far and provide it as the `cited_best_trial` parameter. -6. Select exactly ONE policy action: no_change, update_active_search_space (via update_search_space), or enqueue_one_manual_trial. -7. If proposing active search space changes, call the tool update_search_space(study_name, space_config, apply=False). If enqueuing a manual trial, pass the parameter dictionary as the `manual_trial` argument when calling submit_agent_review. -8. Call submit_agent_review. Write a 3-5 line summary focusing on specific trial numbers and stats, and including a Git-like diff of bounds changes if updated. -""" - - -def compute_statistical_confidence(n_complete: int) -> str: - """Tiered confidence from completed-trial count (caveat only, never a hard gate).""" - if n_complete < 10: - return "low" - if n_complete < 20: - return "medium" - return "high" - - -def get_best_primary_score(study) -> Optional[float]: - completed = get_completed_trials(study) - if not completed: - return None - score = get_best_score(completed, study) - if score is not None: - return score - if len(study.directions) == 1 and study.best_value is not None: - return float(study.best_value) - return None - - -def validate_review_fields( - estimated_score_improvement: Optional[float], - cited_best_trial: Optional[int], -) -> Dict[str, Any]: - """Required JSON contract for coordinator review submissions.""" - errors: List[str] = [] - if cited_best_trial is None: - errors.append("cited_best_trial is required (int trial number).") - if estimated_score_improvement is None: - errors.append("estimated_score_improvement is required (float).") - else: - try: - float(estimated_score_improvement) - except (TypeError, ValueError): - errors.append("estimated_score_improvement must be a number.") - return {"ok": len(errors) == 0, "errors": errors} - - -def compute_coordinator_accuracy(study_name: str) -> Dict[str, Any]: - """MAE from measured StudyReview outcomes (excludes inconclusive, sentinel, and flagged).""" - with get_db_session() as session: - rows = ( - session.query(StudyReview) - .filter( - StudyReview.study_name == study_name, - StudyReview.outcome_status == "measured", - StudyReview.quality_flagged.is_(False), - StudyReview.estimated_score_improvement.isnot(None), - StudyReview.actual_score_improvement.isnot(None), - ) - .order_by(StudyReview.id.asc()) - .all() - ) - scored = [] - for r in rows: - if r.estimated_score_improvement == -1.0: - continue - scored.append({ - "review_id": r.id, - "estimated_score_improvement": r.estimated_score_improvement, - "actual_score_improvement": r.actual_score_improvement, - "absolute_error": abs(r.estimated_score_improvement - r.actual_score_improvement), - }) - n = len(scored) - result: Dict[str, Any] = { - "n_scored_reviews": n, - "insufficient_data": n < 3, - "mean_absolute_error": None, - "accuracy_rate_05": None, - "recent_predictions": scored[-5:], - } - if n > 0: - errors = [s["absolute_error"] for s in scored] - result["mean_absolute_error"] = sum(errors) / n - result["accuracy_rate_05"] = sum(1 for e in errors if e <= 0.05) / n - return result - - -def mark_review_applied(study_name: str) -> None: - """Record when a coordinator search-space patch was committed.""" - study = optuna.load_study(study_name=study_name, storage=DATABASE_URL) - complete_count = len(get_completed_trials(study)) - now = datetime.datetime.utcnow() - with get_db_session() as session: - review = ( - session.query(StudyReview) - .filter_by(study_name=study_name) - .filter(StudyReview.policy_action != "no_change") - .filter(StudyReview.outcome_status == "pending") - .filter(StudyReview.applied_at_completed_count.is_(None)) - .order_by(StudyReview.created_at.desc(), StudyReview.id.desc()) - .first() - ) - if review: - review.applied_at_completed_count = complete_count - review.applied_at = now - - -def backfill_review_outcomes(study_name: str) -> None: - """Measure coordinator forecast accuracy after post-apply trial windows.""" - study = optuna.load_study(study_name=study_name, storage=DATABASE_URL) - complete_count = len(get_completed_trials(study)) - - with get_db_session() as session: - pending = ( - session.query(StudyReview) - .filter( - StudyReview.study_name == study_name, - StudyReview.outcome_status == "pending", - StudyReview.policy_action != "no_change", - StudyReview.applied_at_completed_count.isnot(None), - ) - .all() - ) - for review in pending: - if review.applied_at is None: - continue - complete_since = complete_count - review.applied_at_completed_count - applied_at = review.applied_at.replace(tzinfo=None) if review.applied_at.tzinfo else review.applied_at - finished_since = [ - t for t in study.trials - if t.state in TERMINAL_STATES - and t.datetime_complete - and ( - t.datetime_complete.replace(tzinfo=None) - if getattr(t.datetime_complete, "tzinfo", None) - else t.datetime_complete - ) >= applied_at - ] - if len(finished_since) >= 15 and complete_since < 3: - review.outcome_status = "inconclusive" - review.outcome_measured_at = datetime.datetime.utcnow() - continue - if complete_since >= 5: - new_best = get_best_primary_score(study) - baseline = review.baseline_best_score - if new_best is not None and baseline is not None: - review.actual_score_improvement = new_best - baseline - review.outcome_status = "measured" - review.outcome_measured_at = datetime.datetime.utcnow() - - -def flag_study_review(review_id: int, flagged: bool = True) -> Dict[str, Any]: - with get_db_session() as session: - review = session.query(StudyReview).filter_by(id=review_id).first() - if not review: - return {"success": False, "error": f"Review id {review_id} not found."} - review.quality_flagged = flagged - session.flush() - return {"success": True, "review": review.to_dict()} - - -def build_review_prompt(study_name: str) -> str: - config = load_hpo_config(study_name) - score_label = config.get("metric_score_label", "Score") - loss_label = config.get("metric_loss_label", "Loss") - - best_trial_info = "None (No trials completed yet)" - stat_confidence = "low" - try: - study = optuna.load_study(study_name=study_name, storage=DATABASE_URL) - completed = get_completed_trials(study) - stat_confidence = compute_statistical_confidence(len(completed)) - if completed: - best_t = get_best_trial(completed, study) or study.best_trial - score_val = get_score(best_t, study) if best_t else 0.0 - best_trial_info = f"Trial #{best_t.number} with {score_label}: {score_val:.4f}" - except Exception as e: - best_trial_info = f"Error reading best trial: {e}" - - prompt = REVIEW_PROMPT.format( - study_name=study_name, - metric_score_label=score_label, - metric_loss_label=loss_label, - best_trial_info=best_trial_info - ) - if stat_confidence == "low": - prompt = ( - "STATISTICAL CONFIDENCE: LOW — fewer than 10 completed trials. " - "Treat fANOVA and Spearman signals as noisy. Use estimated_score_improvement=-1.0 " - "when you cannot justify a numeric forecast.\n\n" - ) + prompt - elif stat_confidence == "medium": - prompt = ( - "STATISTICAL CONFIDENCE: MEDIUM — 10–19 completed trials. " - "Correlations may stabilize but remain cautious on bound changes.\n\n" - ) + prompt - return prompt - - -def load_active_search_space(study_name: Optional[str] = None) -> Dict[str, Any]: - from .settings import settings - if not study_name: - study_name = settings.study_name - with get_db_session() as session: - row = session.query(SystemConfiguration).filter_by( - study_name=study_name, config_key="active_search_space" - ).first() - if row: - try: - return json.loads(row.config_value) - except Exception as e: - logger.warning(f"Failed to parse active_search_space for {study_name}: {e}") - return {} - - -def get_ranks(v: List[float]) -> List[float]: - n = len(v) - indexed = sorted(enumerate(v), key=lambda x: x[1]) - ranks = [0.0] * n - i = 0 - while i < n: - j = i - while j < n and indexed[j][1] == indexed[i][1]: - j += 1 - avg_rank = sum(range(i + 1, j + 1)) / (j - i) - for k in range(i, j): - ranks[indexed[k][0]] = avg_rank - i = j - return ranks - - -def compute_spearman_rank_correlation(x: List[float], y: List[float]) -> float: - n = len(x) - if n < 3: - return 0.0 - rx = get_ranks(x) - ry = get_ranks(y) - mean_x = sum(rx) / n - mean_y = sum(ry) / n - num = sum((rx[i] - mean_x) * (ry[i] - mean_y) for i in range(n)) - den_x = sum((rx[i] - mean_x) ** 2 for i in range(n)) - den_y = sum((ry[i] - mean_y) ** 2 for i in range(n)) - if den_x == 0 or den_y == 0: - return 0.0 - raw_corr = num / (den_x * den_y) ** 0.5 - return raw_corr - - -def compute_prune_rate_clusters(study, search_space: Dict[str, Any]) -> Dict[str, Any]: - clusters = {} - continuous_params = [] - for p_name, p_info in search_space.items(): - p_type = p_info.get("type", "") - if p_type in ("float", "float_log", "int"): - continuous_params.append(p_name) - - for p_name in continuous_params: - vals = [] - states = [] - for t in study.trials: - if t.state in (TrialState.COMPLETE, TrialState.PRUNED) and p_name in t.params: - val = t.params[p_name] - if val is not None: - vals.append(float(val)) - states.append(t.state) - - if not vals: - continue - - v_min, v_max = min(vals), max(vals) - if v_max <= v_min: - continue - - w = (v_max - v_min) / 3.0 - bins = [ - {"min": v_min, "max": v_min + w, "total": 0, "pruned": 0}, - {"min": v_min + w, "max": v_min + 2*w, "total": 0, "pruned": 0}, - {"min": v_min + 2*w, "max": v_max, "total": 0, "pruned": 0} - ] - - for val, state in zip(vals, states): - if val <= bins[0]["max"]: - bin_idx = 0 - elif val <= bins[1]["max"]: - bin_idx = 1 - else: - bin_idx = 2 - - bins[bin_idx]["total"] += 1 - if state == TrialState.PRUNED: - bins[bin_idx]["pruned"] += 1 - - for b in bins: - b["prune_rate"] = b["pruned"] / b["total"] if b["total"] > 0 else 0.0 - - clusters[p_name] = bins - return clusters - - -def check_boundary_hits(study, pareto_numbers: List[int], search_space: Dict[str, Any]) -> Dict[str, Any]: - hits = {} - pareto_trials = [t for t in study.trials if t.number in pareto_numbers and t.state == TrialState.COMPLETE] - n_pareto = len(pareto_trials) - if n_pareto == 0: - return hits - - for p_name, p_info in search_space.items(): - p_type = p_info.get("type", "") - if p_type not in ("float", "float_log", "int"): - continue - - s_min = p_info.get("min") - s_max = p_info.get("max") - if s_min is None or s_max is None or s_max <= s_min: - continue - - s_min, s_max = float(s_min), float(s_max) - margin = 0.1 * (s_max - s_min) - - near_min_count = 0 - near_max_count = 0 - - for t in pareto_trials: - val = t.params.get(p_name) - if val is not None: - val = float(val) - if val <= s_min + margin: - near_min_count += 1 - if val >= s_max - margin: - near_max_count += 1 - - total_hits = near_min_count + near_max_count - ratio = total_hits / n_pareto - - if ratio > 0.6: - hits[p_name] = { - "near_min_count": near_min_count, - "near_max_count": near_max_count, - "total_pareto": n_pareto, - "hit_ratio": ratio, - "bound_hit": "min" if near_min_count > near_max_count else "max" if near_max_count > near_min_count else "both" - } - return hits - - -def compute_fidelity_durations(study, config: Dict[str, Any]) -> Dict[str, Any]: - ev = config.get("eval_protocol", {}) - train_param = ev.get("train_resolution_param", "resolution") - - groups = {} - for t in study.trials: - if t.state != TrialState.COMPLETE: - continue - val = t.params.get(train_param) - if val is None: - continue - try: - val = int(val) - except (TypeError, ValueError): - continue - - groups.setdefault(val, []).append(t) - - res_stats = {} - for val, trials in groups.items(): - durations = [] - epoch_durations = [] - for t in trials: - if t.datetime_start and t.datetime_complete: - dur = (t.datetime_complete - t.datetime_start).total_seconds() - durations.append(dur) - history = t.user_attrs.get("history", []) - epochs = len(history) if history else t.user_attrs.get("latest_epoch") - if not epochs: - epochs = max([h.get("epoch", 1) for h in history] or [1]) - if epochs > 0: - epoch_durations.append(dur / epochs) - - if durations: - res_stats[val] = { - "avg_total_duration": sum(durations) / len(durations), - "avg_epoch_duration": sum(epoch_durations) / len(epoch_durations) if epoch_durations else None, - "count": len(durations) - } - - if not res_stats: - return {} - - lowest_scale = min(res_stats.keys()) - base_dur = res_stats[lowest_scale]["avg_total_duration"] - - for val, stats in res_stats.items(): - if base_dur > 0: - stats["overhead_ratio"] = stats["avg_total_duration"] / base_dur - else: - stats["overhead_ratio"] = 1.0 - - return { - "fidelity_param": train_param, - "lowest_scale": lowest_scale, - "scales": res_stats - } -def fit_vram_model(trials: List[Any], db_metrics: Dict[int, Any], train_param: str) -> Optional[Dict[str, Any]]: - points = [] - for t in trials: - if t.state != TrialState.COMPLETE: - continue - metric = db_metrics.get(t.number, {}) - oom = metric.get("oom_triggered") or t.user_attrs.get("oom_triggered", False) - if oom: - continue - vram = metric.get("max_vram_gb") or t.user_attrs.get("max_vram_gb") - if vram is None: - continue - bs = t.params.get("batch_size") - res = t.params.get(train_param) - if bs is not None and res is not None: - try: - points.append((float(bs), float(res), float(vram))) - except (ValueError, TypeError): - continue - - # We require at least 6 points to fit the model to prevent overfitting - if len(points) < 6: - return None - - X = [p[0] * (p[1] ** 2) for p in points] - Y = [p[2] for p in points] - - # Verify variance in X - if len(set(X)) < 2: - return None - - n = len(points) - sum_x = sum(X) - sum_y = sum(Y) - sum_xx = sum(x*x for x in X) - sum_xy = sum(X[i]*Y[i] for i in range(n)) - - denom = n * sum_xx - sum_x * sum_x - if abs(denom) < 1e-12: - return None - - slope = (n * sum_xy - sum_x * sum_y) / denom - intercept = (sum_y - slope * sum_x) / n - - # Calculate Residual Standard Error - ssr = sum((Y[i] - (slope * X[i] + intercept)) ** 2 for i in range(n)) - rse = (ssr / (n - 2)) ** 0.5 if n > 2 else 0.0 - - return { - "slope": slope, - "intercept": intercept, - "n_points": n, - "rse": rse - } - - -def compute_vram_telemetry(study, db_metrics: Dict[int, Any], search_space: Dict[str, Any], config: Dict[str, Any]) -> Dict[str, Any]: - ev = config.get("eval_protocol", {}) - train_param = ev.get("train_resolution_param", "resolution") - - trials = list(study.trials) - model = fit_vram_model(trials, db_metrics, train_param) - - gpu_capacity_gb = 0.0 - gpu_models = [] - oom_count = 0 - - for t in trials: - metric = db_metrics.get(t.number, {}) - vram = metric.get("max_vram_gb") or t.user_attrs.get("max_vram_gb") - gpu = metric.get("gpu_model") or t.user_attrs.get("gpu_model") - oom = metric.get("oom_triggered") or t.user_attrs.get("oom_triggered", False) - - if vram: - gpu_capacity_gb = max(gpu_capacity_gb, float(vram)) - if gpu: - gpu_models.append(gpu) - if oom: - oom_count += 1 - - gpu_model = max(set(gpu_models), key=gpu_models.count) if gpu_models else "Unknown" - - oom_risk = None - if model and gpu_capacity_gb > 0: - max_bs = None - bs_info = search_space.get("batch_size", {}) - if bs_info.get("type") == "categorical": - active_bs = bs_info.get("active", []) - if active_bs: - max_bs = max(active_bs) - else: - max_bs = bs_info.get("max") - - max_res = None - res_info = search_space.get(train_param, {}) - if res_info.get("type") == "categorical": - active_res = res_info.get("active", []) - if active_res: - max_res = max(active_res) - else: - max_res = res_info.get("max") - - if max_bs is not None and max_res is not None: - predicted_mean_vram = model["slope"] * (float(max_bs) * (float(max_res) ** 2)) + model["intercept"] - margin = max(1.0, 1.96 * model["rse"]) - predicted_max_vram = predicted_mean_vram + margin - - if predicted_max_vram > 0.9 * gpu_capacity_gb: - oom_risk = { - "max_batch_size": max_bs, - "max_resolution": max_res, - "predicted_mean_vram_gb": predicted_mean_vram, - "margin_gb": margin, - "predicted_max_vram_gb": predicted_max_vram, - "gpu_capacity_gb": gpu_capacity_gb, - "risk_level": "high" if predicted_max_vram > gpu_capacity_gb else "medium" - } - - return { - "gpu_model": gpu_model, - "gpu_capacity_gb": gpu_capacity_gb, - "oom_count": oom_count, - "vram_model": model, - "bounds_oom_risk": oom_risk - } - - -# --- Train-resolution helper (shared with broker pruning/pareto) --- -def trial_train_resolution(trial, train_param: str) -> Optional[int]: - val = trial.params.get(train_param) - if val is None: - return None - try: - return int(val) - except (TypeError, ValueError): - return None - - -def pareto_trial_numbers_deploy_aware(study, hpo_config: Dict[str, Any]) -> List[int]: - """ - Pareto set for dashboard: optional filter excluding low train-res trials that inflate Dice. - Uses fixed-eval Dice when available on completed trials. - """ - ev = hpo_config.get("eval_protocol", {}) - train_param = ev.get("train_resolution_param", "resolution") - low_warn = ev.get("low_train_res_warning") - low_warn = int(low_warn) if low_warn is not None else None - deploy_only = ev.get("pareto_deploy_resolution_only", True) - dice_fixed_key = ev.get("fixed_dice_attr", "dice_eval_fixed") - - points: List[tuple] = [] - for t in study.trials: - if t.state != TrialState.COMPLETE: - continue - loss_val = get_loss(t, study) - score_val = get_score(t, study) - if loss_val is None or score_val is None: - continue - train_res = trial_train_resolution(t, train_param) - if deploy_only and low_warn is not None and train_res is not None and train_res < low_warn: - continue - if ev.get("enabled"): - fd = t.user_attrs.get(dice_fixed_key) - if fd is not None: - score_val = float(fd) - points.append((t.number, float(loss_val), float(score_val))) - - if not points: - try: - return [t.number for t in study.best_trials] - except Exception: - return [] - - pareto: List[int] = [] - for num_i, loss_i, score_i in points: - dominated = False - for num_j, loss_j, score_j in points: - if num_i == num_j: - continue - if loss_j <= loss_i and score_j >= score_i and (loss_j < loss_i or score_j > score_i): - dominated = True - break - if not dominated: - pareto.append(num_i) - return pareto - - -def study_eval_insights(study, config: Dict[str, Any]) -> Dict[str, Any]: - ev = config.get("eval_protocol", {}) - train_param = ev.get("train_resolution_param", "resolution") - fixed_res = ev.get("fixed_resolution") - low_warn = ev.get("low_train_res_warning") - dice_fixed_key, _ = get_eval_attr_names(ev) - - complete = get_completed_trials(study) - by_res: Dict[int, List] = {} - warnings = [] - - for t in complete: - tr = t.params.get(train_param) - if tr is not None: - by_res.setdefault(int(tr), []).append(t) - - res_summary = {} - for res, trials in sorted(by_res.items()): - scores = [get_score(t, study) for t in trials] - scores = [s for s in scores if s is not None] - fixed_scores = [ - t.user_attrs.get(dice_fixed_key) - for t in trials - if t.user_attrs.get(dice_fixed_key) is not None - ] - res_summary[res] = { - "count": len(trials), - "best_dice_train": max(scores) if scores else None, - "best_dice_fixed": max(fixed_scores) if fixed_scores else None, - "best_score_train": max(scores) if scores else None, - "best_score_fixed": max(fixed_scores) if fixed_scores else None, - } - - # Suppress low-fidelity warning if a valid deploy-scale candidate exists (res >= low_warn) - valid_deploy_exists = False - if low_warn is not None: - for t in complete: - tr = t.params.get(train_param) - fd = t.user_attrs.get(dice_fixed_key) - if tr is not None and int(tr) >= int(low_warn) and fd is not None: - valid_deploy_exists = True - break - - if complete and ev.get("enabled") and fixed_res: - best_train = get_best_trial(complete, study) - if best_train is None: - best_train = complete[0] - train_res = best_train.params.get(train_param) - if train_res is not None and low_warn and int(train_res) < int(low_warn) and not valid_deploy_exists: - warnings.append( - { - "code": "low_train_res_pareto", - "trial_number": best_train.number, - "message": ( - f"Pareto-best trial #{best_train.number} trained at scale {train_res}, " - f"below warning threshold {low_warn}. Check {ev.get('dice_fixed_label', 'fixed eval')}." - ), - } - ) - fd = best_train.user_attrs.get(dice_fixed_key) - td = get_score(best_train, study) - if fd is not None and td is not None and (td - fd) > 0.08: - warnings.append( - { - "code": "train_eval_gap", - "trial_number": best_train.number, - "message": ( - f"Trial #{best_train.number}: train score {td:.3f} vs " - f"fixed-eval score {fd:.3f} — train scale/resolution may be inflating scores." - ), - } - ) - - best_deploy = None - if ev.get("enabled"): - ranked = [ - t - for t in complete - if t.user_attrs.get(dice_fixed_key) is not None - ] - if ranked: - best_deploy = max(ranked, key=lambda t: t.user_attrs.get(dice_fixed_key)) - - return { - "resolution_summary": res_summary, - "warnings": warnings, - "best_deploy_trial_number": best_deploy.number if best_deploy else None, - "best_deploy_dice_fixed": ( - best_deploy.user_attrs.get(dice_fixed_key) if best_deploy else None - ), - "best_deploy_score_fixed": ( - best_deploy.user_attrs.get(dice_fixed_key) if best_deploy else None - ), - } - - -def count_evaluated_trials(study) -> int: - """Finished trials (COMPLETE / PRUNED / FAIL) — the idempotency window for reviews.""" - return sum( - 1 - for t in study.trials - if t.state in TERMINAL_STATES - ) - - -def compute_health_tier(study, study_name: str) -> tuple[str, Optional[str]]: - """Evaluates study health using a tiered severity model (Healthy, Watch, Intervene). - - Returns (health_tier, health_reason) - """ - import math - from optuna.trial import TrialState - - trials = list(study.trials) - finished = sorted([t for t in trials if t.state in TERMINAL_STATES], key=lambda t: t.number) - completed = sorted(get_completed_trials(study), key=lambda t: t.number) - - # 🔴 Intervene Triggers - - # 1. NaN or Inf detected in reported metrics - for t in trials: - if t.values: - for v in t.values: - if v is not None and (math.isnan(v) or math.isinf(v)): - return "intervene", f"NaN or Inf detected in reported metrics for Trial #{t.number}" - - # 2. Same parameter combination caused 2+ OOM failures - try: - with get_db_session() as session: - oom_trials = session.query(TrialResult).filter_by(study_name=study_name, oom_triggered=True).all() - if len(oom_trials) >= 2: - trial_params_map = {t._trial_id: t.params for t in study.trials} - oom_combos = {} - for r in oom_trials: - params = trial_params_map.get(r.trial_id) - if params: - key = tuple(sorted((k, str(v)) for k, v in params.items())) - oom_combos[key] = oom_combos.get(key, 0) + 1 - if oom_combos[key] >= 2: - params_desc = ", ".join(f"{k}={v}" for k, v in key) - return "intervene", f"OOM cluster detected: parameter combination ({params_desc}) failed with OOM 2+ times" - except Exception as e: - print(f"Error checking OOM clusters: {e}") - - # 3. Best score hasn't improved in 2x the study's average improvement interval - if len(completed) >= 5: - improvements = [] - best_so_far = -float("inf") - for i, t in enumerate(completed): - score = get_score(t, study) - if score is None: - continue - if score > best_so_far + 1e-4: - best_so_far = score - improvements.append(i) - - if len(improvements) >= 2: - intervals = [improvements[j] - improvements[j-1] for j in range(1, len(improvements))] - avg_interval = sum(intervals) / len(intervals) - trials_since_last_improvement = len(completed) - 1 - improvements[-1] - threshold = max(4, int(math.ceil(2 * avg_interval))) - if trials_since_last_improvement >= threshold: - return "intervene", f"Score stagnation: no improvement over last {trials_since_last_improvement} completed trials (average improvement interval is {avg_interval:.1f} trials, threshold is {threshold})" - - # 4. Train-eval metric gap exceeds 2σ of the study's historical gap distribution - if len(completed) >= 4: - config = load_hpo_config(study_name) - dice_fixed_key = config.get("eval_protocol", {}).get("fixed_dice_attr", "dice_eval_fixed") - gaps = [] - for t in completed: - fd = t.user_attrs.get(dice_fixed_key) - td = get_score(t, study) - if fd is not None and td is not None: - gaps.append(float(td) - float(fd)) - - if len(gaps) >= 4: - mean_gap = sum(gaps) / len(gaps) - var_gap = sum((g - mean_gap) ** 2 for g in gaps) / len(gaps) - std_gap = var_gap ** 0.5 - latest_gap = gaps[-1] - if std_gap > 0 and latest_gap > mean_gap + 2 * std_gap: - return "intervene", f"Train-eval gap anomaly: latest trial gap ({latest_gap:.4f}) exceeds 2 standard deviations of historical gap distribution (mean={mean_gap:.4f}, std={std_gap:.4f}, threshold={mean_gap + 2*std_gap:.4f})" - - # 🟡 Watch Triggers - - # 1. Prune rate over last 5 trials exceeds 80% (>= 4 out of last 5 finished trials are pruned) - if len(finished) >= 5: - recent_finished = finished[-5:] - pruned_count = sum(1 for t in recent_finished if t.state == TrialState.PRUNED) - if pruned_count >= 4: - return "watch", f"High prune rate: {pruned_count}/5 ({pruned_count*20}%) of recent trials were pruned" - - # 2. Score variance in top quartile drops below epsilon (stagnation) - if len(completed) >= 4: - scores = [get_score(t, study) for t in completed] - scores = [s for s in scores if s is not None] - scores.sort(reverse=True) - top_quartile_count = max(1, len(scores) // 4) - top_scores = scores[:top_quartile_count] - if len(top_scores) >= 2: - mean_top = sum(top_scores) / len(top_scores) - var_top = sum((x - mean_top) ** 2 for x in top_scores) / len(top_scores) - if var_top < 1e-4: - return "watch", f"Score convergence: top quartile score variance ({var_top:.6f}) is below 1e-4" - - # 3. Trial-level warnings - running_trials = [t for t in trials if t.state == TrialState.RUNNING] - latest_completed = completed[-1:] if completed else [] - for t in (running_trials + latest_completed): - t_tier = t.user_attrs.get("health_tier") - t_reason = t.user_attrs.get("health_reason") - if t_tier == "watch" and t_reason: - return "watch", f"Trial #{t.number} warning: {t_reason}" - - return "healthy", "No issues detected. Search space is healthy." - - -def compute_review_heuristics( - study, insights: Dict[str, Any], config: Dict[str, Any], study_name: str -) -> Dict[str, Any]: - """Simplified heuristics adapter using compute_health_tier.""" - health_tier, health_reason = compute_health_tier(study, study_name) - review_recommended = health_tier in ("watch", "intervene") - finished = [t for t in study.trials if t.state in TERMINAL_STATES] - n_eval = len(finished) - latest = get_latest_study_review(study_name) - - # Check if this trial count has been reviewed or dismissed - already_reviewed = latest is not None and latest.get("trials_evaluated") == n_eval - - already_dismissed = False - with get_db_session() as session: - status_row = session.query(StudyStatus).filter_by(study_name=study_name).first() - if status_row and status_row.nudge_dismissed_trials is not None: - if status_row.nudge_dismissed_trials == n_eval: - already_dismissed = True - - if already_reviewed or already_dismissed: - review_recommended = False - - return { - "review_recommended": review_recommended, - "health_tier": health_tier, - "health_reason": health_reason, - "reasons": [{"code": health_tier, "message": health_reason}] if health_reason else [], - "trials_evaluated": n_eval, - "already_reviewed": already_reviewed, - "already_dismissed": already_dismissed, - "last_review_trials_evaluated": latest.get("trials_evaluated") if latest else None, - } - - -# --- Review persistence (idempotent) --- -def get_latest_study_review(study_name: str) -> Optional[Dict[str, Any]]: - with get_db_session() as session: - row = ( - session.query(StudyReview) - .filter_by(study_name=study_name) - .order_by(StudyReview.created_at.desc(), StudyReview.id.desc()) - .first() - ) - return row.to_dict() if row else None - - -def get_recent_study_reviews(study_name: str, limit: int = 10) -> List[Dict[str, Any]]: - with get_db_session() as session: - rows = ( - session.query(StudyReview) - .filter_by(study_name=study_name) - .order_by(StudyReview.created_at.desc(), StudyReview.id.desc()) - .limit(limit) - .all() - ) - return [r.to_dict() for r in rows] - - -def save_study_review( - study_name: str, - summary: str, - *, - health_rating: Optional[int] = None, - policy_action: str = "no_change", - model_version: str = "unspecified", - prompt_strategy: str = "coordinator_review", - reasons: Optional[List[Dict[str, Any]]] = None, - trials_evaluated: int = 0, - estimated_score_improvement: Optional[float] = None, - cited_best_trial: Optional[int] = None, - force: bool = False, -) -> Dict[str, Any]: - """Persist a coordinator review. Idempotent per trial window unless force=True.""" - if policy_action not in POLICY_ACTIONS: - policy_action = "no_change" - if health_rating is not None: - try: - health_rating = max(1, min(5, int(health_rating))) - except (TypeError, ValueError): - health_rating = None - if estimated_score_improvement is not None: - try: - estimated_score_improvement = float(estimated_score_improvement) - except (TypeError, ValueError): - estimated_score_improvement = None - - # Load study first to run validation assertions - study = optuna.load_study(study_name=study_name, storage=DATABASE_URL) - completed_trials = get_completed_trials(study) - completed_count = len(completed_trials) - - if completed_count < MIN_COMPLETED_FOR_FIRST_REVIEW: - estimated_score_improvement = -1.0 - - baseline_best_score = get_best_primary_score(study) - if baseline_best_score is None: - baseline_best_score = -1.0 - - if policy_action == "no_change": - outcome_status = "not_applicable" - else: - outcome_status = "pending" - - # 1. Require at least one evaluated trial (ValueError, not assert, so it survives `python -O`). - if trials_evaluated <= 0: - raise ValueError("No trials have been evaluated yet. Cannot save review.") - - # 2. Verify trials_evaluated matches finished-trial idempotency window (COMPLETE+PRUNED+FAIL) - evaluated_count = count_evaluated_trials(study) - if trials_evaluated != evaluated_count: - raise ValueError( - f"Idempotency key trials_evaluated ({trials_evaluated}) must match " - f"actual count of evaluated trials ({evaluated_count})." - ) - - # 3. Compare cited trial score with actual best trial score - confidence = "high" - if completed_trials: - best_t = get_best_trial(completed_trials, study) or study.best_trial - actual_best_score = get_score(best_t, study) - - if cited_best_trial is not None: - cited_t = None - for t in completed_trials: - if t.number == cited_best_trial: - cited_t = t - break - if cited_t: - cited_score = get_score(cited_t, study) - if actual_best_score is None or cited_score is None: - confidence = "low" - elif abs(actual_best_score - cited_score) > 0.10: - confidence = "low" - else: - # Cited a non-existent completed trial - confidence = "low" - else: - # Did not specify cited best trial - confidence = "low" - - latest = get_latest_study_review(study_name) - if latest and not force and latest.get("trials_evaluated") == trials_evaluated: - return {"success": True, "duplicate": True, "review": latest} - - with get_db_session() as session: - review = StudyReview( - study_name=study_name, - health_rating=health_rating, - summary=summary, - policy_action=policy_action, - model_version=model_version or "unspecified", - prompt_strategy=prompt_strategy or "coordinator_review", - trials_evaluated=trials_evaluated, - estimated_score_improvement=estimated_score_improvement, - cited_best_trial=cited_best_trial, - confidence=confidence, - baseline_best_score=baseline_best_score, - outcome_status=outcome_status, - ) - review.set_reasons(reasons) - session.add(review) - - # Record review window without masking underlying study health - status = session.query(StudyStatus).filter_by(study_name=study_name).first() - if status: - tier, reason = compute_health_tier(study, study_name) - status.health_tier = tier - status.health_reason = reason - - session.flush() - saved = review.to_dict() - return {"success": True, "duplicate": False, "review": saved} - - -# --- fANOVA + packet assembly --- -def get_fanova_importances(study, config: Dict[str, Any]) -> Dict[str, float]: - complete = get_completed_trials(study) - if len(complete) < 2: - return {} - importances: Dict[str, float] = {} - try: - if len(study.directions) > 1: - _si = score_objective_index(study) - if _si is not None: - importances = optuna.importance.get_param_importances( - study, - target=lambda t, idx=_si: t.values[idx] if (t.values and len(t.values) > idx) else None, - evaluator=optuna.importance.FanovaImportanceEvaluator(), - ) - else: - importances = optuna.importance.get_param_importances( - study, evaluator=optuna.importance.FanovaImportanceEvaluator() - ) - except Exception: - return {} - - aliases = config.get("legacy_param_aliases", {}) - display: Dict[str, float] = {} - for param, value in importances.items(): - canonical = aliases.get(param, param) - label = param_display_name(canonical, config) - display[label] = max(display.get(label, 0.0), float(value)) - return display - - -def _recent_trials_summary(study, config: Dict[str, Any], limit: int) -> List[Dict[str, Any]]: - ev = config.get("eval_protocol", {}) - train_param = ev.get("train_resolution_param", "resolution") - dice_fixed_key = ev.get("fixed_dice_attr", "dice_eval_fixed") - bce_fixed_key = ev.get("fixed_bce_attr", "bce_eval_fixed") - - ordered = sorted(study.trials, key=lambda t: t.number, reverse=True)[:limit] - out: List[Dict[str, Any]] = [] - for t in ordered: - dice = get_score(t, study) if t.state == TrialState.COMPLETE else t.user_attrs.get("latest_dice") - bce = get_loss(t, study) if t.state == TrialState.COMPLETE else t.user_attrs.get("latest_bce") - out.append({ - "number": t.number, - "state": t.state.name, - "params": normalize_trial_params(dict(t.params), config), - "train_resolution": trial_train_resolution(t, train_param), - "dice_train": dice, - "bce_train": bce, - "dice_eval_fixed": t.user_attrs.get(dice_fixed_key), - "bce_eval_fixed": t.user_attrs.get(bce_fixed_key), - "latest_epoch": t.user_attrs.get("latest_epoch"), - }) - return out - - -def build_review_packet(study_name: str) -> Dict[str, Any]: - """Assemble the compacted HPO review packet, utilizing a lazy materialization cache layer. - - This is the single source of truth for both the HTTP broker API and the MCP server. - """ - try: - study = optuna.load_study(study_name=study_name, storage=DATABASE_URL) - n_eval = count_evaluated_trials(study) - - # Check compacted packets cache - with get_db_session() as session: - cached = session.query(CompactedPacket).filter_by( - study_name=study_name, trials_evaluated=n_eval - ).first() - if cached: - try: - packet = json.loads(cached.packet_json) - n_complete = len(get_completed_trials(study)) - packet["statistical_confidence"] = compute_statistical_confidence(n_complete) - packet["coordinator_accuracy"] = compute_coordinator_accuracy(study_name) - packet["review_prompt"] = build_review_prompt(study_name) - packet["policy_actions"] = list(POLICY_ACTIONS) - packet["latest_review"] = get_latest_study_review(study_name) - return packet - except Exception as e: - logger.warning(f"Failed to load cached packet for {study_name}: {e}") - - # Otherwise materialize from scratch - with get_db_session() as session: - # 1. Fetch DB metrics - db_metrics = {} - rows = session.query(TrialResult).filter_by(study_name=study_name).all() - for r in rows: - db_metrics[r.trial_id] = r.to_dict() - - # 2. Fetch search space and config - search_space = load_active_search_space(study_name) - config = load_hpo_config(study_name) - - # 3. Fetch project context - project_context = {} - context_row = session.query(SystemConfiguration).filter_by( - study_name=study_name, config_key="project_context" - ).first() - if context_row: - try: - project_context = json.loads(context_row.config_value) - except Exception as e: - logger.warning(f"Failed to parse project_context for {study_name}: {e}") - - # 4. Fetch health tier - health_tier, health_reason = compute_health_tier(study, study_name) - - # 5. Fetch past reviews - past_reviews = [] - rev_rows = ( - session.query(StudyReview) - .filter_by(study_name=study_name) - .order_by(StudyReview.created_at.desc(), StudyReview.id.desc()) - .limit(3) - .all() - ) - for r in rev_rows: - past_reviews.append(r.to_dict()) - - # 6. Accuracy + statistical confidence - n_complete = len(get_completed_trials(study)) - statistical_confidence = compute_statistical_confidence(n_complete) - accuracy_stats = compute_coordinator_accuracy(study_name) - - # 7. Assemble compacted packet using build_compacted_packet - from .analytics import build_compacted_packet - packet = build_compacted_packet( - study_name, - study, - db_metrics, - search_space, - config, - health_tier, - health_reason, - past_reviews, - accuracy_stats, - project_context, - statistical_confidence, - ) - - # Cache it in compacted_packets table - with get_db_session() as session: - session.merge(CompactedPacket( - study_name=study_name, - trials_evaluated=n_eval, - packet_json=json.dumps(packet) - )) - - packet["statistical_confidence"] = statistical_confidence - packet["coordinator_accuracy"] = accuracy_stats - packet["review_prompt"] = build_review_prompt(study_name) - packet["policy_actions"] = list(POLICY_ACTIONS) - packet["latest_review"] = get_latest_study_review(study_name) - return packet - except Exception as e: - import traceback - traceback.print_exc() - return {"success": False, "error": f"Failed to build review packet: {str(e)}"} - - -def _validate_manual_parameters(manual_parameters: Dict[str, Any], study_name: str) -> Dict[str, Any]: - """Validate agent-proposed params against DB-backed search space.""" - from .validators import UNetHyperparameters, LEGACY_UNET_PARAMS - - config = load_hpo_config(study_name) - norm = normalize_trial_params(dict(manual_parameters), config) - - space = load_active_search_space(study_name) - space_keys = {k for k in space.keys() if not k.startswith("_") and isinstance(space.get(k), dict)} - - # Study-specific validator for bridge-crack U-Net params - if space_keys == LEGACY_UNET_PARAMS: - try: - valid = UNetHyperparameters(**norm) - return {"ok": True, "params": valid.model_dump(), "error": None, "warnings": []} - except Exception as exc: - return {"ok": False, "params": {}, "error": str(exc), "warnings": []} - - warnings: List[str] = [] - out: Dict[str, Any] = {} - - for name, spec in space.items(): - if name.startswith("_") or not isinstance(spec, dict): - continue - ptype = spec.get("type", "float") - if name not in norm: - return {"ok": False, "params": {}, "error": f"Missing required parameter '{name}'.", "warnings": warnings} - value = norm[name] - - if ptype in ("float", "float_log", "int"): - try: - num = float(value) - except (TypeError, ValueError): - return {"ok": False, "params": {}, "error": f"Parameter '{name}' must be numeric.", "warnings": warnings} - lo, hi = spec.get("min"), spec.get("max") - if lo is not None and num < float(lo): - return {"ok": False, "params": {}, "error": f"'{name}'={num} below min {lo}.", "warnings": warnings} - if hi is not None and num > float(hi): - return {"ok": False, "params": {}, "error": f"'{name}'={num} above max {hi}.", "warnings": warnings} - out[name] = int(round(num)) if ptype == "int" else num - elif ptype == "categorical": - options = spec.get("options", []) - coerced = value - if value not in options: - for opt in options: - if str(opt) == str(value): - coerced = opt - break - if coerced not in options: - return {"ok": False, "params": {}, "error": f"'{name}'={value} not in options {options}.", "warnings": warnings} - active = spec.get("active", options) - if coerced not in active: - warnings.append(f"'{name}'={coerced} is allowed but not in the active set {active}.") - out[name] = coerced - else: - out[name] = value - - extra = [k for k in norm if k not in space_keys] - if extra: - warnings.append(f"Ignoring parameters not in search space: {extra}.") - - return {"ok": True, "params": out, "error": None, "warnings": warnings} - - -def load_study_cards(study_name: Optional[str] = None) -> List[Dict[str, Any]]: - """Query and return generated study cards, loading their markdown contents from disk if available.""" - from .schema import StudyCard - with get_db_session() as session: - query = session.query(StudyCard) - if study_name: - query = query.filter_by(study_name=study_name) - cards = query.all() - - result = [] - for c in cards: - content = "" - root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) - full_path = os.path.join(root_dir, c.file_path) - if os.path.exists(full_path): - try: - with open(full_path, "r") as f: - content = f.read() - except Exception as e: - logger.warning(f"Failed to read model card {full_path}: {e}") - card_dict = c.to_dict() - card_dict["markdown_content"] = content - result.append(card_dict) - return result - - -_last_hook_trigger: Dict[str, float] = {} - -def write_ide_status_file(study_name: str, health_tier: str, health_reason: str, study) -> None: - """Writes the current health status to .hpo_status.json in the workspace root.""" - import datetime - - trials_evaluated = count_evaluated_trials(study) - - # Form status payload - payload = { - "study_name": study_name, - "health_tier": health_tier.lower(), - "health_reason": health_reason, - "trials_evaluated": trials_evaluated, - "review_recommended": health_tier.lower() in ("watch", "intervene"), - "last_updated": datetime.datetime.utcnow().isoformat() - } - - # Write to .hpo_status.json in workspace root - from pathlib import Path - root_dir = Path(__file__).resolve().parent.parent - status_file_path = root_dir / ".hpo_status.json" - - try: - with open(status_file_path, "w") as f: - json.dump(payload, f, indent=4) - except Exception as e: - print(f"Error writing .hpo_status.json: {e}") - - diff --git a/src/hpo_daemon.py b/src/hpo_daemon.py index 494a321..a5fb3ab 100644 --- a/src/hpo_daemon.py +++ b/src/hpo_daemon.py @@ -1,107 +1,64 @@ -"""Background polling daemon for monitoring study health. +"""Background polling daemon for study health monitoring only. -Runs inside the FastAPI broker process as a background thread. It is intentionally NOT an -autopilot: it reclaims expired trial leases, recomputes each study's health tier, writes the -``.hpo_status.json`` hint file, and (optionally) fires a desktop notification so the human can -open the dashboard. It never calls an LLM and never mutates the search space — coordinator -reviews are always human-initiated (dashboard "Apply Proposal" or the MCP tools). +Runs inside the FastAPI broker process as a background thread. It reclaims expired trial +leases, recomputes each study's health tier via ``compute_health_tier``, writes the +``.hpo_status.json`` hint file, and updates ``StudyStatus`` in the database. It never calls +an LLM, never fires desktop notifications, and never recommends coordinator reviews. """ +import logging import time -import subprocess -from typing import Dict +from typing import List from src.db_manager import get_db_session +from src.health import compute_health_tier, write_ide_status_file from src.schema import StudyStatus -from src.hpo_coordinator import compute_review_heuristics -from src.hpo_config import load_hpo_config -DEFAULT_STUDY = "bridge_crack_study" - -# Cooldown to avoid alert spamming: {study_name: (last_alert_time, last_alert_trials)} -_ALERT_COOLDOWN_PERIOD = 300 # 5 minutes -_alert_history: Dict[str, tuple] = {} - - -def trigger_macos_notification(title: str, subtitle: str, message: str): - """Triggers a native macOS desktop alert popup using AppleScript.""" - try: - title_esc = title.replace('"', '\\"') - subtitle_esc = subtitle.replace('"', '\\"') - msg_esc = message.replace('"', '\\"') - - script = f'display notification "{msg_esc}" with title "{title_esc}" subtitle "{subtitle_esc}"' - subprocess.run(["osascript", "-e", script], check=True) - except Exception as e: - print(f"Error triggering macOS desktop notification: {e}") +logger = logging.getLogger(__name__) def check_and_alert_study(study_name: str): - """Recompute a study's health, refresh the status hint file, and notify if warranted. - - Notify-only: when a review is recommended and desktop notifications are enabled, fire a - macOS notification pointing the user at the dashboard. No LLM is ever called here. - """ + """Recompute a study's health tier, write the IDE status file, and persist to DB.""" try: - from .suggest import get_or_create_study + from .suggest import load_study try: - study = get_or_create_study(study_name) + study = load_study(study_name) except Exception: - # Study might not exist yet, skip silently return - hpo_config = load_hpo_config(study_name) - notifs_enabled = hpo_config.get("desktop_notifications_enabled", False) - - from src.hpo_coordinator import study_eval_insights, write_ide_status_file - insights = study_eval_insights(study, hpo_config) - heuristics = compute_review_heuristics(study, insights, hpo_config, study_name) + health_tier, health_reason = compute_health_tier(study, study_name) - n_eval = heuristics["trials_evaluated"] - health_tier = heuristics["health_tier"] - health_reason = heuristics["health_reason"] - - # Refresh the IDE status hint file (informational; agents may read it on demand). write_ide_status_file(study_name, health_tier, health_reason, study) - if not heuristics["review_recommended"]: - # Healthy, or already reviewed/dismissed for the current trial window. - return - - # Cooldown to avoid repeat alerts for the same trial window. - now = time.time() - if study_name in _alert_history: - last_time, last_trials = _alert_history[study_name] - if last_trials == n_eval or (now - last_time) < _ALERT_COOLDOWN_PERIOD: - return - _alert_history[study_name] = (now, n_eval) - - print(f"⚠️ Pathfinder Alert [{study_name.upper()}]: {health_tier.upper()} state. Reason: {health_reason}") - if notifs_enabled: - subtitle = f"Study Health: {health_tier.upper()}" - trigger_macos_notification("Pathfinder", subtitle, f"Trial #{n_eval} | {health_reason} (Open dashboard to review)") + with get_db_session() as session: + status = session.query(StudyStatus).filter_by(study_name=study_name).first() + if status is None: + status = StudyStatus(study_name=study_name) + session.add(status) + status.health_tier = health_tier + status.health_reason = health_reason + session.commit() - except Exception as e: - print(f"Error checking study health: {e}") + except Exception: + logger.exception("Error checking health for study %s", study_name) def reclaim_expired_leases(): - from datetime import datetime + from datetime import datetime, timezone import optuna from src.schema import TrialLease - from .suggest import get_or_create_study + from .suggest import load_study try: with get_db_session() as session: - now = datetime.utcnow() + now = datetime.now(timezone.utc).replace(tzinfo=None) expired = session.query(TrialLease).filter( TrialLease.lease_expires_at < now ).all() if expired: for lease in expired: try: - study = get_or_create_study(lease.study_name) - # Find corresponding trial number in Optuna + study = load_study(lease.study_name) trial_number = None for t in study.trials: if t._trial_id == lease.trial_id: @@ -113,21 +70,23 @@ def reclaim_expired_leases(): ) if trial_obj and trial_obj.state == optuna.trial.TrialState.RUNNING: study.tell(trial_number, state=optuna.trial.TrialState.FAIL) - print( - f"Daemon: Terminated expired leased Trial {trial_number} " - f"(ID {lease.trial_id}) in study '{lease.study_name}'." + logger.info( + "Terminated expired leased Trial %d (ID %s) in study '%s'.", + trial_number, lease.trial_id, lease.study_name, ) - except Exception as e: - print(f"Daemon: Failed to cleanly terminate expired trial ID {lease.trial_id}: {e}") + except Exception: + logger.exception( + "Failed to cleanly terminate expired trial ID %s", lease.trial_id + ) session.delete(lease) session.commit() - except Exception as e: - print(f"Daemon: Error in reclaim_expired_leases: {e}") + except Exception: + logger.exception("Error in reclaim_expired_leases") def run_daemon_loop(interval_seconds: int = 10): - """Indefinite daemon polling loop (notify-only health monitoring + lease reclamation).""" - print(f"Starting background health daemon thread (notify-only, interval: {interval_seconds}s)...") + """Indefinite daemon polling loop (health monitoring + lease reclamation).""" + logger.info("Starting background health daemon thread (interval: %ds)...", interval_seconds) last_reap_time = 0.0 while True: try: @@ -136,21 +95,18 @@ def run_daemon_loop(interval_seconds: int = 10): reclaim_expired_leases() last_reap_time = now - studies = [] + studies: List[str] = [] try: with get_db_session() as session: rows = session.query(StudyStatus).all() studies = [r.study_name for r in rows] - except Exception as e: - print(f"Failed to fetch studies: {e}") - - if not studies: - studies = [DEFAULT_STUDY] + except Exception: + logger.exception("Failed to fetch studies") for name in studies: check_and_alert_study(name) - except Exception as e: - print(f"Daemon loop error: {e}") + except Exception: + logger.exception("Daemon loop error") time.sleep(interval_seconds) diff --git a/src/leases.py b/src/leases.py index a65e571..7f6072b 100644 --- a/src/leases.py +++ b/src/leases.py @@ -1,5 +1,5 @@ import traceback -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from typing import Optional from pydantic import BaseModel from fastapi import HTTPException @@ -16,7 +16,7 @@ def _reap_stale_running_trials(study, study_name: str, session) -> int: """Fail RUNNING trials whose worker lease expired or has no active lease.""" - now = datetime.utcnow() + now = datetime.now(timezone.utc).replace(tzinfo=None) expired_trial_ids = [ row.trial_id for row in session.query(TrialLease.trial_id).filter( @@ -35,6 +35,13 @@ def _reap_stale_running_trials(study, study_name: str, session) -> int: for t in study.trials: if t.state != TrialState.RUNNING: continue + # Skip trials that were created very recently (no lease row yet) to avoid + # reaping a trial that a concurrent suggest call is still setting up. + if t._trial_id not in active_leased_ids and t._trial_id not in expired_trial_ids: + if t.datetime_start is not None: + age = (now - t.datetime_start.replace(tzinfo=None)).total_seconds() + if age < LEASE_TTL_SECONDS: + continue stale = t._trial_id in expired_trial_ids or t._trial_id not in active_leased_ids if not stale: continue @@ -66,7 +73,7 @@ def _try_claim_lease(session, study_name: str, trial_id: int, worker_id: str, sole mechanism preventing two workers from being handed the same trial. Safe under concurrency: SQLite (WAL + busy_timeout) serializes writers and Postgres locks the row. """ - now = datetime.utcnow() + now = datetime.now(timezone.utc).replace(tzinfo=None) new_expiry = now + timedelta(seconds=ttl_seconds) updated = session.query(TrialLease).filter( TrialLease.trial_id == trial_id, @@ -101,8 +108,7 @@ def _lease_is_owned(study_name: str, trial_id: int, worker_id: Optional[str]) -> Workers prove ownership with the ``worker_id`` returned from /api/suggest_trial before they may report epochs or complete an in-flight trial. Terminal trials skip this check (see callers): idempotent retries and post-prune completes must still record results even - though the lease was already deleted. Note: compared against naive UTC because lease - timestamps are stored via ``datetime.utcnow()``. + though the lease was already deleted. Lease timestamps are stored as naive UTC. """ if not worker_id: return False @@ -112,7 +118,7 @@ def _lease_is_owned(study_name: str, trial_id: int, worker_id: Optional[str]) -> ).first() if lease is None: return False - if lease.lease_expires_at is not None and lease.lease_expires_at < datetime.utcnow(): + if lease.lease_expires_at is not None and lease.lease_expires_at < datetime.now(timezone.utc).replace(tzinfo=None): return False return True @@ -132,7 +138,7 @@ def handle_api_heartbeat(req: HeartbeatRequest): leased_to=req.worker_id ).first() if lease: - lease.lease_expires_at = datetime.utcnow() + timedelta(seconds=LEASE_TTL_SECONDS) + lease.lease_expires_at = datetime.now(timezone.utc).replace(tzinfo=None) + timedelta(seconds=LEASE_TTL_SECONDS) session.commit() return {"success": True, "message": "Heartbeat acknowledged"} return {"success": False, "message": "Lease not found or expired"} diff --git a/src/manifest.py b/src/manifest.py index 372d114..1d284e5 100644 --- a/src/manifest.py +++ b/src/manifest.py @@ -459,15 +459,14 @@ def _manifest_to_hpo_config(data: Dict[str, Any]) -> Dict[str, Any]: "config_version": 2, "metric_loss_label": loss_label, "metric_score_label": score_label, - "desktop_notifications_enabled": False, "eval_protocol": { "enabled": eval_proto_enabled, "fixed_resolution": fixed_res, "train_resolution_param": train_res_param, - "fixed_dice_attr": score_eval_attr, - "fixed_bce_attr": loss_eval_attr, - "dice_train_label": f"{score_label} (train)", - "dice_fixed_label": f"{score_label} (eval)", + "fixed_score_attr": score_eval_attr, + "fixed_loss_attr": loss_eval_attr, + "score_train_label": f"{score_label} (train)", + "score_fixed_label": f"{score_label} (eval)", "use_fixed_metric_for_pruning": eval_proto_enabled, "prune_min_epoch": 5, "prune_compare_same_resolution_only": True, diff --git a/src/metrics.py b/src/metrics.py index fdc9476..5c64781 100644 --- a/src/metrics.py +++ b/src/metrics.py @@ -4,8 +4,7 @@ ``directions`` list instead of assuming [minimize, maximize] order. This makes the codebase work for: - single-objective (maximize *or* minimize) - - multi-objective with any number of objectives and any direction ordering - - the original 2-obj [minimize, maximize] setup (fully backward-compatible) + - multi-objective with up to 2 objectives (one maximize, one minimize) """ from typing import List, Optional, Sequence @@ -134,12 +133,12 @@ def get_loss_from_dirs(trial: FrozenTrial, directions: Sequence[StudyDirection]) def _trial_metric_snapshot( trial: FrozenTrial, history: List[dict], - dice_fixed_attr: str, - bce_fixed_attr: str, + score_fixed_attr: str, + loss_fixed_attr: str, directions: Sequence[StudyDirection] = None, ) -> dict: """Score/Loss for dashboard: completed values, else latest epoch / user_attrs.""" - bce = dice = dice_eval_fixed = bce_eval_fixed = None + _score = _loss = _score_eval_fixed = _loss_eval_fixed = None latest_epoch = trial.user_attrs.get("latest_epoch") from optuna.trial import TrialState @@ -147,45 +146,41 @@ def _trial_metric_snapshot( if trial.state == TrialState.COMPLETE and (trial.values or trial.value is not None): if trial.values and len(trial.values) > 1: - bce = get_loss_from_dirs(trial, directions or []) - dice = get_score_from_dirs(trial, directions or []) + _loss = get_loss_from_dirs(trial, directions or []) + _score = get_score_from_dirs(trial, directions or []) else: if directions and directions[0] == StudyDirection.MINIMIZE: - bce = trial.value + _loss = trial.value else: - dice = trial.value + _score = trial.value else: - dice = trial.user_attrs.get("latest_score", trial.user_attrs.get("latest_dice")) - bce = trial.user_attrs.get("latest_loss", trial.user_attrs.get("latest_bce")) - dice_eval_fixed = trial.user_attrs.get("score_eval_fixed", trial.user_attrs.get(dice_fixed_attr)) - bce_eval_fixed = trial.user_attrs.get("loss_eval_fixed", trial.user_attrs.get(bce_fixed_attr)) + _score = trial.user_attrs.get("latest_score") + _loss = trial.user_attrs.get("latest_loss") + _score_eval_fixed = trial.user_attrs.get("score_eval_fixed", trial.user_attrs.get(score_fixed_attr)) + _loss_eval_fixed = trial.user_attrs.get("loss_eval_fixed", trial.user_attrs.get(loss_fixed_attr)) if history: last = max(history, key=lambda e: e.get("epoch", 0)) latest_epoch = latest_epoch or last.get("epoch") - if dice is None: - dice = last.get("score", last.get("dice")) - if bce is None: - bce = last.get("loss", last.get("bce")) - if dice_eval_fixed is None: - dice_eval_fixed = last.get("score_eval_fixed", last.get("dice_eval_fixed")) - if bce_eval_fixed is None: - bce_eval_fixed = last.get("loss_eval_fixed", last.get("bce_eval_fixed")) - - if dice_eval_fixed is None: - dice_eval_fixed = trial.user_attrs.get("score_eval_fixed", trial.user_attrs.get(dice_fixed_attr)) - if bce_eval_fixed is None: - bce_eval_fixed = trial.user_attrs.get("loss_eval_fixed", trial.user_attrs.get(bce_fixed_attr)) + if _score is None: + _score = last.get("score") + if _loss is None: + _loss = last.get("loss") + if _score_eval_fixed is None: + _score_eval_fixed = last.get("score_eval_fixed") + if _loss_eval_fixed is None: + _loss_eval_fixed = last.get("loss_eval_fixed") + + if _score_eval_fixed is None: + _score_eval_fixed = trial.user_attrs.get("score_eval_fixed", trial.user_attrs.get(score_fixed_attr)) + if _loss_eval_fixed is None: + _loss_eval_fixed = trial.user_attrs.get("loss_eval_fixed", trial.user_attrs.get(loss_fixed_attr)) return { - "bce": bce, - "dice": dice, - "score": dice, - "loss": bce, - "dice_eval_fixed": dice_eval_fixed, - "bce_eval_fixed": bce_eval_fixed, - "score_eval_fixed": dice_eval_fixed, - "loss_eval_fixed": bce_eval_fixed, + "score": _score, + "loss": _loss, + "score_eval_fixed": _score_eval_fixed, + "loss_eval_fixed": _loss_eval_fixed, "latest_epoch": latest_epoch, } diff --git a/src/onboarding.py b/src/onboarding.py index 7c6970f..9c50e74 100644 --- a/src/onboarding.py +++ b/src/onboarding.py @@ -1,4 +1,3 @@ -import os import json import logging import optuna @@ -150,7 +149,11 @@ def init_study_from_manifest_dict(data: Dict[str, Any], force: bool = False) -> if study_exists and force: # Call the thorough delete_study tool to purge all trials and metadata - delete_study_internal(study_name=study_name, confirm=True) + result = delete_study_internal(study_name=study_name, confirm=True) + if not result.get("success"): + raise RuntimeError( + f"Failed to delete existing study '{study_name}': {result.get('error', 'unknown error')}" + ) metrics = data["metrics"] active_search_space = _manifest_params_to_search_space(data["params"]) diff --git a/src/pruning.py b/src/pruning.py index 3454661..dfaa088 100644 --- a/src/pruning.py +++ b/src/pruning.py @@ -2,7 +2,7 @@ from typing import Optional, Dict, Any, List from optuna.trial import TrialState -from src.hpo_coordinator import trial_train_resolution as _trial_train_resolution +from src.analytics import trial_train_resolution as _trial_train_resolution def _effective_train_resolution( @@ -24,7 +24,7 @@ def _effective_train_resolution( def _epoch_composite_score(study, trial, epoch: int, ev: Dict[str, Any]) -> Optional[float]: - """Composite score at epoch, Z-score normalized against study rolling history at the same epoch, fallback to (dice - bce).""" + """Composite score at epoch, Z-score normalized against study rolling history at the same epoch, fallback to (score - loss).""" use_fixed = ev.get("enabled") and ev.get("use_fixed_metric_for_pruning") # 1. Retrieve current trial's metrics at this epoch @@ -34,12 +34,12 @@ def _epoch_composite_score(study, trial, epoch: int, ev: Dict[str, Any]) -> Opti if isinstance(history, list): for entry in history: if entry.get("epoch") == epoch: - if use_fixed and (entry.get("score_eval_fixed") is not None or entry.get("dice_eval_fixed") is not None): - curr_score = entry.get("score_eval_fixed", entry.get("dice_eval_fixed")) - curr_loss = entry.get("loss_eval_fixed", entry.get("bce_eval_fixed", entry.get("loss", entry.get("bce", 0.0)))) + if use_fixed and entry.get("score_eval_fixed") is not None: + curr_score = entry.get("score_eval_fixed") + curr_loss = entry.get("loss_eval_fixed", entry.get("loss", 0.0)) else: - curr_score = entry.get("score", entry.get("dice")) - curr_loss = entry.get("loss", entry.get("bce")) + curr_score = entry.get("score") + curr_loss = entry.get("loss") break if curr_score is None or curr_loss is None: @@ -57,12 +57,12 @@ def _epoch_composite_score(study, trial, epoch: int, ev: Dict[str, Any]) -> Opti if isinstance(t_history, list): for entry in t_history: if entry.get("epoch") == epoch: - if use_fixed and (entry.get("score_eval_fixed") is not None or entry.get("dice_eval_fixed") is not None): - s = entry.get("score_eval_fixed", entry.get("dice_eval_fixed")) - l = entry.get("loss_eval_fixed", entry.get("bce_eval_fixed", entry.get("loss", entry.get("bce", 0.0)))) + if use_fixed and entry.get("score_eval_fixed") is not None: + s = entry.get("score_eval_fixed") + l = entry.get("loss_eval_fixed", entry.get("loss", 0.0)) else: - s = entry.get("score", entry.get("dice")) - l = entry.get("loss", entry.get("bce")) + s = entry.get("score") + l = entry.get("loss") if s is not None and l is not None: scores.append(float(s)) losses.append(float(l)) @@ -70,8 +70,8 @@ def _epoch_composite_score(study, trial, epoch: int, ev: Dict[str, Any]) -> Opti # 3. Z-score normalize if we have enough history (>= 10 values) if len(scores) < 10: - # Not enough history: return score only (conservative fallback) - return float(curr_score) + # Not enough history: simple linear composite (score - loss) + return float(curr_score) - float(curr_loss) if curr_loss is not None else float(curr_score) score_mean, score_std = np.mean(scores), np.std(scores) loss_mean, loss_std = np.mean(losses), np.std(losses) diff --git a/src/reporting.py b/src/reporting.py index c6d73cd..06fcb74 100644 --- a/src/reporting.py +++ b/src/reporting.py @@ -6,10 +6,10 @@ from optuna.trial import TrialState from src.db_manager import get_db_session, get_or_create_study_status -from src.schema import TrialResult, AgentReasoningLog, StudyStatus, TrialLease +from src.schema import TrialResult from src.hpo_config import load_hpo_config from src.metrics import get_score, get_loss, loss_objective_index, score_objective_index, TERMINAL_STATES, has_invalid_metrics -from src.hpo_coordinator import compute_health_tier, write_ide_status_file, backfill_review_outcomes +from src.health import compute_health_tier, write_ide_status_file from src.leases import _lease_is_owned, delete_lease_by_trial_id from src.pruning import _epoch_composite_score, _pruning_peer_trials from src.suggest import load_study @@ -149,8 +149,6 @@ def handle_api_report_epoch(req: ReportEpochRequest): # Save user attributes for real-time dashboard monitoring study._storage.set_trial_user_attr(trial_obj._trial_id, "latest_score", final_score) study._storage.set_trial_user_attr(trial_obj._trial_id, "latest_loss", final_loss) - study._storage.set_trial_user_attr(trial_obj._trial_id, "latest_dice", final_score) - study._storage.set_trial_user_attr(trial_obj._trial_id, "latest_bce", final_loss) study._storage.set_trial_user_attr(trial_obj._trial_id, "latest_epoch", req.epoch) study._storage.set_trial_user_attr(trial_obj._trial_id, "gpu_memory", req.gpu_memory) study._storage.set_trial_user_attr(trial_obj._trial_id, "speed_ips", req.speed_ips) @@ -159,14 +157,14 @@ def handle_api_report_epoch(req: ReportEpochRequest): ev = hpo_config.get("eval_protocol", {}) if final_score_fixed is not None: study._storage.set_trial_user_attr( - trial_obj._trial_id, ev.get("fixed_dice_attr", "dice_eval_fixed"), final_score_fixed + trial_obj._trial_id, ev.get("fixed_score_attr", "score_eval_fixed"), final_score_fixed ) study._storage.set_trial_user_attr( trial_obj._trial_id, "score_eval_fixed", final_score_fixed ) if final_loss_fixed is not None: study._storage.set_trial_user_attr( - trial_obj._trial_id, ev.get("fixed_bce_attr", "bce_eval_fixed"), final_loss_fixed + trial_obj._trial_id, ev.get("fixed_loss_attr", "loss_eval_fixed"), final_loss_fixed ) study._storage.set_trial_user_attr( trial_obj._trial_id, "loss_eval_fixed", final_loss_fixed @@ -178,15 +176,11 @@ def handle_api_report_epoch(req: ReportEpochRequest): "epoch": req.epoch, "score": final_score, "loss": final_loss, - "dice": final_score, - "bce": final_loss } if final_score_fixed is not None: epoch_entry["score_eval_fixed"] = final_score_fixed - epoch_entry["dice_eval_fixed"] = final_score_fixed if final_loss_fixed is not None: epoch_entry["loss_eval_fixed"] = final_loss_fixed - epoch_entry["bce_eval_fixed"] = final_loss_fixed history.append(epoch_entry) study._storage.set_trial_user_attr(trial_obj._trial_id, "history", history) @@ -310,12 +304,15 @@ def handle_api_complete_trial(req: CompleteTrialRequest): detail="Rejecting complete: trial reported 0.0 for both score and loss. Likely training did not run.", ) - invalid_metric = has_invalid_metrics(score=final_score, loss=final_loss, score_eval_fixed=final_score_fixed, loss_eval_fixed=final_loss_fixed) - if invalid_metric: - raise HTTPException( - status_code=400, - detail=f"Rejecting complete: {invalid_metric} is NaN or Inf, which is invalid.", - ) + # NaN/Inf rejection only for COMPLETE state — FAIL trials must pass through + # so the failure-tagging and health-monitoring logic below can record them. + if t_state == TrialState.COMPLETE: + invalid_metric = has_invalid_metrics(score=final_score, loss=final_loss, score_eval_fixed=final_score_fixed, loss_eval_fixed=final_loss_fixed) + if invalid_metric: + raise HTTPException( + status_code=400, + detail=f"Rejecting complete: {invalid_metric} is NaN or Inf, which is invalid.", + ) # Check trial metrics health health_tier, health_reason = check_trial_health(study, final_score, final_loss, req.history) @@ -338,14 +335,14 @@ def handle_api_complete_trial(req: CompleteTrialRequest): if final_score_fixed is not None: study._storage.set_trial_user_attr( - trial_obj._trial_id, ev.get("fixed_dice_attr", "dice_eval_fixed"), final_score_fixed + trial_obj._trial_id, ev.get("fixed_score_attr", "score_eval_fixed"), final_score_fixed ) study._storage.set_trial_user_attr( trial_obj._trial_id, "score_eval_fixed", final_score_fixed ) if final_loss_fixed is not None: study._storage.set_trial_user_attr( - trial_obj._trial_id, ev.get("fixed_bce_attr", "bce_eval_fixed"), final_loss_fixed + trial_obj._trial_id, ev.get("fixed_loss_attr", "loss_eval_fixed"), final_loss_fixed ) study._storage.set_trial_user_attr( trial_obj._trial_id, "loss_eval_fixed", final_loss_fixed @@ -443,34 +440,7 @@ def handle_api_complete_trial(req: CompleteTrialRequest): is_minimize_only = len(study.directions) == 1 and study.directions[0] == optuna.study.StudyDirection.MINIMIZE - try: - prior_trials = [t for t in study.trials if t.number < trial_obj.number and t.state == TrialState.COMPLETE] - best_prior_score = 0.0 - - if prior_trials: - if is_minimize_only: - losses = [get_loss(t, study) for t in prior_trials] - losses = [l for l in losses if l is not None] - best_prior_score = min(losses) if losses else 0.0 - else: - scores = [get_score(t, study) for t in prior_trials] - scores = [s for s in scores if s is not None] - best_prior_score = max(scores) if scores else 0.0 - if is_minimize_only: - safe_final_loss = final_loss if final_loss is not None else 0.0 - actual_improvement = best_prior_score - safe_final_loss - else: - safe_final_score = final_score if final_score is not None else 0.0 - actual_improvement = safe_final_score - best_prior_score - with get_db_session() as session: - reasoning_log = session.query(AgentReasoningLog).filter_by(trial_id=req.trial_id).first() - if reasoning_log: - reasoning_log.actual_score_improvement = actual_improvement - session.commit() - except Exception as reas_err: - print(f"Error updating reasoning logs: {reas_err}") - # Compute health tier and update study status try: health_tier, health_reason = compute_health_tier(study, req.study_name) @@ -483,11 +453,6 @@ def handle_api_complete_trial(req: CompleteTrialRequest): except Exception as err: print(f"Error updating coordinator health status: {err}") - try: - backfill_review_outcomes(req.study_name) - except Exception as bf_err: - print(f"Error backfilling review outcomes: {bf_err}") - # Fetch completed scores for sparkline completed_scores = [] for t in study.trials: @@ -505,8 +470,6 @@ def handle_api_complete_trial(req: CompleteTrialRequest): "success": True, "completed_scores": completed_scores, "best_score": best_score, - "completed_dices": completed_scores, - "best_dice": best_score, "trial_number": trial_obj.number } except HTTPException as he: diff --git a/src/routers/dashboard.py b/src/routers/dashboard.py index 97e13a8..35bccce 100644 --- a/src/routers/dashboard.py +++ b/src/routers/dashboard.py @@ -2,7 +2,7 @@ import json import traceback import logging -from typing import Optional, Dict, Any, List +from typing import Optional, Dict, Any from fastapi import APIRouter, HTTPException, Request from fastapi.responses import JSONResponse from pydantic import BaseModel @@ -12,12 +12,8 @@ from ..db_manager import get_db_session, get_or_create_study_status from ..settings import settings from ..schema import ( - StudyStatus, TrialResult, SystemConfiguration, - CoordinatorMetric, - SuggestMetric, - InvalidProposal, ) from ..hpo_config import ( load_hpo_config, @@ -25,36 +21,26 @@ normalize_trial_params, param_display_name, ) -from ..metrics import score_objective_index, _trial_metric_snapshot +from ..metrics import _trial_metric_snapshot from ..search_space import ( - _migrate_search_space, load_search_space, - _apply_search_space_patch, handle_api_get_search_space, handle_api_update_search_space, + _fixed_categorical_params, ) from ..pruning import _effective_train_resolution -from ..suggest import ( - get_or_create_study, - load_study, - _enqueue_manual_trial, -) +from ..suggest import load_study from ..leases import ( _reap_stale_running_trials, _reap_expired_leases, ) -from ..hpo_coordinator import ( +from ..health import compute_health_tier, compute_statistical_confidence, count_evaluated_trials +from ..analytics import ( study_eval_insights as _study_eval_insights, pareto_trial_numbers_deploy_aware as _pareto_trial_numbers_deploy_aware, - build_review_packet, - save_study_review, - get_recent_study_reviews, - count_evaluated_trials, - compute_review_heuristics, - compute_statistical_confidence, - validate_review_fields, - mark_review_applied, - flag_study_review, + build_study_packet, + load_study_cards, + get_fanova_importances, ) logger = logging.getLogger(__name__) @@ -66,25 +52,6 @@ class LoginRequest(BaseModel): token: str -class InitFromManifestRequest(BaseModel): - yaml: str - - -class AgentReviewRequest(BaseModel): - study_name: str - summary: str - health_rating: Optional[int] = None # 1-5 - policy_action: Optional[str] = "no_change" # no_change | update_active_search_space | enqueue_one_manual_trial - model_version: Optional[str] = "coordinator" - prompt_strategy: Optional[str] = "coordinator_review" - reasons: Optional[List[Dict[str, Any]]] = None - search_space_patch: Optional[Dict[str, Any]] = None - manual_trial: Optional[Dict[str, Any]] = None - estimated_score_improvement: Optional[float] = None - cited_best_trial: Optional[int] = None - force: Optional[bool] = False - - @router.post("/login") def api_login(req: LoginRequest, request: Request): """Exchange the shared token for an httpOnly session cookie (dashboard login).""" @@ -137,161 +104,27 @@ def api_update_search_space(space: Dict[str, Any], study_name: Optional[str] = N @router.get("/study_health") def api_get_study_health(study_name: str): try: - study = get_or_create_study(study_name) + study = load_study(study_name) with get_db_session() as session: _reap_stale_running_trials(study, study_name, session) except Exception as reap_err: print(f"study_health reap skipped for '{study_name}': {reap_err}") - with get_db_session() as session: - status = session.query(StudyStatus).filter_by(study_name=study_name).first() - is_dismissed = False - if status: - try: - study = get_or_create_study(study_name) - trials_evaluated = count_evaluated_trials(study) - if status.nudge_dismissed_trials == trials_evaluated: - is_dismissed = True - except Exception as e: - logger.warning(f"Failed to load study {study_name} when checking dismissal status: {e}") - return { - "study_name": study_name, - "health_tier": status.health_tier, - "health_reason": status.health_reason, - "health_updated_at": status.health_updated_at.isoformat() if status.health_updated_at else None, - "is_dismissed": is_dismissed - } - return { - "study_name": study_name, - "health_tier": "healthy", - "health_reason": "No status found, defaulting to healthy.", - "health_updated_at": None, - "is_dismissed": False - } + study = load_study(study_name) + health_tier, health_reason = compute_health_tier(study, study_name) + trials_evaluated = count_evaluated_trials(study) - -@router.get("/pending_changes") -def api_get_pending_changes(study_name: Optional[str] = None): - if not study_name: - study_name = settings.study_name with get_db_session() as session: - row = session.query(SystemConfiguration).filter_by( - study_name=study_name, config_key="pending_search_space" - ).first() - if row: - try: - return {"proposed_changes": json.loads(row.config_value)} - except Exception as e: - return {"proposed_changes": None, "error": str(e)} - return {"proposed_changes": None} - - -@router.post("/apply_pending_changes") -def api_apply_pending_changes(study_name: Optional[str] = None): - if not study_name: - study_name = settings.study_name - try: - with get_db_session() as session: - pending_row = session.query(SystemConfiguration).filter_by( - study_name=study_name, config_key="pending_search_space" - ).first() - if not pending_row: - raise HTTPException(status_code=400, detail="No pending changes found.") - - proposed = json.loads(pending_row.config_value) - current = load_search_space(study_name) - - for key, new_val in proposed.items(): - if key not in current: - raise HTTPException(status_code=400, detail=f"Parameter {key} not in active search space.") - - p_type = current[key].get("type") - if p_type == "categorical": - if "active" in new_val: - allowed = current[key].get("options", []) - invalid = [x for x in new_val["active"] if x not in allowed] - if invalid: - raise HTTPException(status_code=400, detail=f"Invalid active options for {key}: {invalid}") - if not new_val["active"]: - raise HTTPException(status_code=400, detail=f"Must keep at least one active option for {key}.") - current[key]["active"] = new_val["active"] - else: - if "min" in new_val: - current[key]["min"] = float(new_val["min"]) - if "max" in new_val: - current[key]["max"] = float(new_val["max"]) - - session.merge(SystemConfiguration( - study_name=study_name, - config_key="active_search_space", - config_value=json.dumps(_migrate_search_space(current)) - )) - session.delete(pending_row) - mark_review_applied(study_name) - return {"success": True, "space": current} - except HTTPException as he: - raise he - except Exception as e: - raise HTTPException(status_code=500, detail=f"Failed to apply pending changes: {str(e)}") + status = get_or_create_study_status(session, study_name) + status.health_tier = health_tier + status.health_reason = health_reason - -@router.post("/discard_pending_changes") -def api_discard_pending_changes(study_name: Optional[str] = None): - if not study_name: - study_name = settings.study_name - try: - with get_db_session() as session: - row = session.query(SystemConfiguration).filter_by( - study_name=study_name, config_key="pending_search_space" - ).first() - if row: - session.delete(row) - return {"success": True} - except Exception as e: - raise HTTPException(status_code=500, detail=f"Failed to discard pending changes: {str(e)}") - - -@router.post("/validate_manifest") -def api_validate_manifest(req: InitFromManifestRequest): - import yaml - from ..manifest import validate_manifest - try: - data = yaml.safe_load(req.yaml) - except Exception as e: - return {"success": False, "errors": [f"Invalid YAML structure: {str(e)}"], "warnings": []} - - if not isinstance(data, dict): - return {"success": False, "errors": ["Manifest root must be a dictionary"], "warnings": []} - - errors, warnings = validate_manifest(data) - return {"success": len(errors) == 0, "errors": errors, "warnings": warnings} - - -@router.post("/init_from_manifest") -def api_init_from_manifest(req: InitFromManifestRequest, force: bool = False): - import yaml - from ..manifest import validate_manifest - from ..onboarding import init_study_from_manifest_dict - - try: - data = yaml.safe_load(req.yaml) - except Exception as e: - raise HTTPException(status_code=400, detail=f"Invalid YAML structure: {str(e)}") - - if not isinstance(data, dict): - raise HTTPException(status_code=400, detail="Manifest root must be a dictionary") - - errors, warnings = validate_manifest(data) - if errors: - return {"success": False, "errors": errors, "warnings": warnings} - - try: - result = init_study_from_manifest_dict(data, force=force) - return {"success": True, "study_name": data["study_name"], "message": result, "warnings": warnings} - except ValueError as ve: - raise HTTPException(status_code=400, detail=str(ve)) - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + return { + "study_name": study_name, + "health_tier": health_tier, + "health_reason": health_reason, + "trials_evaluated": trials_evaluated, + } @router.get("/studies") @@ -303,41 +136,10 @@ def api_list_studies(): return {"success": False, "error": str(e)} -@router.get("/study_setup") -def api_study_setup(study_name: str): - try: - with get_db_session() as session: - context_row = session.query(SystemConfiguration).filter_by( - study_name=study_name, config_key="project_context" - ).first() - hpo_config_row = session.query(SystemConfiguration).filter_by( - study_name=study_name, config_key="hpo_config" - ).first() - context_val = context_row.config_value if context_row else None - hpo_config_val = hpo_config_row.config_value if hpo_config_row else None - - context = json.loads(context_val) if context_val else {} - hpo_config = json.loads(hpo_config_val) if hpo_config_val else {} - - is_reference = (study_name == "bridge_crack_study") and ("worker_entrypoint" not in context) - - return { - "success": True, - "study_name": study_name, - "worker_entrypoint": context.get("worker_entrypoint"), - "worker_env": context.get("worker_env"), - "is_reference": is_reference, - "manifest_metrics": hpo_config.get("manifest_metrics"), - "colab_snippet": context.get("colab_snippet") - } - except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) - - @router.get("/study_details") def api_study_details(study_name: str): try: - study = get_or_create_study(study_name) + study = load_study(study_name) with get_db_session() as session: _reap_expired_leases(study, study_name, session) @@ -374,8 +176,8 @@ def api_study_details(study_name: str): hpo_config = load_hpo_config(study_name) space = load_search_space(study_name) ev = hpo_config.get("eval_protocol", {}) - dice_fixed_attr = ev.get("fixed_dice_attr", "dice_eval_fixed") - bce_fixed_attr = ev.get("fixed_bce_attr", "bce_eval_fixed") + score_fixed_attr = ev.get("fixed_score_attr", "score_eval_fixed") + loss_fixed_attr = ev.get("fixed_loss_attr", "loss_eval_fixed") train_param = ev.get("train_resolution_param", "resolution") trials_list = [] @@ -387,9 +189,12 @@ def api_study_details(study_name: str): if not history and t._trial_id in metrics_dict: history = metrics_dict[t._trial_id] - metrics = _trial_metric_snapshot(t, history, dice_fixed_attr, bce_fixed_attr, study.directions) + metrics = _trial_metric_snapshot(t, history, score_fixed_attr, loss_fixed_attr, study.directions) train_res = _effective_train_resolution(t, hpo_config, space) norm_params = normalize_trial_params(dict(t.params), hpo_config) + for k, v in _fixed_categorical_params(space).items(): + if k not in norm_params: + norm_params[k] = v if train_res is not None and train_param not in norm_params: norm_params[train_param] = train_res @@ -404,12 +209,8 @@ def api_study_details(study_name: str): "params_display": { param_display_name(k, hpo_config): v for k, v in norm_params.items() }, - "bce": metrics["bce"], - "dice": metrics["dice"], "score": metrics["score"], "loss": metrics["loss"], - "dice_eval_fixed": metrics["dice_eval_fixed"], - "bce_eval_fixed": metrics["bce_eval_fixed"], "score_eval_fixed": metrics["score_eval_fixed"], "loss_eval_fixed": metrics["loss_eval_fixed"], "train_resolution": train_res, @@ -441,8 +242,8 @@ def api_study_details(study_name: str): pareto_trial_numbers = _pareto_trial_numbers_deploy_aware(study, hpo_config) insights = _study_eval_insights(study, hpo_config) - review = compute_review_heuristics(study, insights, hpo_config, study_name) n_complete = sum(1 for t in study.trials if t.state == TrialState.COMPLETE) + health_tier, health_reason = compute_health_tier(study, study_name) return { "study_name": study_name, @@ -452,10 +253,9 @@ def api_study_details(study_name: str): "study_directions": [d.name for d in study.directions], "hpo_config": hpo_config, "eval_insights": insights, - "review": review, + "health": {"tier": health_tier, "reason": health_reason}, "statistical_confidence": compute_statistical_confidence(n_complete), "completed_count": n_complete, - "past_reviews": get_recent_study_reviews(study_name, limit=10), } except Exception as e: traceback.print_exc() @@ -465,12 +265,11 @@ def api_study_details(study_name: str): @router.get("/fanova") def api_fanova(study_name: str): try: - study = get_or_create_study(study_name) + study = load_study(study_name) complete_trials = [t for t in study.trials if t.state == TrialState.COMPLETE] if len(complete_trials) < 2: return {"success": False, "message": "Need at least 2 completed trials for importance analysis"} config = load_hpo_config(study_name) - from ..hpo_coordinator import get_fanova_importances display = get_fanova_importances(study, config) return {"success": True, "importances": display} @@ -478,11 +277,11 @@ def api_fanova(study_name: str): return {"success": False, "message": str(e)} -@router.get("/review_packet") -def api_review_packet(study_name: str): +@router.get("/study_packet") +def api_study_packet(study_name: str): """Read-only context for the IDE coordinator: Pareto, fANOVA, eval insights, drift reasons.""" try: - return build_review_packet(study_name) + return build_study_packet(study_name) except Exception as e: traceback.print_exc() raise HTTPException(status_code=500, detail=str(e)) @@ -492,14 +291,15 @@ def api_review_packet(study_name: str): def api_pareto_front(study_name: str): """Exposes Pareto front trials for export in the GUI dashboard.""" try: - study = get_or_create_study(study_name) + study = load_study(study_name) if len(study.directions) < 2: return {"success": True, "pareto_front": []} hpo_config = load_hpo_config(study_name) + space = load_search_space(study_name) ev = hpo_config.get("eval_protocol", {}) - dice_fixed_attr = ev.get("fixed_dice_attr", "dice_eval_fixed") - bce_fixed_attr = ev.get("fixed_bce_attr", "bce_eval_fixed") + score_fixed_attr = ev.get("fixed_score_attr", "score_eval_fixed") + loss_fixed_attr = ev.get("fixed_loss_attr", "loss_eval_fixed") with get_db_session() as session: metrics = session.query(TrialResult).filter_by(study_name=study_name).all() @@ -512,14 +312,15 @@ def api_pareto_front(study_name: str): if not history and t._trial_id in metrics_dict: history = metrics_dict[t._trial_id] - metrics_vals = _trial_metric_snapshot(t, history, dice_fixed_attr, bce_fixed_attr, study.directions) + metrics_vals = _trial_metric_snapshot(t, history, score_fixed_attr, loss_fixed_attr, study.directions) norm_params = normalize_trial_params(dict(t.params), hpo_config) + for k, v in _fixed_categorical_params(space).items(): + if k not in norm_params: + norm_params[k] = v pareto_trials.append({ "number": t.number, "trial_id": t._trial_id, - "bce": metrics_vals["bce"], - "dice": metrics_vals["dice"], "score": metrics_vals["score"], "loss": metrics_vals["loss"], "params": norm_params @@ -531,24 +332,6 @@ def api_pareto_front(study_name: str): raise HTTPException(status_code=500, detail=str(e)) -@router.post("/dismiss_coordinator_nudge") -def api_dismiss_coordinator_nudge(study_name: str): - """Dismisses the coordinator nudge for the current trial window by persisting it in SQLite.""" - try: - study = get_or_create_study(study_name) - trials_evaluated = count_evaluated_trials(study) - - with get_db_session() as session: - status = get_or_create_study_status(session, study_name) - status.nudge_dismissed_trials = trials_evaluated - session.commit() - - return {"success": True, "dismissed_trials": trials_evaluated} - except Exception as e: - traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) - - @router.get("/study_cards") def api_get_study_cards(study_name: Optional[str] = None): """Exposes generated study cards and their markdown content for dashboard retrieval.""" @@ -559,71 +342,6 @@ def api_get_study_cards(study_name: Optional[str] = None): raise HTTPException(status_code=500, detail=str(e)) -@router.post("/agent_review") -def api_agent_review(req: AgentReviewRequest): - """Persist a coordinator review. Idempotent per trial window unless force=True.""" - try: - study = load_study(req.study_name) - space = load_search_space(req.study_name) - trials_evaluated = count_evaluated_trials(study) - - if req.manual_trial: - from ..hpo_coordinator import _validate_manual_parameters - val_res = _validate_manual_parameters(req.manual_trial, req.study_name) - if not val_res["ok"]: - with get_db_session() as session: - session.add(InvalidProposal( - study_name=req.study_name, - model_version=req.model_version or "coordinator", - prompt_strategy=req.prompt_strategy or "coordinator_review", - invalid_parameters=json.dumps(req.manual_trial), - validation_error=val_res["error"] - )) - return {"success": False, "error": f"Invalid manual parameters: {val_res['error']}"} - - validation = validate_review_fields(req.estimated_score_improvement, req.cited_best_trial) - if not validation["ok"]: - return {"success": False, "error": "; ".join(validation["errors"])} - result = save_study_review( - req.study_name, - req.summary, - health_rating=req.health_rating, - policy_action=req.policy_action or "no_change", - model_version=req.model_version or "coordinator", - prompt_strategy=req.prompt_strategy or "coordinator_review", - reasons=req.reasons, - trials_evaluated=trials_evaluated, - estimated_score_improvement=req.estimated_score_improvement, - cited_best_trial=req.cited_best_trial, - force=bool(req.force), - ) - - applied = {} - if not result.get("duplicate"): - if req.search_space_patch: - applied["search_space"] = _apply_search_space_patch(req.search_space_patch, space, req.study_name) - if req.manual_trial: - applied["manual_trial"] = _enqueue_manual_trial(study, req.manual_trial, space, req.summary) - - result["applied"] = applied - return result - except HTTPException: - raise - except Exception as e: - traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) - - -@router.post("/flag_review") -def api_flag_review(review_id: int, flagged: bool = True): - """Mark a coordinator review as low-quality (excluded from accuracy MAE).""" - try: - return flag_study_review(review_id, flagged=flagged) - except Exception as e: - traceback.print_exc() - raise HTTPException(status_code=500, detail=str(e)) - - @router.get("/tunnel_url") def api_get_tunnel_url(): """Returns the active remote broker URL if established.""" @@ -656,7 +374,7 @@ def api_config_audit(study_name: Optional[str] = None): db_space = load_search_space(study_name) try: - study = get_or_create_study(study_name) + study = load_study(study_name) for t in study.trials: if t.state not in (TrialState.COMPLETE, TrialState.RUNNING): continue @@ -699,42 +417,67 @@ def api_config_audit(study_name: Optional[str] = None): return report -@router.get("/metrics/coordinator") -def api_metrics_coordinator(study_name: Optional[str] = None): - if not study_name: - study_name = settings.study_name - with get_db_session() as session: - rows = session.query(CoordinatorMetric).filter_by(study_name=study_name).all() - return {"success": True, "metrics": [r.to_dict() for r in rows]} +@router.post("/quickstart_demo") +def api_quickstart_demo(request: Request): + from ..onboarding import init_study_from_manifest_dict + from threading import Thread + from simulators.training_worker import run_training_worker + DEMO_STUDY = "demo_segmentation_study" -@router.get("/metrics/suggest") -def api_metrics_suggest(study_name: Optional[str] = None): - if not study_name: - study_name = settings.study_name - with get_db_session() as session: - rows = session.query(SuggestMetric).filter_by(study_name=study_name).all() - return {"success": True, "metrics": [r.to_dict() for r in rows]} + # If the demo study already exists and has trials, just redirect — no re-spawn needed + try: + existing = optuna.load_study(study_name=DEMO_STUDY, storage=settings.database_url) + if len(existing.trials) > 0: + return {"success": True, "study_name": DEMO_STUDY} + except KeyError: + pass # Study doesn't exist yet — proceed + + DEMO_MANIFEST = { + "study_name": DEMO_STUDY, + "metrics": { + "primary_score": "score", + "objectives": [ + {"name": "loss", "direction": "minimize", "label": "Loss"}, + {"name": "score", "direction": "maximize", "label": "Score"}, + ], + }, + "params": [ + {"name": "learning_rate", "type": "float_log", "min": 0.0001, "max": 0.1}, + {"name": "batch_size", "type": "categorical", "options": [4, 8, 16, 32]}, + {"name": "resolution", "type": "categorical", "options": [256, 512, 1024]}, + {"name": "loss_weight_ratio", "type": "float", "min": 0.0, "max": 1.0}, + {"name": "model_capacity", "type": "categorical", "options": ["narrow", "wide"]}, + ], + "worker": {"entrypoint": "python simulators/training_worker.py"}, + } + + init_study_from_manifest_dict(DEMO_MANIFEST, force=True) + + broker_url = settings.broker_url or str(request.base_url).rstrip("/") + Thread( + target=run_training_worker, + args=(DEMO_STUDY,), + kwargs={"max_trials": 5, "broker_url": broker_url}, + daemon=True, + ).start() + return {"success": True, "study_name": DEMO_STUDY} -@router.get("/mcp_info") -def api_mcp_info(): + +@router.get("/worker_snippet") +def api_worker_snippet(study_name: str): + broker_url = settings.broker_url or "http://localhost:8000" + secret_token = settings.secret_token return { "success": True, - "mcp_server_name": "pathfinder", - "active_study": settings.study_name, - "mcp_tools": [ - "initialize_study", - "get_study_data", - "validate_search_space", - "update_search_space", - "delete_study", - "generate_model_card", - "submit_agent_review", - "validate_integration", - "get_study_cards", - "validate_manifest", - "init_from_manifest", - "export_manifest", - ] + "broker_url": broker_url, + "study_name": study_name, + "auth_required": bool(secret_token), + "snippet": ( + f"export HPO_BROKER_URL={broker_url}\n" + f"export HPO_STUDY_NAME={study_name}\n" + + (f"export HPO_SECRET_TOKEN={secret_token}\n" if secret_token else "") + + "python your_worker.py" + ), } diff --git a/src/routers/static.py b/src/routers/static.py index ed8b531..c8ebc2d 100644 --- a/src/routers/static.py +++ b/src/routers/static.py @@ -40,13 +40,6 @@ def get_styles(): ) raise HTTPException(status_code=404, detail="styles.css not found") -@router.get("/colab_worker.py") -def get_colab_worker(): - worker_path = os.path.join(_base_dir, "colab_worker.py") - if os.path.exists(worker_path): - return FileResponse(worker_path, media_type="text/x-python", filename="colab_worker.py") - raise HTTPException(status_code=404, detail="colab_worker.py not found") - @router.get("/hpo_client.py") def get_hpo_client(): client_path = os.path.join(_base_dir, "src", "hpo_client.py") diff --git a/src/routers/worker.py b/src/routers/worker.py index bc0ff0a..a48bcb3 100644 --- a/src/routers/worker.py +++ b/src/routers/worker.py @@ -12,13 +12,13 @@ def api_suggest_trial_help(): "error": "Method not allowed: use POST, not GET", "post_url": "/api/suggest_trial", "body_example": { - "study_name": "bridge_crack_study", + "study_name": "my_study", "reasoning": "Autonomous worker suggestion request.", }, "curl_example": ( 'curl -X POST "$BROKER_URL/api/suggest_trial" ' '-H "Content-Type: application/json" ' - '-d \'{"study_name":"bridge_crack_study"}\'' + '-d \'{"study_name":"my_study"}\'' ), } diff --git a/src/schema.py b/src/schema.py index 5237599..21d0672 100644 --- a/src/schema.py +++ b/src/schema.py @@ -1,7 +1,8 @@ import datetime import json +from datetime import timezone from typing import Optional, List, Dict, Any -from sqlalchemy import String, Integer, Float, DateTime, Text, ForeignKey, Boolean, UniqueConstraint +from sqlalchemy import String, Integer, Float, DateTime, Text, Boolean, UniqueConstraint from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column class Base(DeclarativeBase): @@ -27,7 +28,7 @@ class TrialResult(Base): health_tier: Mapped[Optional[str]] = mapped_column(String(50), nullable=True) health_reason: Mapped[Optional[str]] = mapped_column(Text, nullable=True) created_at: Mapped[datetime.datetime] = mapped_column( - DateTime, default=datetime.datetime.utcnow + DateTime, default=lambda: datetime.datetime.now(timezone.utc).replace(tzinfo=None) ) def get_history(self) -> List[Dict[str, Any]]: @@ -71,7 +72,7 @@ class TrialMetadata(Base): meta_key: Mapped[str] = mapped_column(String(100), nullable=False) meta_value: Mapped[str] = mapped_column(Text, nullable=False) created_at: Mapped[datetime.datetime] = mapped_column( - DateTime, default=datetime.datetime.utcnow + DateTime, default=lambda: datetime.datetime.now(timezone.utc).replace(tzinfo=None) ) def to_dict(self) -> Dict[str, Any]: @@ -92,7 +93,7 @@ class SystemConfiguration(Base): config_value: Mapped[str] = mapped_column(Text, nullable=False) version: Mapped[int] = mapped_column(Integer, default=1) updated_at: Mapped[datetime.datetime] = mapped_column( - DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow + DateTime, default=lambda: datetime.datetime.now(timezone.utc).replace(tzinfo=None), onupdate=lambda: datetime.datetime.now(timezone.utc).replace(tzinfo=None) ) def to_dict(self) -> Dict[str, Any]: @@ -113,7 +114,7 @@ class CompactedPacket(Base): trials_evaluated: Mapped[int] = mapped_column(Integer, nullable=False) packet_json: Mapped[str] = mapped_column(Text, nullable=False) created_at: Mapped[datetime.datetime] = mapped_column( - DateTime, default=datetime.datetime.utcnow + DateTime, default=lambda: datetime.datetime.now(timezone.utc).replace(tzinfo=None) ) class StudyCard(Base): @@ -126,7 +127,7 @@ class StudyCard(Base): content_hash: Mapped[str] = mapped_column(String(64), nullable=False) metadata_json: Mapped[Optional[str]] = mapped_column(Text, nullable=True) created_at: Mapped[datetime.datetime] = mapped_column( - DateTime, default=datetime.datetime.utcnow + DateTime, default=lambda: datetime.datetime.now(timezone.utc).replace(tzinfo=None) ) def to_dict(self) -> Dict[str, Any]: @@ -140,92 +141,9 @@ def to_dict(self) -> Dict[str, Any]: "created_at": self.created_at.isoformat() if self.created_at else None, } -class AgentReasoningLog(Base): - __tablename__ = "agent_reasoning_logs" - trial_id: Mapped[int] = mapped_column(Integer, primary_key=True) - study_name: Mapped[str] = mapped_column(String(200), nullable=False) - model_version: Mapped[str] = mapped_column(String(100), nullable=False) - prompt_strategy: Mapped[str] = mapped_column(String(100), nullable=False) - predicted_outcome_rationale: Mapped[str] = mapped_column(Text, nullable=False) - estimated_score_improvement: Mapped[float] = mapped_column(Float, nullable=False) - actual_score_improvement: Mapped[Optional[float]] = mapped_column(Float, nullable=True) - created_at: Mapped[datetime.datetime] = mapped_column( - DateTime, default=datetime.datetime.utcnow - ) - - def to_dict(self) -> Dict[str, Any]: - return { - "trial_id": self.trial_id, - "study_name": self.study_name, - "model_version": self.model_version, - "prompt_strategy": self.prompt_strategy, - "predicted_outcome_rationale": self.predicted_outcome_rationale, - "estimated_score_improvement": self.estimated_score_improvement, - "actual_score_improvement": self.actual_score_improvement, - "created_at": self.created_at.isoformat() if self.created_at else None, - } - -class StudyReview(Base): - __tablename__ = "study_reviews" - - id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) - study_name: Mapped[str] = mapped_column(String(200), nullable=False) - health_rating: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) # 1-5 - summary: Mapped[str] = mapped_column(Text, nullable=False) - policy_action: Mapped[str] = mapped_column(String(60), nullable=False, default="no_change") - model_version: Mapped[str] = mapped_column(String(100), nullable=False, default="unspecified") - prompt_strategy: Mapped[str] = mapped_column(String(100), nullable=False, default="coordinator_review") - reasons_json: Mapped[Optional[str]] = mapped_column(Text, nullable=True) # JSON list of trigger reasons - trials_evaluated: Mapped[int] = mapped_column(Integer, nullable=False, default=0) # idempotency window key - estimated_score_improvement: Mapped[Optional[float]] = mapped_column(Float, nullable=True) - cited_best_trial: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) - confidence: Mapped[Optional[str]] = mapped_column(String(50), nullable=True, default="high") - baseline_best_score: Mapped[Optional[float]] = mapped_column(Float, nullable=True) - applied_at_completed_count: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) - applied_at: Mapped[Optional[datetime.datetime]] = mapped_column(DateTime, nullable=True) - actual_score_improvement: Mapped[Optional[float]] = mapped_column(Float, nullable=True) - outcome_measured_at: Mapped[Optional[datetime.datetime]] = mapped_column(DateTime, nullable=True) - outcome_status: Mapped[str] = mapped_column(String(30), nullable=False, default="pending") - quality_flagged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) - created_at: Mapped[datetime.datetime] = mapped_column( - DateTime, default=datetime.datetime.utcnow - ) - def get_reasons(self) -> List[Dict[str, Any]]: - if not self.reasons_json: - return [] - try: - return json.loads(self.reasons_json) - except Exception: - return [] - def set_reasons(self, reasons: Optional[List[Dict[str, Any]]]): - self.reasons_json = json.dumps(reasons or []) - - def to_dict(self) -> Dict[str, Any]: - return { - "id": self.id, - "study_name": self.study_name, - "health_rating": self.health_rating, - "summary": self.summary, - "policy_action": self.policy_action, - "model_version": self.model_version, - "prompt_strategy": self.prompt_strategy, - "reasons": self.get_reasons(), - "trials_evaluated": self.trials_evaluated, - "estimated_score_improvement": self.estimated_score_improvement, - "cited_best_trial": self.cited_best_trial, - "confidence": self.confidence or "high", - "baseline_best_score": self.baseline_best_score, - "applied_at_completed_count": self.applied_at_completed_count, - "applied_at": self.applied_at.isoformat() if self.applied_at else None, - "actual_score_improvement": self.actual_score_improvement, - "outcome_measured_at": self.outcome_measured_at.isoformat() if self.outcome_measured_at else None, - "outcome_status": self.outcome_status, - "quality_flagged": self.quality_flagged, - "created_at": self.created_at.isoformat() if self.created_at else None, - } class StudyStatus(Base): __tablename__ = "study_status" @@ -234,7 +152,7 @@ class StudyStatus(Base): health_tier: Mapped[str] = mapped_column(String(50), default="healthy") health_reason: Mapped[Optional[str]] = mapped_column(Text, nullable=True) health_updated_at: Mapped[datetime.datetime] = mapped_column( - DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow + DateTime, default=lambda: datetime.datetime.now(timezone.utc).replace(tzinfo=None), onupdate=lambda: datetime.datetime.now(timezone.utc).replace(tzinfo=None) ) nudge_dismissed_trials: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) @@ -247,38 +165,7 @@ def to_dict(self) -> Dict[str, Any]: "nudge_dismissed_trials": self.nudge_dismissed_trials, } -class InvalidProposal(Base): - __tablename__ = "invalid_proposals" - id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) - study_name: Mapped[str] = mapped_column(String(200), nullable=False) - model_version: Mapped[str] = mapped_column(String(100), nullable=False) - prompt_strategy: Mapped[str] = mapped_column(String(100), nullable=False) - invalid_parameters: Mapped[str] = mapped_column(Text, nullable=False) # JSON string of parameters proposed - validation_error: Mapped[str] = mapped_column(Text, nullable=False) # Reason/exception string - created_at: Mapped[datetime.datetime] = mapped_column( - DateTime, default=datetime.datetime.utcnow - ) - - def get_parameters(self) -> Dict[str, Any]: - try: - return json.loads(self.invalid_parameters) - except Exception: - return {} - - def set_parameters(self, params: Dict[str, Any]): - self.invalid_parameters = json.dumps(params) - - def to_dict(self) -> Dict[str, Any]: - return { - "id": self.id, - "study_name": self.study_name, - "model_version": self.model_version, - "prompt_strategy": self.prompt_strategy, - "invalid_parameters": self.get_parameters(), - "validation_error": self.validation_error, - "created_at": self.created_at.isoformat() if self.created_at else None, - } class TrialLease(Base): __tablename__ = "trial_leases" @@ -296,42 +183,6 @@ def to_dict(self) -> Dict[str, Any]: "lease_expires_at": self.lease_expires_at.isoformat() if self.lease_expires_at else None } -class CoordinatorMetric(Base): - __tablename__ = "coordinator_metrics" - id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) - study_name: Mapped[str] = mapped_column(String(200), nullable=False) - timestamp: Mapped[datetime.datetime] = mapped_column(DateTime, default=datetime.datetime.utcnow) - model: Mapped[str] = mapped_column(String(100), nullable=False) - latency_ms: Mapped[float] = mapped_column(Float, nullable=False) - action_taken: Mapped[str] = mapped_column(String(100), nullable=False) - trials_at_review: Mapped[int] = mapped_column(Integer, nullable=False) - def to_dict(self) -> Dict[str, Any]: - return { - "id": self.id, - "study_name": self.study_name, - "timestamp": self.timestamp.isoformat() if self.timestamp else None, - "model": self.model, - "latency_ms": self.latency_ms, - "action_taken": self.action_taken, - "trials_at_review": self.trials_at_review - } - -class SuggestMetric(Base): - __tablename__ = "suggest_metrics" - - id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) - study_name: Mapped[str] = mapped_column(String(200), nullable=False) - timestamp: Mapped[datetime.datetime] = mapped_column(DateTime, default=datetime.datetime.utcnow) - latency_ms: Mapped[float] = mapped_column(Float, nullable=False) - source: Mapped[str] = mapped_column(String(50), nullable=False) # new_trial, reclaimed_lease, recycled_running - def to_dict(self) -> Dict[str, Any]: - return { - "id": self.id, - "study_name": self.study_name, - "timestamp": self.timestamp.isoformat() if self.timestamp else None, - "latency_ms": self.latency_ms, - "source": self.source - } diff --git a/src/search_space.py b/src/search_space.py index f429611..4f80448 100644 --- a/src/search_space.py +++ b/src/search_space.py @@ -5,22 +5,16 @@ logger = logging.getLogger(__name__) from fastapi import HTTPException -import optuna -from optuna.distributions import CategoricalDistribution from optuna.trial import TrialState from src.db_manager import get_db_session -from src.schema import SystemConfiguration +from src.schema import SystemConfiguration, CompactedPacket from src.hpo_config import load_hpo_config -from src.hpo_coordinator import mark_review_applied # Default search space definition DEFAULT_SEARCH_SPACE = { "learning_rate": {"min": 1e-5, "max": 1e-2, "type": "float_log"}, - "batch_size": {"options": [2, 4, 8, 16, 32, 64], "active": [2, 4, 8, 16, 32, 64], "type": "categorical"}, - "resolution": {"options": [256, 512, 1024], "active": [256, 512, 1024], "type": "categorical"}, - "model_capacity": {"options": ["narrow", "wide"], "active": ["narrow", "wide"], "type": "categorical"}, - "loss_weight_ratio": {"min": 0.0, "max": 1.0, "type": "float"}, + "batch_size": {"options": [16, 32, 64, 128], "active": [16, 32, 64, 128], "type": "categorical"}, } @@ -40,7 +34,9 @@ def load_search_space(study_name: Optional[str] = None) -> Dict[str, Any]: from .settings import settings if not study_name: study_name = settings.study_name - + if not study_name or not study_name.strip(): + return DEFAULT_SEARCH_SPACE.copy() + try: with get_db_session() as session: row = session.query(SystemConfiguration).filter_by( @@ -64,6 +60,8 @@ def save_search_space(space: Dict[str, Any], study_name: Optional[str] = None): from .settings import settings if not study_name: study_name = settings.study_name + if not study_name or not study_name.strip(): + raise ValueError("study_name cannot be empty.") try: with get_db_session() as session: row = session.query(SystemConfiguration).filter_by( @@ -82,6 +80,13 @@ def save_search_space(space: Dict[str, Any], study_name: Optional[str] = None): except Exception as e: print(f"Error saving search space to DB: {e}") + # Bust cached study packets — search space changes invalidate analytics + try: + with get_db_session() as session: + session.query(CompactedPacket).filter_by(study_name=study_name).delete() + except Exception: + pass + def _expected_search_params(space: Dict[str, Any]) -> List[str]: return [k for k, v in space.items() if isinstance(v, dict) and v.get("type")] @@ -103,14 +108,22 @@ def _worker_ready_params(trial, space: Dict[str, Any]) -> Dict[str, Any]: def _cleanup_stuck_running_trials(study, space: Dict[str, Any]) -> None: - """Fail RUNNING trials that never received a full parameter set (crashed mid-suggest).""" + """Fail RUNNING trials that never received a full parameter set (crashed mid-suggest). + + Trials that started less than LEASE_TTL_SECONDS ago are skipped to avoid a race + where worker A's trial was just created by ``study.ask()`` but hasn't yet had + its parameters written when worker B's ``/api/suggest_trial`` triggers cleanup. + """ + from .leases import LEASE_TTL_SECONDS + from datetime import datetime, timedelta, timezone + cutoff = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(seconds=LEASE_TTL_SECONDS) for t in list(study.trials): - if t.state == TrialState.RUNNING and not _trial_has_full_params( - _worker_ready_params(t, space), space - ): - print( - f"Failing stuck RUNNING trial #{t.number} (incomplete params: {list(t.params.keys())})" - ) + if t.state != TrialState.RUNNING: + continue + if not _trial_has_full_params(_worker_ready_params(t, space), space): + if t.datetime_start is not None and t.datetime_start.replace(tzinfo=None) > cutoff: + continue + print(f"Failing stuck RUNNING trial #{t.number} (incomplete params: {list(t.params.keys())})") try: study.tell(t.number, state=TrialState.FAIL) except Exception as exc: @@ -152,21 +165,6 @@ def _finalize_trial_params(params: Dict[str, Any], space: Dict[str, Any]) -> Dic return out -def _persist_fixed_categorical_params(study, trial, space: Dict[str, Any]) -> None: - """Write single-active categoricals into Optuna storage so dashboards see them.""" - fixed = _fixed_categorical_params(space) - for param, value in fixed.items(): - if param in trial.params: - continue - cfg = space.get(param, {}) - choices = tuple(_study_categorical_choices(study, param, cfg)) - dist = CategoricalDistribution(choices=choices) - # Optuna RDB storage expects internal index (0..n-1), not external choice value. - internal = float(dist.to_internal_repr(value)) - study._storage.set_trial_param(trial._trial_id, param, internal, dist) - study._storage.set_trial_user_attr(trial._trial_id, param, value) - - def _enqueue_single_active_categoricals(study, space: Dict[str, Any]) -> None: """Optional hint for Optuna; workers still receive fixed values via _finalize_trial_params.""" fixed = _fixed_categorical_params(space) @@ -185,7 +183,7 @@ def _suggest_categorical_compatible(study, trial, param: str, cfg: Dict[str, Any trial.suggest_categorical(param, choices) -def _validate_params_against_active(params: Dict[str, Any], space: Dict[str, Any]) -> List[str]: +def _validate_categorical_against_active(params: Dict[str, Any], space: Dict[str, Any]) -> List[str]: """Return list of human-readable violations when TPE samples outside active constraints.""" errors = [] for param, cfg in space.items(): @@ -210,12 +208,14 @@ def suggest_params_from_space(study, trial, space: Dict[str, Any]) -> Dict[str, trial.suggest_float(param, float(cfg["min"]), float(cfg["max"]), log=True) elif ptype == "float": trial.suggest_float(param, float(cfg["min"]), float(cfg["max"])) + elif ptype == "int": + trial.suggest_int(param, int(cfg["min"]), int(cfg["max"])) elif ptype == "categorical": _suggest_categorical_compatible(study, trial, param, cfg) else: raise ValueError(f"Unsupported parameter type '{ptype}' for '{param}'.") params = _finalize_trial_params(trial.params, space) - violations = _validate_params_against_active(params, space) + violations = _validate_categorical_against_active(params, space) if violations: raise ValueError( "Sampled parameters outside active search bounds: " @@ -246,7 +246,6 @@ def _apply_search_space_patch(patch: Dict[str, Any], space: Dict[str, Any], stud if "max" in new_val: cfg["max"] = float(new_val["max"]) save_search_space(space, study_name) - mark_review_applied(study_name) return "Search space updated." @@ -301,40 +300,26 @@ def handle_api_update_search_space(space: Dict[str, Any], study_name: Optional[s detail=f"Hyperparameter '{param_name}' is not recognized in the search space." ) - # Save to pending_search_space in DB - try: - with get_db_session() as session: - # Check if there is already a pending configuration - pending_row = session.query(SystemConfiguration).filter_by( - study_name=study_name, config_key="pending_search_space" - ).first() - - if validated_proposals: - if pending_row: - # Merge with existing pending configuration - existing_pending = json.loads(pending_row.config_value) - for key, val in validated_proposals.items(): - if key in existing_pending: - existing_pending[key].update(val) - else: - existing_pending[key] = val - pending_row.config_value = json.dumps(existing_pending) - pending_row.version += 1 - else: - session.add(SystemConfiguration( - study_name=study_name, - config_key="pending_search_space", - config_value=json.dumps(validated_proposals), - version=1 - )) - else: - # If proposals are empty (reverted to current), delete pending row if it exists - if pending_row: - session.delete(pending_row) - session.commit() - except HTTPException as he: - raise he - except Exception as e: - raise HTTPException(status_code=500, detail=f"Failed to save pending changes: {str(e)}") - - return {"success": True, "space": current, "pending": validated_proposals} + # Apply validated proposals directly to active search space + if validated_proposals: + try: + for param_name, proposal in validated_proposals.items(): + current[param_name].update(proposal) + # Validate min < max for all numeric params after merge + for param_name, cfg in current.items(): + if not isinstance(cfg, dict) or cfg.get("type") not in ("float", "float_log", "int"): + continue + lo = cfg.get("min") + hi = cfg.get("max") + if lo is not None and hi is not None and float(lo) >= float(hi): + raise HTTPException( + status_code=400, + detail=f"Invalid bounds for '{param_name}': min ({lo}) must be strictly less than max ({hi}).", + ) + save_search_space(current, study_name) + except HTTPException: + raise + except Exception as e: + raise HTTPException(status_code=500, detail=f"Failed to update search space: {str(e)}") + + return {"success": True, "space": current} diff --git a/src/settings.py b/src/settings.py index 5a97a82..deea16c 100644 --- a/src/settings.py +++ b/src/settings.py @@ -2,7 +2,7 @@ from typing import Optional, List class Settings: - DEFAULT_DB_FILENAME = "hpo_studies.db" + DEFAULT_DB_FILENAME = ".data/hpo_studies.db" @property def database_url(self) -> str: diff --git a/src/suggest.py b/src/suggest.py index 0e2d150..77cedea 100644 --- a/src/suggest.py +++ b/src/suggest.py @@ -1,8 +1,6 @@ -import time import json -from datetime import datetime, timedelta -from typing import Optional, Dict, Any, List +from typing import Optional from pydantic import BaseModel from fastapi import HTTPException import optuna @@ -11,7 +9,6 @@ from sqlalchemy import text from src.db_manager import get_db_session, DATABASE_URL -from src.schema import AgentReasoningLog, SuggestMetric, TrialLease from src.hpo_config import load_hpo_config, normalize_trial_params from src.search_space import ( load_search_space, @@ -20,22 +17,30 @@ _worker_ready_params, _enqueue_single_active_categoricals, suggest_params_from_space, - _persist_fixed_categorical_params, _finalize_trial_params, _expected_search_params, ) -from src.leases import _reap_expired_leases, _try_claim_lease +from src.leases import _reap_expired_leases, _try_claim_lease, delete_lease_by_trial_id def get_or_create_study(study_name: str): try: return optuna.load_study(study_name=study_name, storage=DATABASE_URL) except KeyError: - print(f"Study '{study_name}' not found. Initializing new multi-objective study...") + print(f"Study '{study_name}' not found. Checking stored config for directions...") + directions = None + try: + cfg = load_hpo_config(study_name) + directions = cfg.get("directions") + except Exception: + pass + if not directions: + directions = ["minimize", "maximize"] + print("No stored directions found. Defaulting to multi-objective (minimize, maximize).") return optuna.create_study( study_name=study_name, storage=DATABASE_URL, - directions=["minimize", "maximize"], + directions=directions, load_if_exists=True ) @@ -83,14 +88,25 @@ def _repair_categorical_param_indices(session: Session, study_name: str) -> int: try: dist_data = json.loads(dist_json) choices = dist_data["attributes"]["choices"] - idx = int(float(param_value)) - if 0 <= idx < len(choices): - continue - external = float(param_value) - if external not in choices and int(external) not in choices: + # Determine if param_value is stored as an internal index or an external value. + # If it can be parsed as a float, check whether it matches a choice BY VALUE first. + try: + external = float(param_value) + except (ValueError, TypeError): + external = param_value + # If the stored value equals one of the choices literally, treat it as an + # external value and convert to its internal index. Otherwise, assume it is + # already a valid internal index. + if external in choices: + internal = choices.index(external) + elif isinstance(external, float) and int(external) in choices: + internal = choices.index(int(external)) + else: + # Already an internal index or unrecognised; skip repair. + idx = int(float(param_value)) + if 0 <= idx < len(choices): + continue continue - value = int(external) if int(external) in choices else external - internal = choices.index(value) session.execute( text( "UPDATE trial_params SET param_value = :v WHERE param_id = :id" @@ -106,15 +122,20 @@ def _repair_categorical_param_indices(session: Session, study_name: str) -> int: def handle_api_suggest_trial(req: SuggestRequest): trial = None - start_time = time.time() try: + # Load the study first to verify it exists. This raises a clean 404 if the study is uninitialized. + study = load_study(req.study_name) + # C2 Fix: thread session explicitly with get_db_session() as session: - repaired = _repair_categorical_param_indices(session, req.study_name) - if repaired: - print(f"Repaired {repaired} corrupt categorical param row(s) in study '{req.study_name}'.") - - study = load_study(req.study_name) + try: + repaired = _repair_categorical_param_indices(session, req.study_name) + if repaired: + print(f"Repaired {repaired} corrupt categorical param row(s) in study '{req.study_name}'.") + except Exception as e: + # Fallback: if there is a database issue during repair, do not crash suggestion + import logging + logging.getLogger(__name__).warning(f"Failed to repair categorical param indices: {e}") space = load_search_space(req.study_name) hpo_config = load_hpo_config(req.study_name) _cleanup_stuck_running_trials(study, space) @@ -131,7 +152,6 @@ def handle_api_suggest_trial(req: SuggestRequest): and _trial_has_full_params(_worker_ready_params(t, space), space) ] - source = "recycled_running" if running_trials: running_trials.sort(key=lambda t: t.number) leased_to = req.worker_id or "anonymous" @@ -149,46 +169,54 @@ def handle_api_suggest_trial(req: SuggestRequest): # 3. If no RUNNING trial is available, ask Optuna for a new one if not trial: - source = "new_trial" _enqueue_single_active_categoricals(study, space) - trial = study.ask() - trial_id = trial._trial_id - # Lease newly created trial immediately (fresh trial_id, so this always wins). - with get_db_session() as session: - _try_claim_lease(session, req.study_name, trial_id, req.worker_id or "anonymous") + # Optuna does not support narrowing categorical distributions after the first + # trial. TPE always samples from the full historical choice set. When a user + # deactivates a choice (dashboard Settings), we must handle TPE sampling an + # inactive value. We retry up to 20 times, failing each attempt in Optuna. + # This tells TPE that the deactivated region is unproductive — a best-effort + # heuristic given Optuna's static-distribution design. On the final attempt, + # we substitute a random active choice instead of 500-ing the worker. + MAX_RESAMPLE_ATTEMPTS = 20 + for attempt in range(MAX_RESAMPLE_ATTEMPTS): + trial = study.ask() + trial_id = trial._trial_id - try: - params = suggest_params_from_space(study, trial, space) - _persist_fixed_categorical_params(study, trial, space) - except Exception: - try: - study.tell(trial.number, state=TrialState.FAIL) - except Exception: - pass with get_db_session() as session: - delete_lease_by_trial_id(session, trial_id) - session.commit() - raise - - with get_db_session() as session: - existing = ( - session.query(AgentReasoningLog).filter_by(trial_id=trial_id).first() - ) - if not existing: - est_imp = req.estimated_score_improvement - session.add( - AgentReasoningLog( - trial_id=trial_id, - study_name=req.study_name, - model_version=req.agent_model or "optuna-tpe", - prompt_strategy=req.prompt_strategy or "tpe_sampler", - predicted_outcome_rationale=req.reasoning or "Autonomous worker suggestion request.", - estimated_score_improvement=float(est_imp if est_imp is not None else 0.0), - ) - ) - session.commit() + claimed = _try_claim_lease(session, req.study_name, trial_id, req.worker_id or "anonymous") + assert claimed, f"Lease claim failed for trial {trial_id}" + try: + params = suggest_params_from_space(study, trial, space) + break + except ValueError: + # TPE sampled an inactive categorical + if attempt < MAX_RESAMPLE_ATTEMPTS - 1: + # Fail the trial and retry with a fresh one + try: + study.tell(trial.number, state=TrialState.FAIL) + except Exception: + pass + with get_db_session() as session: + delete_lease_by_trial_id(session, trial_id) + session.commit() + continue + # Final attempt — fallback: pick random active values for violated categoricals + import random + fixed = _finalize_trial_params(dict(trial.params), space) + for param, cfg in space.items(): + if isinstance(cfg, dict) and cfg.get("type") == "categorical": + active = list(cfg.get("active") or cfg.get("options") or []) + if active and fixed.get(param) not in active: + fallback_choice = random.choice(active) + fixed[param] = fallback_choice + print( + f"WARNING: Trial {trial.number} — {param} " + f"outside active {active}. Falling back to {fallback_choice!r}." + ) + params = fixed + params = _finalize_trial_params(params, space) missing = [p for p in _expected_search_params(space) if p not in params] if missing: @@ -204,20 +232,6 @@ def handle_api_suggest_trial(req: SuggestRequest): detail=f"Trial {trial.number} missing parameters {missing}. Expected {_expected_search_params(space)}.", ) - # Log SuggestMetric - end_time = time.time() - latency_ms = (end_time - start_time) * 1000 - try: - with get_db_session() as session: - session.add(SuggestMetric( - study_name=req.study_name, - latency_ms=latency_ms, - source=source - )) - session.commit() - except Exception as metric_err: - print(f"Error logging suggest metric: {metric_err}") - config = hpo_config return { "success": True, @@ -233,29 +247,3 @@ def handle_api_suggest_trial(req: SuggestRequest): raise HTTPException(status_code=500, detail=str(e)) -def _enqueue_manual_trial(study, manual: Dict[str, Any], space: Dict[str, Any], summary: str = "AI Coordinator suggested manual trial.") -> str: - """Enqueue one coordinator-proposed trial; TPE still drives every other suggest.""" - config = load_hpo_config(study.study_name) - params = normalize_trial_params(dict(manual), config) - missing = [p for p in _expected_search_params(space) if p not in params] - if missing: - return f"Manual trial missing params {missing}; not enqueued." - try: - study.enqueue_trial(params) - waiting = [t for t in study.trials if t.state == TrialState.WAITING] - if waiting: - new_trial = max(waiting, key=lambda t: t._trial_id) - with get_db_session() as session: - session.add( - AgentReasoningLog( - trial_id=new_trial._trial_id, - study_name=study.study_name, - model_version="coordinator", - prompt_strategy="coordinator_review", - predicted_outcome_rationale=summary, - estimated_score_improvement=0.0 - ) - ) - return f"Enqueued manual trial: {params}." - except Exception as e: - return f"Could not enqueue manual trial: {e}" diff --git a/src/tunneling.py b/src/tunneling.py index 341fcae..e83566c 100644 --- a/src/tunneling.py +++ b/src/tunneling.py @@ -1,5 +1,4 @@ import os -import sys import datetime import sqlite3 import glob @@ -82,7 +81,7 @@ def _start_ngrok(port: int, secret_token: Optional[str]) -> Optional[str]: """Spawn ngrok, wait for its agent API to report the public URL, return it.""" try: print(f"Spawning ngrok tunnel for port {port}...") - proc = subprocess.Popen( + subprocess.Popen( ["ngrok", "http", str(port)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, diff --git a/src/validators.py b/src/validators.py deleted file mode 100644 index c138eae..0000000 --- a/src/validators.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Study-specific parameter validators. - -Generic studies use search-space-only bounds validation (in hpo_coordinator.py). -Studies with domain-specific constraints register validators here. -""" -from typing import Any, Callable, Dict, Optional -from pydantic import BaseModel, Field, field_validator - - -# --- Registry --- - -_MANUAL_VALIDATORS: Dict[str, Callable] = {} - - -def register_manual_validator(study_name: str, validator: Callable) -> None: - """Register a study-specific validator for manual trial parameters.""" - _MANUAL_VALIDATORS[study_name] = validator - - -def get_manual_validator(study_name: str) -> Optional[Callable]: - """Return a registered validator for ``study_name``, or None.""" - return _MANUAL_VALIDATORS.get(study_name) - - -# --- U-Net bridge-crack validator --- - -class UNetHyperparameters(BaseModel): - learning_rate: float = Field(..., ge=1e-6, le=1e-1) - batch_size: int = Field(..., ge=2, le=128) - resolution: int = Field(..., ge=128, le=2048) - model_capacity: str = Field(..., pattern="^(narrow|wide)$") - loss_weight_ratio: float = Field(..., ge=0.0, le=1.0) - - @field_validator("resolution") - @classmethod - def validate_resolution(cls, v: int) -> int: - if v % 32 != 0: - raise ValueError("Resolution must be a multiple of 32 for U-Net downsampling compatibility.") - return v - - @field_validator("batch_size") - @classmethod - def validate_batch_size(cls, v: int) -> int: - if v not in [2, 4, 8, 16, 32, 64, 128]: - raise ValueError("Batch size must be a power of 2 (e.g. 2, 4, 8, 16, 32, 64, 128).") - return v - - -LEGACY_UNET_PARAMS = { - "learning_rate", - "batch_size", - "resolution", - "model_capacity", - "loss_weight_ratio", -} - - -def validate_unet_params(params: Dict[str, Any]) -> Dict[str, Any]: - """Validate a parameter dict against UNetHyperparameters constraints.""" - valid = UNetHyperparameters(**params) - return {"ok": True, "params": valid.model_dump(), "error": None, "warnings": []} diff --git a/studies/unet_crack_segmentation_test_model_card.md b/studies/unet_crack_segmentation_test_model_card.md index 0010ba9..f8d837c 100644 --- a/studies/unet_crack_segmentation_test_model_card.md +++ b/studies/unet_crack_segmentation_test_model_card.md @@ -5,29 +5,29 @@ This model card synthesizes results for study `unet_crack_segmentation_test`. - **Best Achieved Score (Dice):** 0.5000 - **Optimal Hyperparameters:** - - `learning_rate`: 1.9171154265835047e-05 - - `batch_size`: 16 - - `resolution`: 512 - - `model_capacity`: wide - - `loss_weight_ratio`: 0.2773533339795067 + - `learning_rate`: 1.66144714685308e-05 + - `batch_size`: 32 + - `resolution`: 1024 + - `model_capacity`: narrow + - `loss_weight_ratio`: 0.5534297134135798 ## Search Space Performance - **Total Trials Evaluated:** 8 -- **Successful Runs:** 8 +- **Successful Runs:** 7 - **Pruned Runs:** 0 -- **Failed/OOM Runs:** 0 +- **Failed/OOM Runs:** 1 ### Key Parameter Importances (fANOVA) -- `batch_size`: 0.3152 -- `loss_weight_ratio`: 0.2959 -- `model_capacity`: 0.2055 -- `learning_rate`: 0.1316 -- `resolution`: 0.0519 +- `learning_rate`: 0.4944 +- `batch_size`: 0.2712 +- `resolution`: 0.1175 +- `loss_weight_ratio`: 0.0701 +- `model_capacity`: 0.0468 ## Telemetry Profile - **GPU Device:** Unknown - **Peak VRAM Recorded:** 0.00 GB -- **OOM Failures:** 0 +- **OOM Failures:** 1 --- -*Generated by Pathfinder on 2026-06-11T02:55:19.556028* +*Generated by Pathfinder on 2026-06-19T13:48:12.043832* diff --git a/templates/worker_minimal.py b/templates/worker_minimal.py index 789169b..82f2a98 100644 --- a/templates/worker_minimal.py +++ b/templates/worker_minimal.py @@ -1,15 +1,13 @@ """Minimal HPO worker template. -Copy this next to your training code, fill in `train_one_epoch`, and run it on your GPU box -(Colab, a server, anywhere). It talks to the broker only through `hpo_client` -- no need to -fork the 600-line bridge-crack `colab_worker.py`. +Copy this next to your training code, fill in ``train_one_epoch``, and run it on your GPU box +(Colab, a server, anywhere). It talks to the broker only through ``hpo_client``. Setup: - export HPO_BROKER_URL="https://hpo.mycustomdomain.com" # Or your Cloudflare/ngrok/Tailscale URL + export HPO_BROKER_URL="https://hpo.mycustomdomain.com" export HPO_STUDY_NAME="my_study" python worker_minimal.py """ -import sys from src.hpo_client import TrialSession NUM_EPOCHS = 15 @@ -19,7 +17,7 @@ def train_one_epoch(params: dict, epoch: int) -> tuple[float, float]: """Run one epoch with the given hyperparameters and return (score, loss). Replace this body with your real training/validation step. `params` contains the - hyperparameters the broker suggested (keys match your active_search_space.json). + hyperparameters the broker suggested (keys match the search space defined in your manifest). NOTE: The return value should be a tuple of (higher_is_better_score, lower_is_better_loss). The parameter names 'score' and 'loss' inside report_epoch are generalized: @@ -74,7 +72,8 @@ def main(): last_epoch, score=score, loss=loss, state="FAIL", gpu_model=gpu_model, max_vram_gb=max_vram_gb, oom_triggered=True ) - sys.exit(1) + print("Trial failed due to GPU OOM. Continuing to next trial.") + return else: # Re-raise standard training exceptions raise exc diff --git a/tests/conftest.py b/tests/conftest.py index 10d69b6..3ecbf8e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -60,7 +60,7 @@ def unique_study_name(): @pytest.fixture def initialized_study(unique_study_name): """Initialize a study (Optuna + config in DB) and return its name.""" - from hpo_mcp_server import initialize_study + from src.onboarding import initialize_study active_search_space = { "learning_rate": {"min": 1e-5, "max": 1e-2, "type": "float_log"}, diff --git a/tests/test_coordinator_packet.py b/tests/test_coordinator_packet.py deleted file mode 100644 index 72af6f1..0000000 --- a/tests/test_coordinator_packet.py +++ /dev/null @@ -1,98 +0,0 @@ -import os -import sys - -# Ensure project root is in sys.path and HPO_DATABASE_URL is set before any src imports -_project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -if _project_root not in sys.path: - sys.path.insert(0, _project_root) - -if "HPO_DATABASE_URL" not in os.environ: - import tempfile - _test_db_fd, TEST_DB_PATH = tempfile.mkstemp(suffix=".db", prefix="hpo_test_suite_") - os.close(_test_db_fd) - os.environ["HPO_DATABASE_URL"] = f"sqlite:///{TEST_DB_PATH}" - import atexit - def _cleanup(): - for suffix in ("", "-shm", "-wal"): - try: - os.unlink(TEST_DB_PATH + suffix) - except OSError: - pass - atexit.register(_cleanup) - -import unittest -import optuna -from optuna.trial import TrialState - -from src.db_manager import init_db, get_db_session -from src.schema import TrialResult, StudyReview -from src.hpo_coordinator import build_review_packet, build_review_prompt, save_study_review - - -class TestCoordinatorPacket(unittest.TestCase): - @classmethod - def setUpClass(cls): - init_db() - - def setUp(self): - self.study_name = "test_study_coord_" + self._testMethodName - self.study = optuna.create_study( - study_name=self.study_name, - storage=os.environ["HPO_DATABASE_URL"], - directions=["minimize", "maximize"], - load_if_exists=True - ) - - def test_build_packet_and_prompt(self): - """Review prompt contains best trial info and packet has required keys.""" - self.study.enqueue_trial({"learning_rate": 1e-3, "batch_size": 16}) - t = self.study.ask() - self.study.tell(t.number, [0.2, 0.8]) - - # Test review prompt references the best trial - prompt = build_review_prompt(self.study_name) - self.assertIn("Trial #0", prompt) - self.assertIn("0.8000", prompt) - - # Test build review packet has binned trials and keys - packet = build_review_packet(self.study_name) - self.assertIn("trial_bins", packet) - self.assertIn("spearman_correlations", packet) - - def test_save_study_review_confidence_low(self): - """Citing a non-existent trial results in low confidence.""" - self.study.enqueue_trial({"learning_rate": 1e-3, "batch_size": 16}) - t = self.study.ask() - self.study.tell(t.number, [0.1, 0.9]) - - res = save_study_review( - self.study_name, - "Citing wrong trial number", - health_rating=3, - policy_action="no_change", - trials_evaluated=1, - cited_best_trial=999 # non-existent trial - ) - self.assertTrue(res["success"]) - self.assertEqual(res["review"]["confidence"], "low") - - def test_save_study_review_confidence_high(self): - """Citing the correct best trial results in high confidence.""" - self.study.enqueue_trial({"learning_rate": 1e-3, "batch_size": 16}) - t = self.study.ask() - self.study.tell(t.number, [0.1, 0.9]) - - res = save_study_review( - self.study_name, - "Citing correct trial number", - health_rating=4, - policy_action="no_change", - trials_evaluated=1, - cited_best_trial=0 - ) - self.assertTrue(res["success"]) - self.assertEqual(res["review"]["confidence"], "high") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_health_tier.py b/tests/test_health_tier.py index 0012bd5..8b9072f 100644 --- a/tests/test_health_tier.py +++ b/tests/test_health_tier.py @@ -26,7 +26,7 @@ def _cleanup(): from src.db_manager import init_db, get_db_session from src.schema import TrialResult, StudyStatus -from src.hpo_coordinator import compute_health_tier +from src.health import compute_health_tier class TestHealthTier(unittest.TestCase): diff --git a/tests/test_http_api.py b/tests/test_http_api.py index c6f917b..26ceb43 100644 --- a/tests/test_http_api.py +++ b/tests/test_http_api.py @@ -46,7 +46,7 @@ def test_worker_lifecycle_records_result(client, initialized_study): "score": 0.7, "loss": 0.3, "weights_path": "model.pt", - "history": [{"epoch": 2, "score": 0.7, "loss": 0.3, "dice": 0.7, "bce": 0.3}], + "history": [{"epoch": 2, "score": 0.7, "loss": 0.3}], "state": "COMPLETE", }, ) @@ -62,7 +62,7 @@ def test_worker_lifecycle_records_result(client, initialized_study): def test_delete_study_removes_optuna_and_metadata(client, initialized_study): - from hpo_mcp_server import delete_study + from src.onboarding import delete_study_internal as delete_study # Produce a trial + a TrialResult row. worker_id = str(uuid.uuid4()) @@ -70,7 +70,7 @@ def test_delete_study_removes_optuna_and_metadata(client, initialized_study): client.post("/api/complete_trial", json={ "study_name": initialized_study, "trial_id": sug["trial_id"], "worker_id": worker_id, "epoch": 1, "score": 0.6, "loss": 0.4, "weights_path": "m.pt", - "history": [{"epoch": 1, "score": 0.6, "loss": 0.4, "dice": 0.6, "bce": 0.4}], "state": "COMPLETE", + "history": [{"epoch": 1, "score": 0.6, "loss": 0.4}], "state": "COMPLETE", }) res = delete_study(initialized_study, confirm=False) @@ -124,7 +124,7 @@ def test_complete_is_idempotent(client, initialized_study): "score": 0.6, "loss": 0.4, "weights_path": "model.pt", - "history": [{"epoch": 1, "score": 0.6, "loss": 0.4, "dice": 0.6, "bce": 0.4}], + "history": [{"epoch": 1, "score": 0.6, "loss": 0.4}], "state": "COMPLETE", } first = client.post("/api/complete_trial", json=payload) diff --git a/tests/test_http_auth.py b/tests/test_http_auth.py index b835a68..1cbcf48 100644 --- a/tests/test_http_auth.py +++ b/tests/test_http_auth.py @@ -46,7 +46,7 @@ def test_complete_requires_lease_for_inflight_trial(client, initialized_study): "/api/complete_trial", json={"study_name": initialized_study, "trial_id": trial_id, "worker_id": intruder, "epoch": 1, "score": 0.6, "loss": 0.4, "weights_path": "m.pt", - "history": [{"epoch": 1, "score": 0.6, "loss": 0.4, "dice": 0.6, "bce": 0.4}], "state": "COMPLETE"}, + "history": [{"epoch": 1, "score": 0.6, "loss": 0.4}], "state": "COMPLETE"}, ) assert bad.status_code == 403 diff --git a/tests/test_integration.py b/tests/test_integration.py index 8b75a4b..68f72e8 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1,366 +1,129 @@ +"""End-to-end integration test: broker startup, worker lifecycle, study packet, auth. + +The broker is started in a background thread for the duration of this test only. +Uses the same SQLite database that conftest.py configures (a temp file). +""" import os -import json import sys +import json import threading import time import requests import uvicorn -import subprocess -# 1. Clean old test database if it exists to start fresh (before database engine initialization) -if __name__ == "__main__": - db_file = "test_hpo_studies.db" - if os.path.exists(db_file): - print(f"Removing existing test database: {db_file}") - try: - os.remove(db_file) - except OSError as e: - print(f"Warning: Could not remove db file: {e}") - # Override database URL to point to a test SQLite database before imports - os.environ["HPO_DATABASE_URL"] = f"sqlite:///{db_file}" - - # Make sure workspace is in python path - sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) +BROKER_PORT = 8123 +BROKER_URL = f"http://127.0.0.1:{BROKER_PORT}" + +MANIFEST_BASE = { + "study_name": "unet_crack_segmentation_test", + "metrics": { + "primary_score": "score", + "objectives": [ + {"name": "loss", "direction": "minimize", "label": "Loss"}, + {"name": "score", "direction": "maximize", "label": "Score"}, + ], + }, + "params": [ + {"name": "learning_rate", "type": "float_log", "min": 1e-5, "max": 1e-2}, + {"name": "batch_size", "type": "categorical", "options": [2, 4, 8, 16]}, + {"name": "resolution", "type": "categorical", "options": [256, 512, 1024]}, + {"name": "model_capacity", "type": "categorical", "options": ["narrow", "wide"]}, + {"name": "loss_weight_ratio", "type": "float", "min": 0.0, "max": 1.0}, + ], +} + + +def _start_broker(): + from broker import app + from src.db_manager import init_db + init_db() + uvicorn.run(app, host="127.0.0.1", port=BROKER_PORT, log_level="warning") - # Start broker in background thread - broker_port = 8123 - os.environ["HPO_BROKER_URL"] = f"http://127.0.0.1:{broker_port}" - def start_broker(): - from broker import app - uvicorn.run(app, host="127.0.0.1", port=broker_port, log_level="warning") +def test_integration(): + os.environ["HPO_BROKER_URL"] = BROKER_URL - print("Starting HTTP broker in background thread...") - broker_thread = threading.Thread(target=start_broker, daemon=True) + broker_thread = threading.Thread(target=_start_broker, daemon=True) broker_thread.start() - time.sleep(2.0) # Wait for uvicorn to bind and start + time.sleep(2.0) - # Import dependencies after environment setup - from src.db_manager import init_db, get_db_session - from src.schema import TrialResult, SystemConfiguration, StudyStatus, StudyCard - from hpo_mcp_server import ( - initialize_study, - get_study_data, - validate_search_space, - update_search_space, - generate_model_card, - submit_agent_review - ) - from simulators.training_worker import run_training_worker + study_name = MANIFEST_BASE["study_name"] + # ---- Step 1: Init study from manifest ---- + from src.onboarding import init_study_from_manifest_dict + manifest = json.loads(json.dumps(MANIFEST_BASE)) + init_study_from_manifest_dict(manifest, force=True) -def run_integration_test(): - print("==================================================") - print("STARTING PATHFINDER INTEGRATION TEST (DB-BACKED)") - print("==================================================\n") - - # Reinitialize DB tables - print("Initializing SQLite database tables...") - init_db() - - study_name = "unet_crack_segmentation_test" - active_search_space = { - "learning_rate": {"min": 1e-5, "max": 1e-2, "type": "float_log"}, - "batch_size": {"options": [2, 4, 8, 16, 32, 64], "active": [2, 4, 8, 16, 32, 64], "type": "categorical"}, - "resolution": {"options": [256, 512, 1024], "active": [256, 512, 1024], "type": "categorical"}, - "model_capacity": {"options": ["narrow", "wide"], "active": ["narrow", "wide"], "type": "categorical"}, - "loss_weight_ratio": {"min": 0.0, "max": 1.0, "type": "float"} - } - hpo_config = { - "eval_protocol": { - "enabled": True, - "fixed_resolution": 512, - "train_resolution_param": "resolution", - "fixed_dice_attr": "dice_eval_fixed", - "fixed_bce_attr": "bce_eval_fixed" - }, - "metric_score_label": "Dice", - "metric_loss_label": "BCE" - } - project_context = { - "hypothesis": "Testing U-Net segmentation models on crack images.", - "gpu_model": "NVIDIA L4", - "gpu_capacity_gb": 24.0 - } + from src.db_manager import get_db_session + from src.schema import SystemConfiguration, StudyStatus - # 1. Test Study Initialization - print("\n--- [Step 1: Initializing Study in Database] ---") - init_msg = initialize_study( - study_name=study_name, - active_search_space=active_search_space, - hpo_config=hpo_config, - project_context=project_context, - multi_objective=True - ) - print(init_msg) - - # Verify records were inserted in SystemConfiguration with get_db_session() as session: - space_row = session.query(SystemConfiguration).filter_by( + row = session.query(SystemConfiguration).filter_by( study_name=study_name, config_key="active_search_space" ).first() - assert space_row is not None, "Active search space should be stored in system_configuration!" - - status_row = session.query(StudyStatus).filter_by(study_name=study_name).first() - assert status_row is not None, "Study status should be initialized!" - assert status_row.health_tier == "healthy", "Initial health tier should be healthy!" + assert row is not None, "Search space should be persisted" + status = session.query(StudyStatus).filter_by(study_name=study_name).first() + assert status is not None, "StudyStatus should exist" - # 2. Test Search Space Pre-flight Validation - print("\n--- [Step 2: Verifying Search Space Validation] ---") - # A. Propose a valid space config - valid_val = validate_search_space(active_search_space) - print(f"Valid validation response: {valid_val}") - assert valid_val["valid"], "Search space configuration should be valid!" - - # B. Propose an invalid space config (min >= max) - invalid_space = { - "learning_rate": {"min": 1e-2, "max": 1e-5, "type": "float_log"} - } - invalid_val = validate_search_space(invalid_space) - print(f"Invalid validation response: {invalid_val}") - assert not invalid_val["valid"], "Search space validation should fail for min >= max!" - assert len(invalid_val["errors"]) > 0, "Errors should be reported!" - - # 2.5 Run a quick mock trial to satisfy trials_evaluated > 0 rule - print("\n--- [Step 2.5: Running a quick mock trial to satisfy trials_evaluated > 0] ---") + # ---- Step 2: Run a single trial via HTTP ---- from src.hpo_client import TrialSession - session = TrialSession(broker_url=f"http://127.0.0.1:{broker_port}", study_name=study_name) - trial_data = session.suggest() - session.complete(epoch=0, score=0.5, loss=0.5) - # 3. Test Manual Parameter Suggestion & Guardrails - print("\n--- [Step 3: Suggesting Next Trial with Manual Parameters] ---") - # A. Propose invalid resolution (not multiple of 32) - invalid_params_1 = { - "learning_rate": 1e-3, - "batch_size": 16, - "resolution": 500, # Invalid - "model_capacity": "narrow", - "loss_weight_ratio": 0.5 - } - print(f"Proposing invalid parameters (resolution 500): {invalid_params_1}") - res_1 = submit_agent_review( - study_name=study_name, - summary="Testing invalid resolution boundary", - health_rating=3, - policy_action="enqueue_one_manual_trial", - model_version="coordinator", - prompt_strategy="test_strategy", - estimated_score_improvement=-1.0, - cited_best_trial=0, - manual_trial=invalid_params_1, - force=True - ) - print(f"Response: {res_1}\n") - assert not res_1["success"], "Should have failed due to resolution constraints!" + sess = TrialSession(broker_url=BROKER_URL, study_name=study_name) + trial = sess.suggest() + assert "params" in trial + sess.complete(epoch=0, score=0.5, loss=0.5) - # B. Propose valid manual parameters - valid_manual = { - "learning_rate": 1e-4, - "batch_size": 8, - "resolution": 256, - "model_capacity": "narrow", - "loss_weight_ratio": 0.3 - } - print(f"Proposing valid manual parameters: {valid_manual}") - res_valid = submit_agent_review( - study_name=study_name, - summary="Starting with a reasonable base configuration", - health_rating=4, - policy_action="enqueue_one_manual_trial", - model_version="coordinator", - prompt_strategy="test_strategy", - estimated_score_improvement=0.05, - cited_best_trial=0, - manual_trial=valid_manual, - force=True - ) - print(f"Response: {res_valid}") - assert res_valid["success"], f"Should have successfully enqueued: {res_valid.get('error')}" + # ---- Step 3: Simulator worker (5 trials) ---- + from simulators.training_worker import run_training_worker - # 4. Simulate Training Worker trials - print("\n--- [Step 4: Running Decentralized Training Worker Simulation via HTTP] ---") run_training_worker( study_name=study_name, - agent_model="gemini-3.5-flash", - prompt_strategy="tpe_guided_v1", - max_trials=7, + max_trials=5, epochs_per_trial=5, - broker_url=f"http://127.0.0.1:{broker_port}" + broker_url=BROKER_URL, ) - # 5. Fetch Study Data Compacted Packet - print("\n--- [Step 5: Fetching Compacted Review Packet] ---") - packet = get_study_data(study_name=study_name) - print(f"Compacted Packet structure keys: {list(packet.keys())}") - assert "trial_bins" in packet, "Packet must contain binned trials." - assert "fanova_importances" in packet, "Packet must contain parameter importances." - assert "spearman_correlations" in packet, "Packet must contain Spearman correlations." - assert "vram_telemetry" in packet, "Packet must contain VRAM telemetry." - - print(f"Elite Trials count: {len(packet['trial_bins']['elite'])}") - print(f"Noise floor trials summary: {packet['trial_bins']['noise_floor']['count']} trials, median score={packet['trial_bins']['noise_floor']['median_score']:.4f}") - print(f"Failure combinations matrix: {packet['trial_bins']['failure_matrix']}") - print(f"fANOVA Importances: {packet['fanova_importances']}") - print(f"VRAM Telemetry details: GPU={packet['vram_telemetry']['gpu_model']}, OOM count={packet['vram_telemetry']['oom_count']}") + # ---- Step 4: Study packet ---- + from src.analytics import build_study_packet - # 6. Test Proposing and Applying Search Space Updates - print("\n--- [Step 6: Proposing and Applying Search Space Updates] ---") - proposal = { - "learning_rate": {"min": 1e-4, "max": 1e-3} - } - print(f"Proposing search space update: {proposal}") - prop_msg = update_search_space(study_name=study_name, space_config=proposal, apply=False) - print(prop_msg) - - # Check pending changes row - with get_db_session() as session: - row = session.query(SystemConfiguration).filter_by( - study_name=study_name, config_key="pending_search_space" - ).first() - assert row is not None, "Pending changes should be written to SQLite!" - - print("Applying pending search space changes...") - apply_msg = update_search_space(study_name=study_name, space_config=proposal, apply=True) - print(apply_msg) - - # Verify change is applied and pending is cleared - with get_db_session() as session: - pending_row = session.query(SystemConfiguration).filter_by( - study_name=study_name, config_key="pending_search_space" - ).first() - assert pending_row is None, "Pending changes should be deleted after apply!" - - space_row = session.query(SystemConfiguration).filter_by( - study_name=study_name, config_key="active_search_space" - ).first() - current_space = json.loads(space_row.config_value) - assert current_space["learning_rate"]["min"] == 1e-4, "Min learning rate should be updated to 1e-4!" - assert current_space["learning_rate"]["max"] == 1e-3, "Max learning rate should be updated to 1e-3!" + packet = build_study_packet(study_name) + assert "trial_bins" in packet + assert "fanova_importances" in packet + assert "vram_telemetry" in packet + assert "health" in packet - # 7. Test Synthesis / Generating Model Card - print("\n--- [Step 7: Generating End-of-Study Model Card] ---") - card_res = generate_model_card(study_name=study_name) - print(card_res) - assert card_res["success"], f"Failed to generate model card: {card_res.get('error')}" - - # Verify card indexed in DB and file exists - assert os.path.exists(card_res["file_path"]), "Model card file should be written to disk!" - with get_db_session() as session: - card_row = session.query(StudyCard).filter_by(study_name=study_name).first() - assert card_row is not None, "Model card should be indexed in SQLite database!" - print(f"Indexed card metadata: {json.loads(card_row.metadata_json)}") + # ---- Step 5: Authentication ---- + os.environ["HPO_SECRET_TOKEN"] = "test_integration_token" - # 8. Test Retrieving Study Cards (Querying Model Card) - print("\n--- [Step 8: Querying Indexed Study Cards] ---") - from hpo_mcp_server import get_study_cards - cards = get_study_cards(study_name=study_name) - print(f"Retrieved {len(cards)} card(s) from database.") - assert len(cards) > 0, "Should have retrieved at least one study card!" - assert cards[0]["markdown_content"].startswith("# Study Model Card:"), "Markdown content should contain generated model card!" - - # Test HTTP endpoint for study cards - resp = requests.get(f"http://127.0.0.1:{broker_port}/api/study_cards?study_name={study_name}") - resp_data = resp.json() - assert resp_data["success"], "HTTP api/study_cards request should be successful!" - assert len(resp_data["cards"]) > 0, "HTTP response should contain study cards!" - print("Study cards query tests passed successfully!") + resp = requests.get(f"{BROKER_URL}/api/study_details?study_name={study_name}") + assert resp.status_code == 401 - # 9. Test Nudge Dismissal - print("\n--- [Step 9: Testing Nudge Dismissal Persistence] ---") - from src.hpo_config import load_hpo_config - from src.suggest import get_or_create_study - from src.hpo_coordinator import study_eval_insights, compute_review_heuristics - hpo_config = load_hpo_config(study_name) - study = get_or_create_study(study_name) - insights = study_eval_insights(study, hpo_config) - heuristics = compute_review_heuristics(study, insights, hpo_config, study_name) - - # Dismiss nudge via HTTP API - resp = requests.post(f"http://127.0.0.1:{broker_port}/api/dismiss_coordinator_nudge?study_name={study_name}") - assert resp.status_code == 200, "Dismiss nudge endpoint should return 200" - assert resp.json()["success"], "Dismiss nudge request should succeed" - - # Re-evaluate heuristics and verify dismissal is respected - heuristics_after = compute_review_heuristics(study, insights, hpo_config, study_name) - assert heuristics_after["already_dismissed"] == True, "already_dismissed should be True after dismissal!" - assert heuristics_after["review_recommended"] == False, "review_recommended should be False after dismissal!" - print("Nudge dismissal persistence tests passed successfully!") + resp = requests.get( + f"{BROKER_URL}/api/study_details?study_name={study_name}", + headers={"X-HPO-Token": "test_integration_token"}, + ) + assert resp.status_code == 200 - # 10. Test HPO_SECRET_TOKEN Authentication - print("\n--- [Step 10: Testing HPO_SECRET_TOKEN Authentication] ---") - os.environ["HPO_SECRET_TOKEN"] = "test_integration_token_123" - - resp_no_token = requests.get(f"http://127.0.0.1:{broker_port}/api/study_details?study_name={study_name}") - assert resp_no_token.status_code == 401, "Request without token should fail with 401!" - - resp_bad_token = requests.get(f"http://127.0.0.1:{broker_port}/api/study_details?study_name={study_name}", headers={"X-HPO-Token": "bad_token"}) - assert resp_bad_token.status_code == 401, "Request with bad token should fail with 401!" - - resp_good_token = requests.get(f"http://127.0.0.1:{broker_port}/api/study_details?study_name={study_name}", headers={"X-HPO-Token": "test_integration_token_123"}) - assert resp_good_token.status_code == 200, "Request with correct header token should succeed!" - - resp_auth_token = requests.get( - f"http://127.0.0.1:{broker_port}/api/study_details?study_name={study_name}", - headers={"Authorization": "Bearer test_integration_token_123"} + resp = requests.get( + f"{BROKER_URL}/api/study_details?study_name={study_name}", + headers={"Authorization": "Bearer test_integration_token"}, ) - assert resp_auth_token.status_code == 200, "Request with correct Authorization token should succeed!" + assert resp.status_code == 200 del os.environ["HPO_SECRET_TOKEN"] - print("HPO_SECRET_TOKEN middleware authentication tests passed successfully!") - # 11. Test CLI Commands - print("\n--- [Step 11: Testing CLI Commands] ---") - - # Test 'python hpo_cli.py status' - cmd_status = subprocess.run( + # ---- Step 6: CLI status ---- + import subprocess + result = subprocess.run( [sys.executable, "hpo_cli.py", "status", "--study", study_name], - capture_output=True, - text=True + capture_output=True, text=True, ) - assert cmd_status.returncode == 0, "hpo_cli.py status command failed!" - assert "STUDY STATUS" in cmd_status.stdout, "CLI status output should contain study status header!" - - # Stage proposed changes manually - with get_db_session() as session: - session.merge(SystemConfiguration( - study_name=study_name, - config_key="pending_search_space", - config_value=json.dumps({"learning_rate": {"min": 5e-5, "max": 5e-4}}) - )) - session.commit() - - cmd_status_pending = subprocess.run( - [sys.executable, "hpo_cli.py", "status", "--study", study_name], - capture_output=True, - text=True - ) - assert "Pending Changes: YES" in cmd_status_pending.stdout, "CLI status should report pending changes!" - - cmd_apply = subprocess.run( - [sys.executable, "hpo_cli.py", "apply", "--study", study_name], - capture_output=True, - text=True - ) - assert cmd_apply.returncode == 0, "hpo_cli.py apply command failed!" - assert "Pending search space changes committed successfully." in cmd_apply.stdout, "CLI apply message missing!" - - with get_db_session() as session: - space_row = session.query(SystemConfiguration).filter_by( - study_name=study_name, config_key="active_search_space" - ).first() - current_space = json.loads(space_row.config_value) - assert current_space["learning_rate"]["min"] == 5e-5, "CLI apply did not update active search space min learning rate!" - assert current_space["learning_rate"]["max"] == 5e-4, "CLI apply did not update active search space max learning rate!" - - print("CLI commands integration tests passed successfully!") - - print("\n==================================================") - print("INTEGRATION TEST COMPLETED SUCCESSFULLY!") - print("==================================================") - + assert result.returncode == 0 + assert "STUDY STATUS" in result.stdout -if __name__ == "__main__": - run_integration_test() + # ---- Step 7: Study cards endpoint ---- + resp = requests.get(f"{BROKER_URL}/api/study_cards?study_name={study_name}") + data = resp.json() + assert data["success"] diff --git a/tests/test_lean_roadmap.py b/tests/test_lean_roadmap.py deleted file mode 100644 index 55c748c..0000000 --- a/tests/test_lean_roadmap.py +++ /dev/null @@ -1,124 +0,0 @@ -import os -import sys -import datetime -import unittest - -_project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -if _project_root not in sys.path: - sys.path.insert(0, _project_root) - -if "HPO_DATABASE_URL" not in os.environ: - import tempfile - _test_db_fd, TEST_DB_PATH = tempfile.mkstemp(suffix=".db", prefix="hpo_lean_") - os.close(_test_db_fd) - os.environ["HPO_DATABASE_URL"] = f"sqlite:///{TEST_DB_PATH}" - import atexit - - def _cleanup(): - for suffix in ("", "-shm", "-wal"): - try: - os.unlink(TEST_DB_PATH + suffix) - except OSError: - pass - - atexit.register(_cleanup) - -import optuna -from optuna.trial import TrialState - -from src.db_manager import init_db, get_db_session -from src.schema import StudyReview -from src.hpo_coordinator import ( - compute_statistical_confidence, - compute_coordinator_accuracy, - backfill_review_outcomes, - mark_review_applied, - validate_review_fields, - save_study_review, - build_review_packet, -) -from hpo_mcp_server import validate_search_space - - -class TestLeanRoadmap(unittest.TestCase): - @classmethod - def setUpClass(cls): - init_db() - - def setUp(self): - self.study_name = "test_lean_" + self._testMethodName - self.study = optuna.create_study( - study_name=self.study_name, - storage=os.environ["HPO_DATABASE_URL"], - directions=["minimize", "maximize"], - load_if_exists=True, - ) - - def _complete_trial(self, loss: float, score: float): - self.study.enqueue_trial({"learning_rate": 1e-3, "batch_size": 8}) - t = self.study.ask() - self.study.tell(t.number, [loss, score]) - - def test_statistical_confidence_tiers(self): - self.assertEqual(compute_statistical_confidence(5), "low") - self.assertEqual(compute_statistical_confidence(15), "medium") - self.assertEqual(compute_statistical_confidence(25), "high") - packet = build_review_packet(self.study_name) - self.assertIn("statistical_confidence", packet) - - def test_validate_review_fields_contract(self): - bad = validate_review_fields(None, None) - self.assertFalse(bad["ok"]) - good = validate_review_fields(0.03, 0) - self.assertTrue(good["ok"]) - - def test_coordinator_accuracy_insufficient_data(self): - acc = compute_coordinator_accuracy(self.study_name) - self.assertTrue(acc["insufficient_data"]) - self.assertEqual(acc["n_scored_reviews"], 0) - - def test_backfill_measured_outcome(self): - for i in range(6): - self._complete_trial(0.5 - i * 0.01, 0.5 + i * 0.02) - - save_study_review( - self.study_name, - "Narrow LR for gain", - health_rating=4, - policy_action="update_active_search_space", - trials_evaluated=6, - estimated_score_improvement=0.05, - cited_best_trial=5, - force=True, - ) - mark_review_applied(self.study_name) - - for i in range(5): - self._complete_trial(0.3, 0.7 + i * 0.01) - - backfill_review_outcomes(self.study_name) - with get_db_session() as session: - review = ( - session.query(StudyReview) - .filter_by(study_name=self.study_name) - .order_by(StudyReview.id.desc()) - .first() - ) - self.assertEqual(review.outcome_status, "measured") - self.assertIsNotNone(review.actual_score_improvement) - - acc = compute_coordinator_accuracy(self.study_name) - self.assertEqual(acc["n_scored_reviews"], 1) - self.assertTrue(acc["insufficient_data"]) - - def test_validate_search_space_empty_tunable(self): - pinned = { - "learning_rate": {"min": 1e-3, "max": 1e-3, "type": "float_log"}, - "batch_size": {"options": [8], "active": [8], "type": "categorical"}, - } - result = validate_search_space(pinned) - self.assertFalse(result["valid"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 69c64b0..9f991b2 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -21,12 +21,12 @@ def base_manifest_data(): return { "study_name": "segmentation_hpo_test", "metrics": { - "primary_score": "dice", + "primary_score": "score", "objectives": [ { - "name": "dice", + "name": "score", "direction": "maximize", - "label": "Dice Score" + "label": "Score" }, { "name": "loss", @@ -89,7 +89,7 @@ def test_validate_manifest_errors(base_manifest_data): # Rule 2: metrics.objectives has at least one objective data = base_manifest_data.copy() - data["metrics"] = {"primary_score": "dice", "objectives": []} + data["metrics"] = {"primary_score": "score", "objectives": []} errors, _ = validate_manifest(data) assert any("metrics.objectives must contain at least one objective definition" in e for e in errors) @@ -97,7 +97,7 @@ def test_validate_manifest_errors(base_manifest_data): data = base_manifest_data.copy() data["metrics"] = { "primary_score": "accuracy", - "objectives": [{"name": "dice", "direction": "maximize", "label": "Dice"}] + "objectives": [{"name": "score", "direction": "maximize", "label": "Score"}] } errors, _ = validate_manifest(data) assert any("metrics.primary_score must reference a valid defined objective name" in e for e in errors) @@ -105,7 +105,7 @@ def test_validate_manifest_errors(base_manifest_data): # Rule 4: Every objective has name, direction, label data = base_manifest_data.copy() data["metrics"] = { - "primary_score": "dice", + "primary_score": "score", "objectives": [{"name": "", "direction": "invalid_dir", "label": ""}] } errors, _ = validate_manifest(data) @@ -114,10 +114,10 @@ def test_validate_manifest_errors(base_manifest_data): # Rule 5: Duplicate objective names data = base_manifest_data.copy() data["metrics"] = { - "primary_score": "dice", + "primary_score": "score", "objectives": [ - {"name": "dice", "direction": "maximize", "label": "Dice"}, - {"name": "dice", "direction": "minimize", "label": "Dice 2"} + {"name": "score", "direction": "maximize", "label": "Score"}, + {"name": "score", "direction": "minimize", "label": "Dice 2"} ] } errors, _ = validate_manifest(data) @@ -264,7 +264,7 @@ def test_mappings(base_manifest_data): config = _manifest_to_hpo_config(base_manifest_data) assert config["config_version"] == 2 - assert config["metric_score_label"] == "Dice Score" + assert config["metric_score_label"] == "Score" assert config["metric_loss_label"] == "BCE Loss" assert config["eval_protocol"]["enabled"] is True assert config["eval_protocol"]["fixed_resolution"] == 512 @@ -359,31 +359,29 @@ def test_cli_init_and_manifest_roundtrip(tmp_path, base_manifest_data): assert len(exported_data["params"]) == len(data["params"]) assert exported_data["metrics"]["primary_score"] == data["metrics"]["primary_score"] -def test_api_endpoints(client, base_manifest_data): +def test_api_endpoints(base_manifest_data): + from src.onboarding import init_study_from_manifest_dict + study_name = "test_api_manifest_study" - base_manifest_data["study_name"] = study_name + data = base_manifest_data.copy() + data["study_name"] = study_name - # Validate endpoint - res = client.post("/api/validate_manifest", json={"yaml": yaml.dump(base_manifest_data)}) - assert res.status_code == 200 - data = res.json() - assert data["success"] is True + # Validate + errors, warnings = validate_manifest(data) + assert len(errors) == 0 - # Init endpoint - res_init = client.post("/api/init_from_manifest?force=true", json={"yaml": yaml.dump(base_manifest_data)}) - assert res_init.status_code == 200 - data_init = res_init.json() - assert data_init["success"] is True - assert data_init["study_name"] == study_name + # Init + result = init_study_from_manifest_dict(data, force=True) + assert "successfully initialized" in result.lower() # Init duplicate error without force - res_dup = client.post("/api/init_from_manifest?force=false", json={"yaml": yaml.dump(base_manifest_data)}) - assert res_dup.status_code == 400 - assert "already exists" in res_dup.json()["detail"] + with pytest.raises(ValueError, match="already exists"): + init_study_from_manifest_dict(data, force=False) def test_manifest_metric_ordering(client, base_manifest_data): import optuna + from src.onboarding import init_study_from_manifest_dict from src.db_manager import get_db_session from src.schema import TrialResult @@ -392,26 +390,25 @@ def test_manifest_metric_ordering(client, base_manifest_data): data_1 = base_manifest_data.copy() data_1["study_name"] = study_name_1 data_1["metrics"] = { - "primary_score": "dice", + "primary_score": "score", "objectives": [ - {"name": "dice", "direction": "maximize", "label": "Dice"}, + {"name": "score", "direction": "maximize", "label": "Score"}, {"name": "loss", "direction": "minimize", "label": "Loss"} ] } - res_init_1 = client.post("/api/init_from_manifest?force=true", json={"yaml": yaml.dump(data_1)}) - assert res_init_1.status_code == 200 + init_study_from_manifest_dict(data_1, force=True) # Suggest trial res_sug_1 = client.post("/api/suggest_trial", json={"study_name": study_name_1, "worker_id": "w1"}) assert res_sug_1.status_code == 200 trial_id_1 = res_sug_1.json()["trial_id"] - # Complete trial with dice=0.95, loss=0.05 + # Complete trial with score=0.95, loss=0.05 res_comp_1 = client.post("/api/complete_trial", json={ "study_name": study_name_1, "trial_id": trial_id_1, "worker_id": "w1", "epoch": 1, "score": 0.95, "loss": 0.05, "weights_path": "m.pt", - "history": [{"epoch": 1, "score": 0.95, "loss": 0.05, "dice": 0.95, "bce": 0.05}], "state": "COMPLETE", + "history": [{"epoch": 1, "score": 0.95, "loss": 0.05}], "state": "COMPLETE", }) assert res_comp_1.status_code == 200 @@ -427,15 +424,14 @@ def test_manifest_metric_ordering(client, base_manifest_data): data_2 = base_manifest_data.copy() data_2["study_name"] = study_name_2 data_2["metrics"] = { - "primary_score": "dice", + "primary_score": "score", "objectives": [ {"name": "loss", "direction": "minimize", "label": "Loss"}, - {"name": "dice", "direction": "maximize", "label": "Dice"} + {"name": "score", "direction": "maximize", "label": "Score"} ] } - res_init_2 = client.post("/api/init_from_manifest?force=true", json={"yaml": yaml.dump(data_2)}) - assert res_init_2.status_code == 200 + init_study_from_manifest_dict(data_2, force=True) res_sug_2 = client.post("/api/suggest_trial", json={"study_name": study_name_2, "worker_id": "w2"}) assert res_sug_2.status_code == 200 @@ -444,7 +440,7 @@ def test_manifest_metric_ordering(client, base_manifest_data): res_comp_2 = client.post("/api/complete_trial", json={ "study_name": study_name_2, "trial_id": trial_id_2, "worker_id": "w2", "epoch": 1, "score": 0.95, "loss": 0.05, "weights_path": "m.pt", - "history": [{"epoch": 1, "score": 0.95, "loss": 0.05, "dice": 0.95, "bce": 0.05}], "state": "COMPLETE", + "history": [{"epoch": 1, "score": 0.95, "loss": 0.05}], "state": "COMPLETE", }) assert res_comp_2.status_code == 200 @@ -457,6 +453,8 @@ def test_manifest_metric_ordering(client, base_manifest_data): def test_single_objective_minimize(client, base_manifest_data): import optuna + from src.onboarding import init_study_from_manifest_dict + study_name = "test_single_min" data = base_manifest_data.copy() data["study_name"] = study_name @@ -467,8 +465,7 @@ def test_single_objective_minimize(client, base_manifest_data): ] } - res_init = client.post("/api/init_from_manifest?force=true", json={"yaml": yaml.dump(data)}) - assert res_init.status_code == 200 + init_study_from_manifest_dict(data, force=True) study = optuna.load_study(study_name=study_name, storage=os.environ["HPO_DATABASE_URL"]) assert len(study.directions) == 1 @@ -481,7 +478,7 @@ def test_single_objective_minimize(client, base_manifest_data): res_comp = client.post("/api/complete_trial", json={ "study_name": study_name, "trial_id": trial_id, "worker_id": "w3", "epoch": 1, "score": 0.0, "loss": 0.035, "weights_path": "m.pt", - "history": [{"epoch": 1, "score": 0.0, "loss": 0.035, "dice": 0.0, "bce": 0.035}], "state": "COMPLETE", + "history": [{"epoch": 1, "score": 0.0, "loss": 0.035}], "state": "COMPLETE", }) assert res_comp.status_code == 200 @@ -492,6 +489,7 @@ def test_single_objective_minimize(client, base_manifest_data): def test_deep_cleanup_on_force_overwrite(client, base_manifest_data): + from src.onboarding import init_study_from_manifest_dict from src.db_manager import get_db_session from src.schema import SystemConfiguration, TrialResult @@ -500,8 +498,7 @@ def test_deep_cleanup_on_force_overwrite(client, base_manifest_data): data["study_name"] = study_name # 1. Initialize first time - res_init1 = client.post("/api/init_from_manifest?force=true", json={"yaml": yaml.dump(data)}) - assert res_init1.status_code == 200 + init_study_from_manifest_dict(data, force=True) res_sug = client.post("/api/suggest_trial", json={"study_name": study_name, "worker_id": "w4"}) assert res_sug.status_code == 200 @@ -510,7 +507,7 @@ def test_deep_cleanup_on_force_overwrite(client, base_manifest_data): res_comp = client.post("/api/complete_trial", json={ "study_name": study_name, "trial_id": trial_id, "worker_id": "w4", "epoch": 1, "score": 0.8, "loss": 0.2, "weights_path": "m.pt", - "history": [{"epoch": 1, "score": 0.8, "loss": 0.2, "dice": 0.8, "bce": 0.2}], "state": "COMPLETE", + "history": [{"epoch": 1, "score": 0.8, "loss": 0.2}], "state": "COMPLETE", }) assert res_comp.status_code == 200 @@ -520,8 +517,7 @@ def test_deep_cleanup_on_force_overwrite(client, base_manifest_data): assert session.query(TrialResult).filter_by(study_name=study_name).count() > 0 # 2. Force re-initialize - res_init2 = client.post("/api/init_from_manifest?force=true", json={"yaml": yaml.dump(data)}) - assert res_init2.status_code == 200 + init_study_from_manifest_dict(data, force=True) # Check that previous TrialResult rows are completely cleaned up and only fresh config remains with get_db_session() as session: diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 95057b3..41a4f0a 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -20,7 +20,7 @@ loss_objective_index, score_objective_index, ) -from src.hpo_coordinator import get_fanova_importances, get_best_primary_score +from src.analytics import get_fanova_importances def _complete_trial(study, values, params=None): @@ -71,12 +71,6 @@ def test_get_score_from_dirs(self): self.assertEqual(get_score_from_dirs(trial, study.directions), 0.8) self.assertEqual(get_loss_from_dirs(trial, study.directions), 0.2) - def test_get_best_primary_score_minimize_fallback(self): - study = optuna.create_study(direction="minimize") - _complete_trial(study, 0.25) - _complete_trial(study, 0.10) - self.assertEqual(get_best_primary_score(study), 0.10) - def test_fanova_importances_no_score_objective(self): study = optuna.create_study( study_name="test_fanova_no_score_" + self._testMethodName, @@ -93,7 +87,7 @@ def test_fanova_importances_mocked_score_index_none(self): study = optuna.create_study(directions=["minimize", "maximize"]) _complete_trial(study, [0.5, 0.6]) _complete_trial(study, [0.4, 0.7]) - with patch("src.hpo_coordinator.score_objective_index", return_value=None): + with patch("src.analytics.score_objective_index", return_value=None): result = get_fanova_importances(study, {}) self.assertEqual(result, {}) diff --git a/tests/test_pruning.py b/tests/test_pruning.py index 01f3c14..a27fb14 100644 --- a/tests/test_pruning.py +++ b/tests/test_pruning.py @@ -54,16 +54,16 @@ def _get_frozen_trial(self, trial_number): raise ValueError(f"Trial #{trial_number} not found") def test_composite_score_thin_data(self): - """Less than 10 completed/running trials -> returns raw score (no Z-score).""" + """Less than 10 completed/running trials -> returns (score - loss) composite.""" self.study.enqueue_trial({"learning_rate": 1e-3, "batch_size": 16}) t = self.study.ask() trial_number = t.number - history = [{"epoch": 1, "score": 0.5, "loss": 0.5, "dice": 0.5, "bce": 0.5}] + history = [{"epoch": 1, "score": 0.5, "loss": 0.5}] self.study._storage.set_trial_user_attr(t._trial_id, "history", history) - + frozen = self._get_frozen_trial(trial_number) score = _epoch_composite_score(self.study, frozen, 1, {"enabled": False}) - self.assertEqual(score, 0.5) + self.assertEqual(score, 0.0) # 0.5 - 0.5 = 0.0 def test_composite_score_zscore_and_zero_variance_clamp(self): """11 trials with identical scores (zero variance) -> Z-score with epsilon clamp returns 0.0.""" @@ -71,7 +71,7 @@ def test_composite_score_zscore_and_zero_variance_clamp(self): for i in range(11): self.study.enqueue_trial({"learning_rate": 1e-3, "batch_size": 16}) t = self.study.ask() - self.study._storage.set_trial_user_attr(t._trial_id, "history", [{"epoch": 1, "score": 0.8, "loss": 0.2, "dice": 0.8, "bce": 0.2}]) + self.study._storage.set_trial_user_attr(t._trial_id, "history", [{"epoch": 1, "score": 0.8, "loss": 0.2}]) if i < 10: self.study.tell(t.number, [0.2, 0.8]) else: @@ -84,17 +84,17 @@ def test_composite_score_zscore_and_zero_variance_clamp(self): def test_composite_score_zscore_normal_variance(self): """11 trials with varying scores -> Z-score normalization produces meaningful non-zero result.""" - dice_scores = [0.3, 0.4, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85] - bce_losses = [0.7, 0.6, 0.5, 0.45, 0.4, 0.35, 0.3, 0.25, 0.2, 0.15] + score_vals = [0.3, 0.4, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85] + loss_vals = [0.7, 0.6, 0.5, 0.45, 0.4, 0.35, 0.3, 0.25, 0.2, 0.15] for i in range(10): self.study.enqueue_trial({"learning_rate": 1e-3, "batch_size": 16}) t = self.study.ask() self.study._storage.set_trial_user_attr( t._trial_id, "history", - [{"epoch": 1, "score": dice_scores[i], "loss": bce_losses[i], "dice": dice_scores[i], "bce": bce_losses[i]}] + [{"epoch": 1, "score": score_vals[i], "loss": loss_vals[i]}] ) - self.study.tell(t.number, [bce_losses[i], dice_scores[i]]) + self.study.tell(t.number, [loss_vals[i], score_vals[i]]) # 11th trial — above-average (score=0.9, loss=0.1) self.study.enqueue_trial({"learning_rate": 1e-3, "batch_size": 16}) @@ -102,7 +102,7 @@ def test_composite_score_zscore_normal_variance(self): outlier_number = outlier.number self.study._storage.set_trial_user_attr( outlier._trial_id, "history", - [{"epoch": 1, "score": 0.9, "loss": 0.1, "dice": 0.9, "bce": 0.1}] + [{"epoch": 1, "score": 0.9, "loss": 0.1}] ) frozen_outlier = self._get_frozen_trial(outlier_number) @@ -116,7 +116,7 @@ def test_composite_score_zscore_normal_variance(self): weak_number = weak.number self.study._storage.set_trial_user_attr( weak._trial_id, "history", - [{"epoch": 1, "score": 0.2, "loss": 0.8, "dice": 0.2, "bce": 0.8}] + [{"epoch": 1, "score": 0.2, "loss": 0.8}] ) frozen_weak = self._get_frozen_trial(weak_number) diff --git a/tests/test_robustness_features.py b/tests/test_robustness_features.py index 97f9fda..47fc2ae 100644 --- a/tests/test_robustness_features.py +++ b/tests/test_robustness_features.py @@ -8,7 +8,7 @@ from fastapi.testclient import TestClient from optuna.trial import TrialState from src.db_manager import get_db_session, DATABASE_URL -from src.schema import TrialResult, SystemConfiguration, StudyReview +from src.schema import TrialResult, SystemConfiguration import hpo_cli def test_zero_metric_rejection(client, initialized_study): @@ -100,20 +100,6 @@ def test_cli_export_import_roundtrip(client, initialized_study): }) assert resp.status_code == 200 - # Add a study review - from src.hpo_coordinator import save_study_review - save_study_review( - study_name=initialized_study, - summary="Test study review summary", - health_rating=4, - policy_action="no_change", - model_version="test_version", - reasons=[], - trials_evaluated=1, - estimated_score_improvement=0.05, - cited_best_trial=0 - ) - # 2. Export the study to a temporary file temp_dir = tempfile.mkdtemp() export_path = os.path.join(temp_dir, "export.json") @@ -134,7 +120,6 @@ class Args: assert data["study_name"] == initialized_study assert len(data["trials"]) == 1 assert len(data["trial_results"]) == 1 - assert len(data["study_reviews"]) == 1 # 3. Import the study under a new name imported_study_name = f"{initialized_study}_imported" @@ -184,7 +169,6 @@ class BackupArgs: cursor.execute("SELECT name FROM sqlite_master WHERE type='table'") tables = [t[0] for t in cursor.fetchall()] assert "trial_results" in tables - assert "study_reviews" in tables conn.close() try: @@ -267,7 +251,7 @@ def test_transient_health_warning_clears(client, initialized_study): resp = client.get(f"/api/study_details?study_name={initialized_study}") assert resp.status_code == 200 details = resp.json() - assert details["review"]["health_tier"] == "watch" + assert details["health"]["tier"] == "watch" # Complete a healthy trial resp = client.post("/api/suggest_trial", json={"study_name": initialized_study, "worker_id": "health_worker"}) @@ -305,7 +289,7 @@ def test_transient_health_warning_clears(client, initialized_study): resp = client.get(f"/api/study_details?study_name={initialized_study}") assert resp.status_code == 200 details = resp.json() - assert details["review"]["health_tier"] == "healthy" + assert details["health"]["tier"] == "healthy" def test_unbounded_metric_study_skips_validation(client, initialized_study): @@ -332,7 +316,7 @@ def test_unbounded_metric_study_skips_validation(client, initialized_study): resp = client.get(f"/api/study_details?study_name={initialized_study}") assert resp.status_code == 200 details = resp.json() - assert details["review"]["health_tier"] == "healthy" + assert details["health"]["tier"] == "healthy" def test_complete_partial_metrics(client, initialized_study): diff --git a/tests/test_vram_telemetry.py b/tests/test_vram_telemetry.py new file mode 100644 index 0000000..94e1886 --- /dev/null +++ b/tests/test_vram_telemetry.py @@ -0,0 +1,50 @@ +import pytest +from optuna.trial import TrialState +from src.analytics import build_study_packet + + +def test_vram_telemetry_flows_to_review_packet(client, initialized_study): + """ + Submits a trial with VRAM data and verifies the review packet contains + populated VRAM telemetry rather than empty defaults. + """ + study_name = initialized_study + + # Submit a trial with explicit GPU/VRAM data + resp = client.post( + "/api/suggest_trial", + json={"study_name": study_name, "worker_id": "vram_test_worker"}, + ) + assert resp.status_code == 200 + trial_id = resp.json()["trial_id"] + + payload = { + "study_name": study_name, + "trial_id": trial_id, + "worker_id": "vram_test_worker", + "epoch": 5, + "score": 0.85, + "loss": 0.14, + "weights_path": "model.pt", + "history": [{"epoch": 5, "score": 0.85, "loss": 0.14}], + "state": "COMPLETE", + "gpu_model": "NVIDIA A100", + "max_vram_gb": 40.0, + "oom_triggered": False, + } + resp = client.post("/api/complete_trial", json=payload) + assert resp.status_code == 200 + + # Fetch the review packet + packet = build_study_packet(study_name) + vram = packet.get("vram_telemetry", {}) + + assert vram.get("gpu_model") == "NVIDIA A100", ( + f"Expected GPU model 'NVIDIA A100', got {vram.get('gpu_model')}" + ) + assert vram.get("gpu_capacity_gb", 0) > 0, ( + f"Expected gpu_capacity_gb > 0, got {vram.get('gpu_capacity_gb')}" + ) + assert vram.get("oom_count", -1) >= 0, ( + f"Expected oom_count >= 0, got {vram.get('oom_count')}" + ) diff --git a/web/index.html b/web/index.html index 3b663e8..6a2c48e 100644 --- a/web/index.html +++ b/web/index.html @@ -4,7 +4,7 @@ Pathfinder - +
@@ -16,7 +16,7 @@
-
+
- + +
@@ -80,7 +90,7 @@

Dashboard

How to run a training worker:

  1. Configure your search space and evaluation protocol in the Search Space tab.
    2. -
  2. Go to the Worker Setup tab to copy the worker integration script (Custom or Colab).
    3. +
  3. Set HPO_BROKER_URL and HPO_STUDY_NAME on your training machine. See docs/INTEGRATION.md for details.
  4. Run the worker script on your GPU machine. The worker will connect to this broker, request trials, and report epoch metrics.
@@ -111,7 +121,7 @@

Show audit history
- +

@@ -123,6 +133,14 @@

Active Search Space
Loading space bounds...
+
+ Remote Worker Setup +
+

export HPO_BROKER_URL=http://your-broker:8000

+

export HPO_STUDY_NAME=your_study_name

+

See docs/INTEGRATION.md for full worker setup instructions.

+
+
@@ -155,7 +173,7 @@

- +
TrialStateScore (train)Score (eval)LossParameters...
TrialStateScore (train)Score (eval)LossParameters...
Loading study results…
@@ -182,13 +200,6 @@

-
@@ -231,42 +242,6 @@

-
-
- -
-
-
-
Remote GPU?

export HPO_SECRET_TOKEN=… then python broker.py --daemon --tunnel.

Colab must send that token as X-HPO-Token on the worker download.

-
Local only?

python broker.py --daemon — no token, no tunnel.

-
-
-
Custom Training Script Integration
-
-
- 

-                                
Customize worker_minimal.py and return (score, loss). The client wrapper handles ASHA early-stopping, CUDA OOM detection, and telemetry.

-                            

-                        

-                        
-                        
-                    

-                

-            

         
     

     
-    
+        
 
 
diff --git a/web/js/chart.min.js b/web/js/chart.min.js
new file mode 100644
index 0000000..9a07c2f
--- /dev/null
+++ b/web/js/chart.min.js
@@ -0,0 +1,20 @@
+/**
+ * Skipped minification because the original files appears to be already minified.
+ * Original file: /npm/chart.js@4.4.0/dist/chart.umd.js
+ *
+ * Do NOT use SRI with dynamically generated files! More information: https://www.jsdelivr.com/using-sri-with-dynamic-files
+ */
+/*!
+ * Chart.js v4.4.0
+ * https://www.chartjs.org
+ * (c) 2023 Chart.js Contributors
+ * Released under the MIT License
+ */
+!function(t,e){"object"==typeof exports&&"undefined"!=typeof module?module.exports=e():"function"==typeof define&&define.amd?define(e):(t="undefined"!=typeof globalThis?globalThis:t||self).Chart=e()}(this,(function(){"use strict";var t=Object.freeze({__proto__:null,get Colors(){return Go},get Decimation(){return Qo},get Filler(){return ma},get Legend(){return ya},get SubTitle(){return ka},get Title(){return Ma},get Tooltip(){return Ba}});function e(){}const i=(()=>{let t=0;return()=>t++})();function s(t){return null==t}function n(t){if(Array.isArray&&Array.isArray(t))return!0;const e=Object.prototype.toString.call(t);return"[object"===e.slice(0,7)&&"Array]"===e.slice(-6)}function o(t){return null!==t&&"[object Object]"===Object.prototype.toString.call(t)}function a(t){return("number"==typeof t||t instanceof Number)&&isFinite(+t)}function r(t,e){return a(t)?t:e}function l(t,e){return void 0===t?e:t}const h=(t,e)=>"string"==typeof t&&t.endsWith("%")?parseFloat(t)/100:+t/e,c=(t,e)=>"string"==typeof t&&t.endsWith("%")?parseFloat(t)/100*e:+t;function d(t,e,i){if(t&&"function"==typeof t.call)return t.apply(i,e)}function u(t,e,i,s){let a,r,l;if(n(t))if(r=t.length,s)for(a=r-1;a>=0;a--)e.call(i,t[a],a);else for(a=0;at,x:t=>t.x,y:t=>t.y};function v(t){const e=t.split("."),i=[];let s="";for(const t of e)s+=t,s.endsWith("\\")?s=s.slice(0,-1)+".":(i.push(s),s="");return i}function M(t,e){const i=y[e]||(y[e]=function(t){const e=v(t);return t=>{for(const i of e){if(""===i)break;t=t&&t[i]}return t}}(e));return i(t)}function w(t){return t.charAt(0).toUpperCase()+t.slice(1)}const k=t=>void 0!==t,S=t=>"function"==typeof t,P=(t,e)=>{if(t.size!==e.size)return!1;for(const i of t)if(!e.has(i))return!1;return!0};function D(t){return"mouseup"===t.type||"click"===t.type||"contextmenu"===t.type}const C=Math.PI,O=2*C,A=O+C,T=Number.POSITIVE_INFINITY,L=C/180,E=C/2,R=C/4,I=2*C/3,z=Math.log10,F=Math.sign;function V(t,e,i){return Math.abs(t-e)t-e)).pop(),e}function N(t){return!isNaN(parseFloat(t))&&isFinite(t)}function H(t,e){const i=Math.round(t);return i-e<=t&&i+e>=t}function j(t,e,i){let s,n,o;for(s=0,n=t.length;sl&&h=Math.min(e,i)-s&&t<=Math.max(e,i)+s}function et(t,e,i){i=i||(i=>t[i]1;)s=o+n>>1,i(s)?o=s:n=s;return{lo:o,hi:n}}const it=(t,e,i,s)=>et(t,i,s?s=>{const n=t[s][e];return nt[s][e]et(t,i,(s=>t[s][e]>=i));function nt(t,e,i){let s=0,n=t.length;for(;ss&&t[n-1]>i;)n--;return s>0||n{const i="_onData"+w(e),s=t[e];Object.defineProperty(t,e,{configurable:!0,enumerable:!1,value(...e){const n=s.apply(this,e);return t._chartjs.listeners.forEach((t=>{"function"==typeof t[i]&&t[i](...e)})),n}})})))}function rt(t,e){const i=t._chartjs;if(!i)return;const s=i.listeners,n=s.indexOf(e);-1!==n&&s.splice(n,1),s.length>0||(ot.forEach((e=>{delete t[e]})),delete t._chartjs)}function lt(t){const e=new Set(t);return e.size===t.length?t:Array.from(e)}const ht="undefined"==typeof window?function(t){return t()}:window.requestAnimationFrame;function ct(t,e){let i=[],s=!1;return function(...n){i=n,s||(s=!0,ht.call(window,(()=>{s=!1,t.apply(e,i)})))}}function dt(t,e){let i;return function(...s){return e?(clearTimeout(i),i=setTimeout(t,e,s)):t.apply(this,s),e}}const ut=t=>"start"===t?"left":"end"===t?"right":"center",ft=(t,e,i)=>"start"===t?e:"end"===t?i:(e+i)/2,gt=(t,e,i,s)=>t===(s?"left":"right")?i:"center"===t?(e+i)/2:e;function pt(t,e,i){const s=e.length;let n=0,o=s;if(t._sorted){const{iScale:a,_parsed:r}=t,l=a.axis,{min:h,max:c,minDefined:d,maxDefined:u}=a.getUserBounds();d&&(n=J(Math.min(it(r,l,h).lo,i?s:it(e,l,a.getPixelForValue(h)).lo),0,s-1)),o=u?J(Math.max(it(r,a.axis,c,!0).hi+1,i?0:it(e,l,a.getPixelForValue(c),!0).hi+1),n,s)-n:s-n}return{start:n,count:o}}function mt(t){const{xScale:e,yScale:i,_scaleRanges:s}=t,n={xmin:e.min,xmax:e.max,ymin:i.min,ymax:i.max};if(!s)return t._scaleRanges=n,!0;const o=s.xmin!==e.min||s.xmax!==e.max||s.ymin!==i.min||s.ymax!==i.max;return Object.assign(s,n),o}class bt{constructor(){this._request=null,this._charts=new Map,this._running=!1,this._lastDate=void 0}_notify(t,e,i,s){const n=e.listeners[s],o=e.duration;n.forEach((s=>s({chart:t,initial:e.initial,numSteps:o,currentStep:Math.min(i-e.start,o)})))}_refresh(){this._request||(this._running=!0,this._request=ht.call(window,(()=>{this._update(),this._request=null,this._running&&this._refresh()})))}_update(t=Date.now()){let e=0;this._charts.forEach(((i,s)=>{if(!i.running||!i.items.length)return;const n=i.items;let o,a=n.length-1,r=!1;for(;a>=0;--a)o=n[a],o._active?(o._total>i.duration&&(i.duration=o._total),o.tick(t),r=!0):(n[a]=n[n.length-1],n.pop());r&&(s.draw(),this._notify(s,i,t,"progress")),n.length||(i.running=!1,this._notify(s,i,t,"complete"),i.initial=!1),e+=n.length})),this._lastDate=t,0===e&&(this._running=!1)}_getAnims(t){const e=this._charts;let i=e.get(t);return i||(i={running:!1,initial:!0,items:[],listeners:{complete:[],progress:[]}},e.set(t,i)),i}listen(t,e,i){this._getAnims(t).listeners[e].push(i)}add(t,e){e&&e.length&&this._getAnims(t).items.push(...e)}has(t){return this._getAnims(t).items.length>0}start(t){const e=this._charts.get(t);e&&(e.running=!0,e.start=Date.now(),e.duration=e.items.reduce(((t,e)=>Math.max(t,e._duration)),0),this._refresh())}running(t){if(!this._running)return!1;const e=this._charts.get(t);return!!(e&&e.running&&e.items.length)}stop(t){const e=this._charts.get(t);if(!e||!e.items.length)return;const i=e.items;let s=i.length-1;for(;s>=0;--s)i[s].cancel();e.items=[],this._notify(t,e,Date.now(),"complete")}remove(t){return this._charts.delete(t)}}var xt=new bt;
+/*!
+ * @kurkle/color v0.3.2
+ * https://github.com/kurkle/color#readme
+ * (c) 2023 Jukka Kurkela
+ * Released under the MIT License
+ */function _t(t){return t+.5|0}const yt=(t,e,i)=>Math.max(Math.min(t,i),e);function vt(t){return yt(_t(2.55*t),0,255)}function Mt(t){return yt(_t(255*t),0,255)}function wt(t){return yt(_t(t/2.55)/100,0,1)}function kt(t){return yt(_t(100*t),0,100)}const St={0:0,1:1,2:2,3:3,4:4,5:5,6:6,7:7,8:8,9:9,A:10,B:11,C:12,D:13,E:14,F:15,a:10,b:11,c:12,d:13,e:14,f:15},Pt=[..."0123456789ABCDEF"],Dt=t=>Pt[15&t],Ct=t=>Pt[(240&t)>>4]+Pt[15&t],Ot=t=>(240&t)>>4==(15&t);function At(t){var e=(t=>Ot(t.r)&&Ot(t.g)&&Ot(t.b)&&Ot(t.a))(t)?Dt:Ct;return t?"#"+e(t.r)+e(t.g)+e(t.b)+((t,e)=>t<255?e(t):"")(t.a,e):void 0}const Tt=/^(hsla?|hwb|hsv)\(\s*([-+.e\d]+)(?:deg)?[\s,]+([-+.e\d]+)%[\s,]+([-+.e\d]+)%(?:[\s,]+([-+.e\d]+)(%)?)?\s*\)$/;function Lt(t,e,i){const s=e*Math.min(i,1-i),n=(e,n=(e+t/30)%12)=>i-s*Math.max(Math.min(n-3,9-n,1),-1);return[n(0),n(8),n(4)]}function Et(t,e,i){const s=(s,n=(s+t/60)%6)=>i-i*e*Math.max(Math.min(n,4-n,1),0);return[s(5),s(3),s(1)]}function Rt(t,e,i){const s=Lt(t,1,.5);let n;for(e+i>1&&(n=1/(e+i),e*=n,i*=n),n=0;n<3;n++)s[n]*=1-e-i,s[n]+=e;return s}function It(t){const e=t.r/255,i=t.g/255,s=t.b/255,n=Math.max(e,i,s),o=Math.min(e,i,s),a=(n+o)/2;let r,l,h;return n!==o&&(h=n-o,l=a>.5?h/(2-n-o):h/(n+o),r=function(t,e,i,s,n){return t===n?(e-i)/s+(e>16&255,o>>8&255,255&o]}return t}(),Ht.transparent=[0,0,0,0]);const e=Ht[t.toLowerCase()];return e&&{r:e[0],g:e[1],b:e[2],a:4===e.length?e[3]:255}}const $t=/^rgba?\(\s*([-+.\d]+)(%)?[\s,]+([-+.e\d]+)(%)?[\s,]+([-+.e\d]+)(%)?(?:[\s,/]+([-+.e\d]+)(%)?)?\s*\)$/;const Yt=t=>t<=.0031308?12.92*t:1.055*Math.pow(t,1/2.4)-.055,Ut=t=>t<=.04045?t/12.92:Math.pow((t+.055)/1.055,2.4);function Xt(t,e,i){if(t){let s=It(t);s[e]=Math.max(0,Math.min(s[e]+s[e]*i,0===e?360:1)),s=Ft(s),t.r=s[0],t.g=s[1],t.b=s[2]}}function qt(t,e){return t?Object.assign(e||{},t):t}function Kt(t){var e={r:0,g:0,b:0,a:255};return Array.isArray(t)?t.length>=3&&(e={r:t[0],g:t[1],b:t[2],a:255},t.length>3&&(e.a=Mt(t[3]))):(e=qt(t,{r:0,g:0,b:0,a:1})).a=Mt(e.a),e}function Gt(t){return"r"===t.charAt(0)?function(t){const e=$t.exec(t);let i,s,n,o=255;if(e){if(e[7]!==i){const t=+e[7];o=e[8]?vt(t):yt(255*t,0,255)}return i=+e[1],s=+e[3],n=+e[5],i=255&(e[2]?vt(i):yt(i,0,255)),s=255&(e[4]?vt(s):yt(s,0,255)),n=255&(e[6]?vt(n):yt(n,0,255)),{r:i,g:s,b:n,a:o}}}(t):Bt(t)}class Zt{constructor(t){if(t instanceof Zt)return t;const e=typeof t;let i;var s,n,o;"object"===e?i=Kt(t):"string"===e&&(o=(s=t).length,"#"===s[0]&&(4===o||5===o?n={r:255&17*St[s[1]],g:255&17*St[s[2]],b:255&17*St[s[3]],a:5===o?17*St[s[4]]:255}:7!==o&&9!==o||(n={r:St[s[1]]<<4|St[s[2]],g:St[s[3]]<<4|St[s[4]],b:St[s[5]]<<4|St[s[6]],a:9===o?St[s[7]]<<4|St[s[8]]:255})),i=n||jt(t)||Gt(t)),this._rgb=i,this._valid=!!i}get valid(){return this._valid}get rgb(){var t=qt(this._rgb);return t&&(t.a=wt(t.a)),t}set rgb(t){this._rgb=Kt(t)}rgbString(){return this._valid?(t=this._rgb)&&(t.a<255?`rgba(${t.r}, ${t.g}, ${t.b}, ${wt(t.a)})`:`rgb(${t.r}, ${t.g}, ${t.b})`):void 0;var t}hexString(){return this._valid?At(this._rgb):void 0}hslString(){return this._valid?function(t){if(!t)return;const e=It(t),i=e[0],s=kt(e[1]),n=kt(e[2]);return t.a<255?`hsla(${i}, ${s}%, ${n}%, ${wt(t.a)})`:`hsl(${i}, ${s}%, ${n}%)`}(this._rgb):void 0}mix(t,e){if(t){const i=this.rgb,s=t.rgb;let n;const o=e===n?.5:e,a=2*o-1,r=i.a-s.a,l=((a*r==-1?a:(a+r)/(1+a*r))+1)/2;n=1-l,i.r=255&l*i.r+n*s.r+.5,i.g=255&l*i.g+n*s.g+.5,i.b=255&l*i.b+n*s.b+.5,i.a=o*i.a+(1-o)*s.a,this.rgb=i}return this}interpolate(t,e){return t&&(this._rgb=function(t,e,i){const s=Ut(wt(t.r)),n=Ut(wt(t.g)),o=Ut(wt(t.b));return{r:Mt(Yt(s+i*(Ut(wt(e.r))-s))),g:Mt(Yt(n+i*(Ut(wt(e.g))-n))),b:Mt(Yt(o+i*(Ut(wt(e.b))-o))),a:t.a+i*(e.a-t.a)}}(this._rgb,t._rgb,e)),this}clone(){return new Zt(this.rgb)}alpha(t){return this._rgb.a=Mt(t),this}clearer(t){return this._rgb.a*=1-t,this}greyscale(){const t=this._rgb,e=_t(.3*t.r+.59*t.g+.11*t.b);return t.r=t.g=t.b=e,this}opaquer(t){return this._rgb.a*=1+t,this}negate(){const t=this._rgb;return t.r=255-t.r,t.g=255-t.g,t.b=255-t.b,this}lighten(t){return Xt(this._rgb,2,t),this}darken(t){return Xt(this._rgb,2,-t),this}saturate(t){return Xt(this._rgb,1,t),this}desaturate(t){return Xt(this._rgb,1,-t),this}rotate(t){return function(t,e){var i=It(t);i[0]=Vt(i[0]+e),i=Ft(i),t.r=i[0],t.g=i[1],t.b=i[2]}(this._rgb,t),this}}function Jt(t){if(t&&"object"==typeof t){const e=t.toString();return"[object CanvasPattern]"===e||"[object CanvasGradient]"===e}return!1}function Qt(t){return Jt(t)?t:new Zt(t)}function te(t){return Jt(t)?t:new Zt(t).saturate(.5).darken(.1).hexString()}const ee=["x","y","borderWidth","radius","tension"],ie=["color","borderColor","backgroundColor"];const se=new Map;function ne(t,e,i){return function(t,e){e=e||{};const i=t+JSON.stringify(e);let s=se.get(i);return s||(s=new Intl.NumberFormat(t,e),se.set(i,s)),s}(e,i).format(t)}const oe={values:t=>n(t)?t:""+t,numeric(t,e,i){if(0===t)return"0";const s=this.chart.options.locale;let n,o=t;if(i.length>1){const e=Math.max(Math.abs(i[0].value),Math.abs(i[i.length-1].value));(e<1e-4||e>1e15)&&(n="scientific"),o=function(t,e){let i=e.length>3?e[2].value-e[1].value:e[1].value-e[0].value;Math.abs(i)>=1&&t!==Math.floor(t)&&(i=t-Math.floor(t));return i}(t,i)}const a=z(Math.abs(o)),r=isNaN(a)?1:Math.max(Math.min(-1*Math.floor(a),20),0),l={notation:n,minimumFractionDigits:r,maximumFractionDigits:r};return Object.assign(l,this.options.ticks.format),ne(t,s,l)},logarithmic(t,e,i){if(0===t)return"0";const s=i[e].significand||t/Math.pow(10,Math.floor(z(t)));return[1,2,3,5,10,15].includes(s)||e>.8*i.length?oe.numeric.call(this,t,e,i):""}};var ae={formatters:oe};const re=Object.create(null),le=Object.create(null);function he(t,e){if(!e)return t;const i=e.split(".");for(let e=0,s=i.length;et.chart.platform.getDevicePixelRatio(),this.elements={},this.events=["mousemove","mouseout","click","touchstart","touchmove"],this.font={family:"'Helvetica Neue', 'Helvetica', 'Arial', sans-serif",size:12,style:"normal",lineHeight:1.2,weight:null},this.hover={},this.hoverBackgroundColor=(t,e)=>te(e.backgroundColor),this.hoverBorderColor=(t,e)=>te(e.borderColor),this.hoverColor=(t,e)=>te(e.color),this.indexAxis="x",this.interaction={mode:"nearest",intersect:!0,includeInvisible:!1},this.maintainAspectRatio=!0,this.onHover=null,this.onClick=null,this.parsing=!0,this.plugins={},this.responsive=!0,this.scale=void 0,this.scales={},this.showLine=!0,this.drawActiveElementsOnTop=!0,this.describe(t),this.apply(e)}set(t,e){return ce(this,t,e)}get(t){return he(this,t)}describe(t,e){return ce(le,t,e)}override(t,e){return ce(re,t,e)}route(t,e,i,s){const n=he(this,t),a=he(this,i),r="_"+e;Object.defineProperties(n,{[r]:{value:n[e],writable:!0},[e]:{enumerable:!0,get(){const t=this[r],e=a[s];return o(t)?Object.assign({},e,t):l(t,e)},set(t){this[r]=t}}})}apply(t){t.forEach((t=>t(this)))}}var ue=new de({_scriptable:t=>!t.startsWith("on"),_indexable:t=>"events"!==t,hover:{_fallback:"interaction"},interaction:{_scriptable:!1,_indexable:!1}},[function(t){t.set("animation",{delay:void 0,duration:1e3,easing:"easeOutQuart",fn:void 0,from:void 0,loop:void 0,to:void 0,type:void 0}),t.describe("animation",{_fallback:!1,_indexable:!1,_scriptable:t=>"onProgress"!==t&&"onComplete"!==t&&"fn"!==t}),t.set("animations",{colors:{type:"color",properties:ie},numbers:{type:"number",properties:ee}}),t.describe("animations",{_fallback:"animation"}),t.set("transitions",{active:{animation:{duration:400}},resize:{animation:{duration:0}},show:{animations:{colors:{from:"transparent"},visible:{type:"boolean",duration:0}}},hide:{animations:{colors:{to:"transparent"},visible:{type:"boolean",easing:"linear",fn:t=>0|t}}}})},function(t){t.set("layout",{autoPadding:!0,padding:{top:0,right:0,bottom:0,left:0}})},function(t){t.set("scale",{display:!0,offset:!1,reverse:!1,beginAtZero:!1,bounds:"ticks",clip:!0,grace:0,grid:{display:!0,lineWidth:1,drawOnChartArea:!0,drawTicks:!0,tickLength:8,tickWidth:(t,e)=>e.lineWidth,tickColor:(t,e)=>e.color,offset:!1},border:{display:!0,dash:[],dashOffset:0,width:1},title:{display:!1,text:"",padding:{top:4,bottom:4}},ticks:{minRotation:0,maxRotation:50,mirror:!1,textStrokeWidth:0,textStrokeColor:"",padding:3,display:!0,autoSkip:!0,autoSkipPadding:3,labelOffset:0,callback:ae.formatters.values,minor:{},major:{},align:"center",crossAlign:"near",showLabelBackdrop:!1,backdropColor:"rgba(255, 255, 255, 0.75)",backdropPadding:2}}),t.route("scale.ticks","color","","color"),t.route("scale.grid","color","","borderColor"),t.route("scale.border","color","","borderColor"),t.route("scale.title","color","","color"),t.describe("scale",{_fallback:!1,_scriptable:t=>!t.startsWith("before")&&!t.startsWith("after")&&"callback"!==t&&"parser"!==t,_indexable:t=>"borderDash"!==t&&"tickBorderDash"!==t&&"dash"!==t}),t.describe("scales",{_fallback:"scale"}),t.describe("scale.ticks",{_scriptable:t=>"backdropPadding"!==t&&"callback"!==t,_indexable:t=>"backdropPadding"!==t})}]);function fe(){return"undefined"!=typeof window&&"undefined"!=typeof document}function ge(t){let e=t.parentNode;return e&&"[object ShadowRoot]"===e.toString()&&(e=e.host),e}function pe(t,e,i){let s;return"string"==typeof t?(s=parseInt(t,10),-1!==t.indexOf("%")&&(s=s/100*e.parentNode[i])):s=t,s}const me=t=>t.ownerDocument.defaultView.getComputedStyle(t,null);function be(t,e){return me(t).getPropertyValue(e)}const xe=["top","right","bottom","left"];function _e(t,e,i){const s={};i=i?"-"+i:"";for(let n=0;n<4;n++){const o=xe[n];s[o]=parseFloat(t[e+"-"+o+i])||0}return s.width=s.left+s.right,s.height=s.top+s.bottom,s}const ye=(t,e,i)=>(t>0||e>0)&&(!i||!i.shadowRoot);function ve(t,e){if("native"in t)return t;const{canvas:i,currentDevicePixelRatio:s}=e,n=me(i),o="border-box"===n.boxSizing,a=_e(n,"padding"),r=_e(n,"border","width"),{x:l,y:h,box:c}=function(t,e){const i=t.touches,s=i&&i.length?i[0]:t,{offsetX:n,offsetY:o}=s;let a,r,l=!1;if(ye(n,o,t.target))a=n,r=o;else{const t=e.getBoundingClientRect();a=s.clientX-t.left,r=s.clientY-t.top,l=!0}return{x:a,y:r,box:l}}(t,i),d=a.left+(c&&r.left),u=a.top+(c&&r.top);let{width:f,height:g}=e;return o&&(f-=a.width+r.width,g-=a.height+r.height),{x:Math.round((l-d)/f*i.width/s),y:Math.round((h-u)/g*i.height/s)}}const Me=t=>Math.round(10*t)/10;function we(t,e,i,s){const n=me(t),o=_e(n,"margin"),a=pe(n.maxWidth,t,"clientWidth")||T,r=pe(n.maxHeight,t,"clientHeight")||T,l=function(t,e,i){let s,n;if(void 0===e||void 0===i){const o=ge(t);if(o){const t=o.getBoundingClientRect(),a=me(o),r=_e(a,"border","width"),l=_e(a,"padding");e=t.width-l.width-r.width,i=t.height-l.height-r.height,s=pe(a.maxWidth,o,"clientWidth"),n=pe(a.maxHeight,o,"clientHeight")}else e=t.clientWidth,i=t.clientHeight}return{width:e,height:i,maxWidth:s||T,maxHeight:n||T}}(t,e,i);let{width:h,height:c}=l;if("content-box"===n.boxSizing){const t=_e(n,"border","width"),e=_e(n,"padding");h-=e.width+t.width,c-=e.height+t.height}h=Math.max(0,h-o.width),c=Math.max(0,s?h/s:c-o.height),h=Me(Math.min(h,a,l.maxWidth)),c=Me(Math.min(c,r,l.maxHeight)),h&&!c&&(c=Me(h/2));return(void 0!==e||void 0!==i)&&s&&l.height&&c>l.height&&(c=l.height,h=Me(Math.floor(c*s))),{width:h,height:c}}function ke(t,e,i){const s=e||1,n=Math.floor(t.height*s),o=Math.floor(t.width*s);t.height=Math.floor(t.height),t.width=Math.floor(t.width);const a=t.canvas;return a.style&&(i||!a.style.height&&!a.style.width)&&(a.style.height=`${t.height}px`,a.style.width=`${t.width}px`),(t.currentDevicePixelRatio!==s||a.height!==n||a.width!==o)&&(t.currentDevicePixelRatio=s,a.height=n,a.width=o,t.ctx.setTransform(s,0,0,s,0,0),!0)}const Se=function(){let t=!1;try{const e={get passive(){return t=!0,!1}};window.addEventListener("test",null,e),window.removeEventListener("test",null,e)}catch(t){}return t}();function Pe(t,e){const i=be(t,e),s=i&&i.match(/^(\d+)(\.\d+)?px$/);return s?+s[1]:void 0}function De(t){return!t||s(t.size)||s(t.family)?null:(t.style?t.style+" ":"")+(t.weight?t.weight+" ":"")+t.size+"px "+t.family}function Ce(t,e,i,s,n){let o=e[n];return o||(o=e[n]=t.measureText(n).width,i.push(n)),o>s&&(s=o),s}function Oe(t,e,i,s){let o=(s=s||{}).data=s.data||{},a=s.garbageCollect=s.garbageCollect||[];s.font!==e&&(o=s.data={},a=s.garbageCollect=[],s.font=e),t.save(),t.font=e;let r=0;const l=i.length;let h,c,d,u,f;for(h=0;hi.length){for(h=0;h0&&t.stroke()}}function Re(t,e,i){return i=i||.5,!e||t&&t.x>e.left-i&&t.xe.top-i&&t.y0&&""!==r.strokeColor;let c,d;for(t.save(),t.font=a.string,function(t,e){e.translation&&t.translate(e.translation[0],e.translation[1]),s(e.rotation)||t.rotate(e.rotation),e.color&&(t.fillStyle=e.color),e.textAlign&&(t.textAlign=e.textAlign),e.textBaseline&&(t.textBaseline=e.textBaseline)}(t,r),c=0;ct[0])){const o=i||t;void 0===s&&(s=ti("_fallback",t));const a={[Symbol.toStringTag]:"Object",_cacheable:!0,_scopes:t,_rootScopes:o,_fallback:s,_getTarget:n,override:i=>je([i,...t],e,o,s)};return new Proxy(a,{deleteProperty:(e,i)=>(delete e[i],delete e._keys,delete t[0][i],!0),get:(i,s)=>qe(i,s,(()=>function(t,e,i,s){let n;for(const o of e)if(n=ti(Ue(o,t),i),void 0!==n)return Xe(t,n)?Je(i,s,t,n):n}(s,e,t,i))),getOwnPropertyDescriptor:(t,e)=>Reflect.getOwnPropertyDescriptor(t._scopes[0],e),getPrototypeOf:()=>Reflect.getPrototypeOf(t[0]),has:(t,e)=>ei(t).includes(e),ownKeys:t=>ei(t),set(t,e,i){const s=t._storage||(t._storage=n());return t[e]=s[e]=i,delete t._keys,!0}})}function $e(t,e,i,s){const a={_cacheable:!1,_proxy:t,_context:e,_subProxy:i,_stack:new Set,_descriptors:Ye(t,s),setContext:e=>$e(t,e,i,s),override:n=>$e(t.override(n),e,i,s)};return new Proxy(a,{deleteProperty:(e,i)=>(delete e[i],delete t[i],!0),get:(t,e,i)=>qe(t,e,(()=>function(t,e,i){const{_proxy:s,_context:a,_subProxy:r,_descriptors:l}=t;let h=s[e];S(h)&&l.isScriptable(e)&&(h=function(t,e,i,s){const{_proxy:n,_context:o,_subProxy:a,_stack:r}=i;if(r.has(t))throw new Error("Recursion detected: "+Array.from(r).join("->")+"->"+t);r.add(t);let l=e(o,a||s);r.delete(t),Xe(t,l)&&(l=Je(n._scopes,n,t,l));return l}(e,h,t,i));n(h)&&h.length&&(h=function(t,e,i,s){const{_proxy:n,_context:a,_subProxy:r,_descriptors:l}=i;if(void 0!==a.index&&s(t))return e[a.index%e.length];if(o(e[0])){const i=e,s=n._scopes.filter((t=>t!==i));e=[];for(const o of i){const i=Je(s,n,t,o);e.push($e(i,a,r&&r[t],l))}}return e}(e,h,t,l.isIndexable));Xe(e,h)&&(h=$e(h,a,r&&r[e],l));return h}(t,e,i))),getOwnPropertyDescriptor:(e,i)=>e._descriptors.allKeys?Reflect.has(t,i)?{enumerable:!0,configurable:!0}:void 0:Reflect.getOwnPropertyDescriptor(t,i),getPrototypeOf:()=>Reflect.getPrototypeOf(t),has:(e,i)=>Reflect.has(t,i),ownKeys:()=>Reflect.ownKeys(t),set:(e,i,s)=>(t[i]=s,delete e[i],!0)})}function Ye(t,e={scriptable:!0,indexable:!0}){const{_scriptable:i=e.scriptable,_indexable:s=e.indexable,_allKeys:n=e.allKeys}=t;return{allKeys:n,scriptable:i,indexable:s,isScriptable:S(i)?i:()=>i,isIndexable:S(s)?s:()=>s}}const Ue=(t,e)=>t?t+w(e):e,Xe=(t,e)=>o(e)&&"adapters"!==t&&(null===Object.getPrototypeOf(e)||e.constructor===Object);function qe(t,e,i){if(Object.prototype.hasOwnProperty.call(t,e))return t[e];const s=i();return t[e]=s,s}function Ke(t,e,i){return S(t)?t(e,i):t}const Ge=(t,e)=>!0===t?e:"string"==typeof t?M(e,t):void 0;function Ze(t,e,i,s,n){for(const o of e){const e=Ge(i,o);if(e){t.add(e);const o=Ke(e._fallback,i,n);if(void 0!==o&&o!==i&&o!==s)return o}else if(!1===e&&void 0!==s&&i!==s)return null}return!1}function Je(t,e,i,s){const a=e._rootScopes,r=Ke(e._fallback,i,s),l=[...t,...a],h=new Set;h.add(s);let c=Qe(h,l,i,r||i,s);return null!==c&&((void 0===r||r===i||(c=Qe(h,l,r,c,s),null!==c))&&je(Array.from(h),[""],a,r,(()=>function(t,e,i){const s=t._getTarget();e in s||(s[e]={});const a=s[e];if(n(a)&&o(i))return i;return a||{}}(e,i,s))))}function Qe(t,e,i,s,n){for(;i;)i=Ze(t,e,i,s,n);return i}function ti(t,e){for(const i of e){if(!i)continue;const e=i[t];if(void 0!==e)return e}}function ei(t){let e=t._keys;return e||(e=t._keys=function(t){const e=new Set;for(const i of t)for(const t of Object.keys(i).filter((t=>!t.startsWith("_"))))e.add(t);return Array.from(e)}(t._scopes)),e}function ii(t,e,i,s){const{iScale:n}=t,{key:o="r"}=this._parsing,a=new Array(s);let r,l,h,c;for(r=0,l=s;re"x"===t?"y":"x";function ai(t,e,i,s){const n=t.skip?e:t,o=e,a=i.skip?e:i,r=q(o,n),l=q(a,o);let h=r/(r+l),c=l/(r+l);h=isNaN(h)?0:h,c=isNaN(c)?0:c;const d=s*h,u=s*c;return{previous:{x:o.x-d*(a.x-n.x),y:o.y-d*(a.y-n.y)},next:{x:o.x+u*(a.x-n.x),y:o.y+u*(a.y-n.y)}}}function ri(t,e="x"){const i=oi(e),s=t.length,n=Array(s).fill(0),o=Array(s);let a,r,l,h=ni(t,0);for(a=0;a!t.skip))),"monotone"===e.cubicInterpolationMode)ri(t,n);else{let i=s?t[t.length-1]:t[0];for(o=0,a=t.length;o0===t||1===t,di=(t,e,i)=>-Math.pow(2,10*(t-=1))*Math.sin((t-e)*O/i),ui=(t,e,i)=>Math.pow(2,-10*t)*Math.sin((t-e)*O/i)+1,fi={linear:t=>t,easeInQuad:t=>t*t,easeOutQuad:t=>-t*(t-2),easeInOutQuad:t=>(t/=.5)<1?.5*t*t:-.5*(--t*(t-2)-1),easeInCubic:t=>t*t*t,easeOutCubic:t=>(t-=1)*t*t+1,easeInOutCubic:t=>(t/=.5)<1?.5*t*t*t:.5*((t-=2)*t*t+2),easeInQuart:t=>t*t*t*t,easeOutQuart:t=>-((t-=1)*t*t*t-1),easeInOutQuart:t=>(t/=.5)<1?.5*t*t*t*t:-.5*((t-=2)*t*t*t-2),easeInQuint:t=>t*t*t*t*t,easeOutQuint:t=>(t-=1)*t*t*t*t+1,easeInOutQuint:t=>(t/=.5)<1?.5*t*t*t*t*t:.5*((t-=2)*t*t*t*t+2),easeInSine:t=>1-Math.cos(t*E),easeOutSine:t=>Math.sin(t*E),easeInOutSine:t=>-.5*(Math.cos(C*t)-1),easeInExpo:t=>0===t?0:Math.pow(2,10*(t-1)),easeOutExpo:t=>1===t?1:1-Math.pow(2,-10*t),easeInOutExpo:t=>ci(t)?t:t<.5?.5*Math.pow(2,10*(2*t-1)):.5*(2-Math.pow(2,-10*(2*t-1))),easeInCirc:t=>t>=1?t:-(Math.sqrt(1-t*t)-1),easeOutCirc:t=>Math.sqrt(1-(t-=1)*t),easeInOutCirc:t=>(t/=.5)<1?-.5*(Math.sqrt(1-t*t)-1):.5*(Math.sqrt(1-(t-=2)*t)+1),easeInElastic:t=>ci(t)?t:di(t,.075,.3),easeOutElastic:t=>ci(t)?t:ui(t,.075,.3),easeInOutElastic(t){const e=.1125;return ci(t)?t:t<.5?.5*di(2*t,e,.45):.5+.5*ui(2*t-1,e,.45)},easeInBack(t){const e=1.70158;return t*t*((e+1)*t-e)},easeOutBack(t){const e=1.70158;return(t-=1)*t*((e+1)*t+e)+1},easeInOutBack(t){let e=1.70158;return(t/=.5)<1?t*t*((1+(e*=1.525))*t-e)*.5:.5*((t-=2)*t*((1+(e*=1.525))*t+e)+2)},easeInBounce:t=>1-fi.easeOutBounce(1-t),easeOutBounce(t){const e=7.5625,i=2.75;return t<1/i?e*t*t:t<2/i?e*(t-=1.5/i)*t+.75:t<2.5/i?e*(t-=2.25/i)*t+.9375:e*(t-=2.625/i)*t+.984375},easeInOutBounce:t=>t<.5?.5*fi.easeInBounce(2*t):.5*fi.easeOutBounce(2*t-1)+.5};function gi(t,e,i,s){return{x:t.x+i*(e.x-t.x),y:t.y+i*(e.y-t.y)}}function pi(t,e,i,s){return{x:t.x+i*(e.x-t.x),y:"middle"===s?i<.5?t.y:e.y:"after"===s?i<1?t.y:e.y:i>0?e.y:t.y}}function mi(t,e,i,s){const n={x:t.cp2x,y:t.cp2y},o={x:e.cp1x,y:e.cp1y},a=gi(t,n,i),r=gi(n,o,i),l=gi(o,e,i),h=gi(a,r,i),c=gi(r,l,i);return gi(h,c,i)}const bi=/^(normal|(\d+(?:\.\d+)?)(px|em|%)?)$/,xi=/^(normal|italic|initial|inherit|unset|(oblique( -?[0-9]?[0-9]deg)?))$/;function _i(t,e){const i=(""+t).match(bi);if(!i||"normal"===i[1])return 1.2*e;switch(t=+i[2],i[3]){case"px":return t;case"%":t/=100}return e*t}const yi=t=>+t||0;function vi(t,e){const i={},s=o(e),n=s?Object.keys(e):e,a=o(t)?s?i=>l(t[i],t[e[i]]):e=>t[e]:()=>t;for(const t of n)i[t]=yi(a(t));return i}function Mi(t){return vi(t,{top:"y",right:"x",bottom:"y",left:"x"})}function wi(t){return vi(t,["topLeft","topRight","bottomLeft","bottomRight"])}function ki(t){const e=Mi(t);return e.width=e.left+e.right,e.height=e.top+e.bottom,e}function Si(t,e){t=t||{},e=e||ue.font;let i=l(t.size,e.size);"string"==typeof i&&(i=parseInt(i,10));let s=l(t.style,e.style);s&&!(""+s).match(xi)&&(console.warn('Invalid font style specified: "'+s+'"'),s=void 0);const n={family:l(t.family,e.family),lineHeight:_i(l(t.lineHeight,e.lineHeight),i),size:i,style:s,weight:l(t.weight,e.weight),string:""};return n.string=De(n),n}function Pi(t,e,i,s){let o,a,r,l=!0;for(o=0,a=t.length;oi&&0===t?0:t+e;return{min:a(s,-Math.abs(o)),max:a(n,o)}}function Ci(t,e){return Object.assign(Object.create(t),e)}function Oi(t,e,i){return t?function(t,e){return{x:i=>t+t+e-i,setWidth(t){e=t},textAlign:t=>"center"===t?t:"right"===t?"left":"right",xPlus:(t,e)=>t-e,leftForLtr:(t,e)=>t-e}}(e,i):{x:t=>t,setWidth(t){},textAlign:t=>t,xPlus:(t,e)=>t+e,leftForLtr:(t,e)=>t}}function Ai(t,e){let i,s;"ltr"!==e&&"rtl"!==e||(i=t.canvas.style,s=[i.getPropertyValue("direction"),i.getPropertyPriority("direction")],i.setProperty("direction",e,"important"),t.prevTextDirection=s)}function Ti(t,e){void 0!==e&&(delete t.prevTextDirection,t.canvas.style.setProperty("direction",e[0],e[1]))}function Li(t){return"angle"===t?{between:Z,compare:K,normalize:G}:{between:tt,compare:(t,e)=>t-e,normalize:t=>t}}function Ei({start:t,end:e,count:i,loop:s,style:n}){return{start:t%i,end:e%i,loop:s&&(e-t+1)%i==0,style:n}}function Ri(t,e,i){if(!i)return[t];const{property:s,start:n,end:o}=i,a=e.length,{compare:r,between:l,normalize:h}=Li(s),{start:c,end:d,loop:u,style:f}=function(t,e,i){const{property:s,start:n,end:o}=i,{between:a,normalize:r}=Li(s),l=e.length;let h,c,{start:d,end:u,loop:f}=t;if(f){for(d+=l,u+=l,h=0,c=l;hx||l(n,b,p)&&0!==r(n,b),v=()=>!x||0===r(o,p)||l(o,b,p);for(let t=c,i=c;t<=d;++t)m=e[t%a],m.skip||(p=h(m[s]),p!==b&&(x=l(p,n,o),null===_&&y()&&(_=0===r(p,n)?t:i),null!==_&&v()&&(g.push(Ei({start:_,end:t,loop:u,count:a,style:f})),_=null),i=t,b=p));return null!==_&&g.push(Ei({start:_,end:d,loop:u,count:a,style:f})),g}function Ii(t,e){const i=[],s=t.segments;for(let n=0;nn&&t[o%e].skip;)o--;return o%=e,{start:n,end:o}}(i,n,o,s);if(!0===s)return Fi(t,[{start:a,end:r,loop:o}],i,e);return Fi(t,function(t,e,i,s){const n=t.length,o=[];let a,r=e,l=t[e];for(a=e+1;a<=i;++a){const i=t[a%n];i.skip||i.stop?l.skip||(s=!1,o.push({start:e%n,end:(a-1)%n,loop:s}),e=r=i.stop?a:null):(r=a,l.skip&&(e=a)),l=i}return null!==r&&o.push({start:e%n,end:r%n,loop:s}),o}(i,a,r{t[a](e[i],n)&&(o.push({element:t,datasetIndex:s,index:l}),r=r||t.inRange(e.x,e.y,n))})),s&&!r?[]:o}var Xi={evaluateInteractionItems:Hi,modes:{index(t,e,i,s){const n=ve(e,t),o=i.axis||"x",a=i.includeInvisible||!1,r=i.intersect?ji(t,n,o,s,a):Yi(t,n,o,!1,s,a),l=[];return r.length?(t.getSortedVisibleDatasetMetas().forEach((t=>{const e=r[0].index,i=t.data[e];i&&!i.skip&&l.push({element:i,datasetIndex:t.index,index:e})})),l):[]},dataset(t,e,i,s){const n=ve(e,t),o=i.axis||"xy",a=i.includeInvisible||!1;let r=i.intersect?ji(t,n,o,s,a):Yi(t,n,o,!1,s,a);if(r.length>0){const e=r[0].datasetIndex,i=t.getDatasetMeta(e).data;r=[];for(let t=0;tji(t,ve(e,t),i.axis||"xy",s,i.includeInvisible||!1),nearest(t,e,i,s){const n=ve(e,t),o=i.axis||"xy",a=i.includeInvisible||!1;return Yi(t,n,o,i.intersect,s,a)},x:(t,e,i,s)=>Ui(t,ve(e,t),"x",i.intersect,s),y:(t,e,i,s)=>Ui(t,ve(e,t),"y",i.intersect,s)}};const qi=["left","top","right","bottom"];function Ki(t,e){return t.filter((t=>t.pos===e))}function Gi(t,e){return t.filter((t=>-1===qi.indexOf(t.pos)&&t.box.axis===e))}function Zi(t,e){return t.sort(((t,i)=>{const s=e?i:t,n=e?t:i;return s.weight===n.weight?s.index-n.index:s.weight-n.weight}))}function Ji(t,e){const i=function(t){const e={};for(const i of t){const{stack:t,pos:s,stackWeight:n}=i;if(!t||!qi.includes(s))continue;const o=e[t]||(e[t]={count:0,placed:0,weight:0,size:0});o.count++,o.weight+=n}return e}(t),{vBoxMaxWidth:s,hBoxMaxHeight:n}=e;let o,a,r;for(o=0,a=t.length;o{s[t]=Math.max(e[t],i[t])})),s}return s(t?["left","right"]:["top","bottom"])}function ss(t,e,i,s){const n=[];let o,a,r,l,h,c;for(o=0,a=t.length,h=0;ot.box.fullSize)),!0),s=Zi(Ki(e,"left"),!0),n=Zi(Ki(e,"right")),o=Zi(Ki(e,"top"),!0),a=Zi(Ki(e,"bottom")),r=Gi(e,"x"),l=Gi(e,"y");return{fullSize:i,leftAndTop:s.concat(o),rightAndBottom:n.concat(l).concat(a).concat(r),chartArea:Ki(e,"chartArea"),vertical:s.concat(n).concat(l),horizontal:o.concat(a).concat(r)}}(t.boxes),l=r.vertical,h=r.horizontal;u(t.boxes,(t=>{"function"==typeof t.beforeLayout&&t.beforeLayout()}));const c=l.reduce(((t,e)=>e.box.options&&!1===e.box.options.display?t:t+1),0)||1,d=Object.freeze({outerWidth:e,outerHeight:i,padding:n,availableWidth:o,availableHeight:a,vBoxMaxWidth:o/2/c,hBoxMaxHeight:a/2}),f=Object.assign({},n);ts(f,ki(s));const g=Object.assign({maxPadding:f,w:o,h:a,x:n.left,y:n.top},n),p=Ji(l.concat(h),d);ss(r.fullSize,g,d,p),ss(l,g,d,p),ss(h,g,d,p)&&ss(l,g,d,p),function(t){const e=t.maxPadding;function i(i){const s=Math.max(e[i]-t[i],0);return t[i]+=s,s}t.y+=i("top"),t.x+=i("left"),i("right"),i("bottom")}(g),os(r.leftAndTop,g,d,p),g.x+=g.w,g.y+=g.h,os(r.rightAndBottom,g,d,p),t.chartArea={left:g.left,top:g.top,right:g.left+g.w,bottom:g.top+g.h,height:g.h,width:g.w},u(r.chartArea,(e=>{const i=e.box;Object.assign(i,t.chartArea),i.update(g.w,g.h,{left:0,top:0,right:0,bottom:0})}))}};class rs{acquireContext(t,e){}releaseContext(t){return!1}addEventListener(t,e,i){}removeEventListener(t,e,i){}getDevicePixelRatio(){return 1}getMaximumSize(t,e,i,s){return e=Math.max(0,e||t.width),i=i||t.height,{width:e,height:Math.max(0,s?Math.floor(e/s):i)}}isAttached(t){return!0}updateConfig(t){}}class ls extends rs{acquireContext(t){return t&&t.getContext&&t.getContext("2d")||null}updateConfig(t){t.options.animation=!1}}const hs="$chartjs",cs={touchstart:"mousedown",touchmove:"mousemove",touchend:"mouseup",pointerenter:"mouseenter",pointerdown:"mousedown",pointermove:"mousemove",pointerup:"mouseup",pointerleave:"mouseout",pointerout:"mouseout"},ds=t=>null===t||""===t;const us=!!Se&&{passive:!0};function fs(t,e,i){t.canvas.removeEventListener(e,i,us)}function gs(t,e){for(const i of t)if(i===e||i.contains(e))return!0}function ps(t,e,i){const s=t.canvas,n=new MutationObserver((t=>{let e=!1;for(const i of t)e=e||gs(i.addedNodes,s),e=e&&!gs(i.removedNodes,s);e&&i()}));return n.observe(document,{childList:!0,subtree:!0}),n}function ms(t,e,i){const s=t.canvas,n=new MutationObserver((t=>{let e=!1;for(const i of t)e=e||gs(i.removedNodes,s),e=e&&!gs(i.addedNodes,s);e&&i()}));return n.observe(document,{childList:!0,subtree:!0}),n}const bs=new Map;let xs=0;function _s(){const t=window.devicePixelRatio;t!==xs&&(xs=t,bs.forEach(((e,i)=>{i.currentDevicePixelRatio!==t&&e()})))}function ys(t,e,i){const s=t.canvas,n=s&&ge(s);if(!n)return;const o=ct(((t,e)=>{const s=n.clientWidth;i(t,e),s{const e=t[0],i=e.contentRect.width,s=e.contentRect.height;0===i&&0===s||o(i,s)}));return a.observe(n),function(t,e){bs.size||window.addEventListener("resize",_s),bs.set(t,e)}(t,o),a}function vs(t,e,i){i&&i.disconnect(),"resize"===e&&function(t){bs.delete(t),bs.size||window.removeEventListener("resize",_s)}(t)}function Ms(t,e,i){const s=t.canvas,n=ct((e=>{null!==t.ctx&&i(function(t,e){const i=cs[t.type]||t.type,{x:s,y:n}=ve(t,e);return{type:i,chart:e,native:t,x:void 0!==s?s:null,y:void 0!==n?n:null}}(e,t))}),t);return function(t,e,i){t.addEventListener(e,i,us)}(s,e,n),n}class ws extends rs{acquireContext(t,e){const i=t&&t.getContext&&t.getContext("2d");return i&&i.canvas===t?(function(t,e){const i=t.style,s=t.getAttribute("height"),n=t.getAttribute("width");if(t[hs]={initial:{height:s,width:n,style:{display:i.display,height:i.height,width:i.width}}},i.display=i.display||"block",i.boxSizing=i.boxSizing||"border-box",ds(n)){const e=Pe(t,"width");void 0!==e&&(t.width=e)}if(ds(s))if(""===t.style.height)t.height=t.width/(e||2);else{const e=Pe(t,"height");void 0!==e&&(t.height=e)}}(t,e),i):null}releaseContext(t){const e=t.canvas;if(!e[hs])return!1;const i=e[hs].initial;["height","width"].forEach((t=>{const n=i[t];s(n)?e.removeAttribute(t):e.setAttribute(t,n)}));const n=i.style||{};return Object.keys(n).forEach((t=>{e.style[t]=n[t]})),e.width=e.width,delete e[hs],!0}addEventListener(t,e,i){this.removeEventListener(t,e);const s=t.$proxies||(t.$proxies={}),n={attach:ps,detach:ms,resize:ys}[e]||Ms;s[e]=n(t,e,i)}removeEventListener(t,e){const i=t.$proxies||(t.$proxies={}),s=i[e];if(!s)return;({attach:vs,detach:vs,resize:vs}[e]||fs)(t,e,s),i[e]=void 0}getDevicePixelRatio(){return window.devicePixelRatio}getMaximumSize(t,e,i,s){return we(t,e,i,s)}isAttached(t){const e=ge(t);return!(!e||!e.isConnected)}}function ks(t){return!fe()||"undefined"!=typeof OffscreenCanvas&&t instanceof OffscreenCanvas?ls:ws}var Ss=Object.freeze({__proto__:null,BasePlatform:rs,BasicPlatform:ls,DomPlatform:ws,_detectPlatform:ks});const Ps="transparent",Ds={boolean:(t,e,i)=>i>.5?e:t,color(t,e,i){const s=Qt(t||Ps),n=s.valid&&Qt(e||Ps);return n&&n.valid?n.mix(s,i).hexString():e},number:(t,e,i)=>t+(e-t)*i};class Cs{constructor(t,e,i,s){const n=e[i];s=Pi([t.to,s,n,t.from]);const o=Pi([t.from,n,s]);this._active=!0,this._fn=t.fn||Ds[t.type||typeof o],this._easing=fi[t.easing]||fi.linear,this._start=Math.floor(Date.now()+(t.delay||0)),this._duration=this._total=Math.floor(t.duration),this._loop=!!t.loop,this._target=e,this._prop=i,this._from=o,this._to=s,this._promises=void 0}active(){return this._active}update(t,e,i){if(this._active){this._notify(!1);const s=this._target[this._prop],n=i-this._start,o=this._duration-n;this._start=i,this._duration=Math.floor(Math.max(o,t.duration)),this._total+=n,this._loop=!!t.loop,this._to=Pi([t.to,e,s,t.from]),this._from=Pi([t.from,s,e])}}cancel(){this._active&&(this.tick(Date.now()),this._active=!1,this._notify(!1))}tick(t){const e=t-this._start,i=this._duration,s=this._prop,n=this._from,o=this._loop,a=this._to;let r;if(this._active=n!==a&&(o||e1?2-r:r,r=this._easing(Math.min(1,Math.max(0,r))),this._target[s]=this._fn(n,a,r))}wait(){const t=this._promises||(this._promises=[]);return new Promise(((e,i)=>{t.push({res:e,rej:i})}))}_notify(t){const e=t?"res":"rej",i=this._promises||[];for(let t=0;t{const a=t[s];if(!o(a))return;const r={};for(const t of e)r[t]=a[t];(n(a.properties)&&a.properties||[s]).forEach((t=>{t!==s&&i.has(t)||i.set(t,r)}))}))}_animateOptions(t,e){const i=e.options,s=function(t,e){if(!e)return;let i=t.options;if(!i)return void(t.options=e);i.$shared&&(t.options=i=Object.assign({},i,{$shared:!1,$animations:{}}));return i}(t,i);if(!s)return[];const n=this._createAnimations(s,i);return i.$shared&&function(t,e){const i=[],s=Object.keys(e);for(let e=0;e{t.options=i}),(()=>{})),n}_createAnimations(t,e){const i=this._properties,s=[],n=t.$animations||(t.$animations={}),o=Object.keys(e),a=Date.now();let r;for(r=o.length-1;r>=0;--r){const l=o[r];if("$"===l.charAt(0))continue;if("options"===l){s.push(...this._animateOptions(t,e));continue}const h=e[l];let c=n[l];const d=i.get(l);if(c){if(d&&c.active()){c.update(d,h,a);continue}c.cancel()}d&&d.duration?(n[l]=c=new Cs(d,t,l,h),s.push(c)):t[l]=h}return s}update(t,e){if(0===this._properties.size)return void Object.assign(t,e);const i=this._createAnimations(t,e);return i.length?(xt.add(this._chart,i),!0):void 0}}function As(t,e){const i=t&&t.options||{},s=i.reverse,n=void 0===i.min?e:0,o=void 0===i.max?e:0;return{start:s?o:n,end:s?n:o}}function Ts(t,e){const i=[],s=t._getSortedDatasetMetas(e);let n,o;for(n=0,o=s.length;n0||!i&&e<0)return n.index}return null}function zs(t,e){const{chart:i,_cachedMeta:s}=t,n=i._stacks||(i._stacks={}),{iScale:o,vScale:a,index:r}=s,l=o.axis,h=a.axis,c=function(t,e,i){return`${t.id}.${e.id}.${i.stack||i.type}`}(o,a,s),d=e.length;let u;for(let t=0;ti[t].axis===e)).shift()}function Vs(t,e){const i=t.controller.index,s=t.vScale&&t.vScale.axis;if(s){e=e||t._parsed;for(const t of e){const e=t._stacks;if(!e||void 0===e[s]||void 0===e[s][i])return;delete e[s][i],void 0!==e[s]._visualValues&&void 0!==e[s]._visualValues[i]&&delete e[s]._visualValues[i]}}}const Bs=t=>"reset"===t||"none"===t,Ws=(t,e)=>e?t:Object.assign({},t);class Ns{static defaults={};static datasetElementType=null;static dataElementType=null;constructor(t,e){this.chart=t,this._ctx=t.ctx,this.index=e,this._cachedDataOpts={},this._cachedMeta=this.getMeta(),this._type=this._cachedMeta.type,this.options=void 0,this._parsing=!1,this._data=void 0,this._objectData=void 0,this._sharedOptions=void 0,this._drawStart=void 0,this._drawCount=void 0,this.enableOptionSharing=!1,this.supportsDecimation=!1,this.$context=void 0,this._syncList=[],this.datasetElementType=new.target.datasetElementType,this.dataElementType=new.target.dataElementType,this.initialize()}initialize(){const t=this._cachedMeta;this.configure(),this.linkScales(),t._stacked=Es(t.vScale,t),this.addElements(),this.options.fill&&!this.chart.isPluginEnabled("filler")&&console.warn("Tried to use the 'fill' option without the 'Filler' plugin enabled. Please import and register the 'Filler' plugin and make sure it is not disabled in the options")}updateIndex(t){this.index!==t&&Vs(this._cachedMeta),this.index=t}linkScales(){const t=this.chart,e=this._cachedMeta,i=this.getDataset(),s=(t,e,i,s)=>"x"===t?e:"r"===t?s:i,n=e.xAxisID=l(i.xAxisID,Fs(t,"x")),o=e.yAxisID=l(i.yAxisID,Fs(t,"y")),a=e.rAxisID=l(i.rAxisID,Fs(t,"r")),r=e.indexAxis,h=e.iAxisID=s(r,n,o,a),c=e.vAxisID=s(r,o,n,a);e.xScale=this.getScaleForId(n),e.yScale=this.getScaleForId(o),e.rScale=this.getScaleForId(a),e.iScale=this.getScaleForId(h),e.vScale=this.getScaleForId(c)}getDataset(){return this.chart.data.datasets[this.index]}getMeta(){return this.chart.getDatasetMeta(this.index)}getScaleForId(t){return this.chart.scales[t]}_getOtherScale(t){const e=this._cachedMeta;return t===e.iScale?e.vScale:e.iScale}reset(){this._update("reset")}_destroy(){const t=this._cachedMeta;this._data&&rt(this._data,this),t._stacked&&Vs(t)}_dataCheck(){const t=this.getDataset(),e=t.data||(t.data=[]),i=this._data;if(o(e))this._data=function(t){const e=Object.keys(t),i=new Array(e.length);let s,n,o;for(s=0,n=e.length;s0&&i._parsed[t-1];if(!1===this._parsing)i._parsed=s,i._sorted=!0,d=s;else{d=n(s[t])?this.parseArrayData(i,s,t,e):o(s[t])?this.parseObjectData(i,s,t,e):this.parsePrimitiveData(i,s,t,e);const a=()=>null===c[l]||f&&c[l]t&&!e.hidden&&e._stacked&&{keys:Ts(i,!0),values:null})(e,i,this.chart),h={min:Number.POSITIVE_INFINITY,max:Number.NEGATIVE_INFINITY},{min:c,max:d}=function(t){const{min:e,max:i,minDefined:s,maxDefined:n}=t.getUserBounds();return{min:s?e:Number.NEGATIVE_INFINITY,max:n?i:Number.POSITIVE_INFINITY}}(r);let u,f;function g(){f=s[u];const e=f[r.axis];return!a(f[t.axis])||c>e||d=0;--u)if(!g()){this.updateRangeFromParsed(h,t,f,l);break}return h}getAllParsedValues(t){const e=this._cachedMeta._parsed,i=[];let s,n,o;for(s=0,n=e.length;s=0&&tthis.getContext(i,s,e)),c);return f.$shared&&(f.$shared=r,n[o]=Object.freeze(Ws(f,r))),f}_resolveAnimations(t,e,i){const s=this.chart,n=this._cachedDataOpts,o=`animation-${e}`,a=n[o];if(a)return a;let r;if(!1!==s.options.animation){const s=this.chart.config,n=s.datasetAnimationScopeKeys(this._type,e),o=s.getOptionScopes(this.getDataset(),n);r=s.createResolver(o,this.getContext(t,i,e))}const l=new Os(s,r&&r.animations);return r&&r._cacheable&&(n[o]=Object.freeze(l)),l}getSharedOptions(t){if(t.$shared)return this._sharedOptions||(this._sharedOptions=Object.assign({},t))}includeOptions(t,e){return!e||Bs(t)||this.chart._animationsDisabled}_getSharedOptions(t,e){const i=this.resolveDataElementOptions(t,e),s=this._sharedOptions,n=this.getSharedOptions(i),o=this.includeOptions(e,n)||n!==s;return this.updateSharedOptions(n,e,i),{sharedOptions:n,includeOptions:o}}updateElement(t,e,i,s){Bs(s)?Object.assign(t,i):this._resolveAnimations(e,s).update(t,i)}updateSharedOptions(t,e,i){t&&!Bs(e)&&this._resolveAnimations(void 0,e).update(t,i)}_setStyle(t,e,i,s){t.active=s;const n=this.getStyle(e,s);this._resolveAnimations(e,i,s).update(t,{options:!s&&this.getSharedOptions(n)||n})}removeHoverStyle(t,e,i){this._setStyle(t,i,"active",!1)}setHoverStyle(t,e,i){this._setStyle(t,i,"active",!0)}_removeDatasetHoverStyle(){const t=this._cachedMeta.dataset;t&&this._setStyle(t,void 0,"active",!1)}_setDatasetHoverStyle(){const t=this._cachedMeta.dataset;t&&this._setStyle(t,void 0,"active",!0)}_resyncElements(t){const e=this._data,i=this._cachedMeta.data;for(const[t,e,i]of this._syncList)this[t](e,i);this._syncList=[];const s=i.length,n=e.length,o=Math.min(n,s);o&&this.parse(0,o),n>s?this._insertElements(s,n-s,t):n{for(t.length+=e,a=t.length-1;a>=o;a--)t[a]=t[a-e]};for(r(n),a=t;a{s[t]=i[t]&&i[t].active()?i[t]._to:this[t]})),s}}function js(t,e){const i=t.options.ticks,n=function(t){const e=t.options.offset,i=t._tickSize(),s=t._length/i+(e?0:1),n=t._maxLength/i;return Math.floor(Math.min(s,n))}(t),o=Math.min(i.maxTicksLimit||n,n),a=i.major.enabled?function(t){const e=[];let i,s;for(i=0,s=t.length;io)return function(t,e,i,s){let n,o=0,a=i[0];for(s=Math.ceil(s),n=0;nn)return e}return Math.max(n,1)}(a,e,o);if(r>0){let t,i;const n=r>1?Math.round((h-l)/(r-1)):null;for($s(e,c,d,s(n)?0:l-n,l),t=0,i=r-1;t"top"===e||"left"===e?t[e]+i:t[e]-i,Us=(t,e)=>Math.min(e||t,t);function Xs(t,e){const i=[],s=t.length/e,n=t.length;let o=0;for(;oa+r)))return h}function Ks(t){return t.drawTicks?t.tickLength:0}function Gs(t,e){if(!t.display)return 0;const i=Si(t.font,e),s=ki(t.padding);return(n(t.text)?t.text.length:1)*i.lineHeight+s.height}function Zs(t,e,i){let s=ut(t);return(i&&"right"!==e||!i&&"right"===e)&&(s=(t=>"left"===t?"right":"right"===t?"left":t)(s)),s}class Js extends Hs{constructor(t){super(),this.id=t.id,this.type=t.type,this.options=void 0,this.ctx=t.ctx,this.chart=t.chart,this.top=void 0,this.bottom=void 0,this.left=void 0,this.right=void 0,this.width=void 0,this.height=void 0,this._margins={left:0,right:0,top:0,bottom:0},this.maxWidth=void 0,this.maxHeight=void 0,this.paddingTop=void 0,this.paddingBottom=void 0,this.paddingLeft=void 0,this.paddingRight=void 0,this.axis=void 0,this.labelRotation=void 0,this.min=void 0,this.max=void 0,this._range=void 0,this.ticks=[],this._gridLineItems=null,this._labelItems=null,this._labelSizes=null,this._length=0,this._maxLength=0,this._longestTextCache={},this._startPixel=void 0,this._endPixel=void 0,this._reversePixels=!1,this._userMax=void 0,this._userMin=void 0,this._suggestedMax=void 0,this._suggestedMin=void 0,this._ticksLength=0,this._borderValue=0,this._cache={},this._dataLimitsCached=!1,this.$context=void 0}init(t){this.options=t.setContext(this.getContext()),this.axis=t.axis,this._userMin=this.parse(t.min),this._userMax=this.parse(t.max),this._suggestedMin=this.parse(t.suggestedMin),this._suggestedMax=this.parse(t.suggestedMax)}parse(t,e){return t}getUserBounds(){let{_userMin:t,_userMax:e,_suggestedMin:i,_suggestedMax:s}=this;return t=r(t,Number.POSITIVE_INFINITY),e=r(e,Number.NEGATIVE_INFINITY),i=r(i,Number.POSITIVE_INFINITY),s=r(s,Number.NEGATIVE_INFINITY),{min:r(t,i),max:r(e,s),minDefined:a(t),maxDefined:a(e)}}getMinMax(t){let e,{min:i,max:s,minDefined:n,maxDefined:o}=this.getUserBounds();if(n&&o)return{min:i,max:s};const a=this.getMatchingVisibleMetas();for(let r=0,l=a.length;rs?s:i,s=n&&i>s?i:s,{min:r(i,r(s,i)),max:r(s,r(i,s))}}getPadding(){return{left:this.paddingLeft||0,top:this.paddingTop||0,right:this.paddingRight||0,bottom:this.paddingBottom||0}}getTicks(){return this.ticks}getLabels(){const t=this.chart.data;return this.options.labels||(this.isHorizontal()?t.xLabels:t.yLabels)||t.labels||[]}getLabelItems(t=this.chart.chartArea){return this._labelItems||(this._labelItems=this._computeLabelItems(t))}beforeLayout(){this._cache={},this._dataLimitsCached=!1}beforeUpdate(){d(this.options.beforeUpdate,[this])}update(t,e,i){const{beginAtZero:s,grace:n,ticks:o}=this.options,a=o.sampleSize;this.beforeUpdate(),this.maxWidth=t,this.maxHeight=e,this._margins=i=Object.assign({left:0,right:0,top:0,bottom:0},i),this.ticks=null,this._labelSizes=null,this._gridLineItems=null,this._labelItems=null,this.beforeSetDimensions(),this.setDimensions(),this.afterSetDimensions(),this._maxLength=this.isHorizontal()?this.width+i.left+i.right:this.height+i.top+i.bottom,this._dataLimitsCached||(this.beforeDataLimits(),this.determineDataLimits(),this.afterDataLimits(),this._range=Di(this,n,s),this._dataLimitsCached=!0),this.beforeBuildTicks(),this.ticks=this.buildTicks()||[],this.afterBuildTicks();const r=a=n||i<=1||!this.isHorizontal())return void(this.labelRotation=s);const h=this._getLabelSizes(),c=h.widest.width,d=h.highest.height,u=J(this.chart.width-c,0,this.maxWidth);o=t.offset?this.maxWidth/i:u/(i-1),c+6>o&&(o=u/(i-(t.offset?.5:1)),a=this.maxHeight-Ks(t.grid)-e.padding-Gs(t.title,this.chart.options.font),r=Math.sqrt(c*c+d*d),l=Y(Math.min(Math.asin(J((h.highest.height+6)/o,-1,1)),Math.asin(J(a/r,-1,1))-Math.asin(J(d/r,-1,1)))),l=Math.max(s,Math.min(n,l))),this.labelRotation=l}afterCalculateLabelRotation(){d(this.options.afterCalculateLabelRotation,[this])}afterAutoSkip(){}beforeFit(){d(this.options.beforeFit,[this])}fit(){const t={width:0,height:0},{chart:e,options:{ticks:i,title:s,grid:n}}=this,o=this._isVisible(),a=this.isHorizontal();if(o){const o=Gs(s,e.options.font);if(a?(t.width=this.maxWidth,t.height=Ks(n)+o):(t.height=this.maxHeight,t.width=Ks(n)+o),i.display&&this.ticks.length){const{first:e,last:s,widest:n,highest:o}=this._getLabelSizes(),r=2*i.padding,l=$(this.labelRotation),h=Math.cos(l),c=Math.sin(l);if(a){const e=i.mirror?0:c*n.width+h*o.height;t.height=Math.min(this.maxHeight,t.height+e+r)}else{const e=i.mirror?0:h*n.width+c*o.height;t.width=Math.min(this.maxWidth,t.width+e+r)}this._calculatePadding(e,s,c,h)}}this._handleMargins(),a?(this.width=this._length=e.width-this._margins.left-this._margins.right,this.height=t.height):(this.width=t.width,this.height=this._length=e.height-this._margins.top-this._margins.bottom)}_calculatePadding(t,e,i,s){const{ticks:{align:n,padding:o},position:a}=this.options,r=0!==this.labelRotation,l="top"!==a&&"x"===this.axis;if(this.isHorizontal()){const a=this.getPixelForTick(0)-this.left,h=this.right-this.getPixelForTick(this.ticks.length-1);let c=0,d=0;r?l?(c=s*t.width,d=i*e.height):(c=i*t.height,d=s*e.width):"start"===n?d=e.width:"end"===n?c=t.width:"inner"!==n&&(c=t.width/2,d=e.width/2),this.paddingLeft=Math.max((c-a+o)*this.width/(this.width-a),0),this.paddingRight=Math.max((d-h+o)*this.width/(this.width-h),0)}else{let i=e.height/2,s=t.height/2;"start"===n?(i=0,s=t.height):"end"===n&&(i=e.height,s=0),this.paddingTop=i+o,this.paddingBottom=s+o}}_handleMargins(){this._margins&&(this._margins.left=Math.max(this.paddingLeft,this._margins.left),this._margins.top=Math.max(this.paddingTop,this._margins.top),this._margins.right=Math.max(this.paddingRight,this._margins.right),this._margins.bottom=Math.max(this.paddingBottom,this._margins.bottom))}afterFit(){d(this.options.afterFit,[this])}isHorizontal(){const{axis:t,position:e}=this.options;return"top"===e||"bottom"===e||"x"===t}isFullSize(){return this.options.fullSize}_convertTicksToLabels(t){let e,i;for(this.beforeTickToLabelConversion(),this.generateTickLabels(t),e=0,i=t.length;e{const i=t.gc,s=i.length/2;let n;if(s>e){for(n=0;n({width:r[t]||0,height:l[t]||0});return{first:P(0),last:P(e-1),widest:P(k),highest:P(S),widths:r,heights:l}}getLabelForValue(t){return t}getPixelForValue(t,e){return NaN}getValueForPixel(t){}getPixelForTick(t){const e=this.ticks;return t<0||t>e.length-1?null:this.getPixelForValue(e[t].value)}getPixelForDecimal(t){this._reversePixels&&(t=1-t);const e=this._startPixel+t*this._length;return Q(this._alignToPixels?Ae(this.chart,e,0):e)}getDecimalForPixel(t){const e=(t-this._startPixel)/this._length;return this._reversePixels?1-e:e}getBasePixel(){return this.getPixelForValue(this.getBaseValue())}getBaseValue(){const{min:t,max:e}=this;return t<0&&e<0?e:t>0&&e>0?t:0}getContext(t){const e=this.ticks||[];if(t>=0&&ta*s?a/i:r/s:r*s0}_computeGridLineItems(t){const e=this.axis,i=this.chart,s=this.options,{grid:n,position:a,border:r}=s,h=n.offset,c=this.isHorizontal(),d=this.ticks.length+(h?1:0),u=Ks(n),f=[],g=r.setContext(this.getContext()),p=g.display?g.width:0,m=p/2,b=function(t){return Ae(i,t,p)};let x,_,y,v,M,w,k,S,P,D,C,O;if("top"===a)x=b(this.bottom),w=this.bottom-u,S=x-m,D=b(t.top)+m,O=t.bottom;else if("bottom"===a)x=b(this.top),D=t.top,O=b(t.bottom)-m,w=x+m,S=this.top+u;else if("left"===a)x=b(this.right),M=this.right-u,k=x-m,P=b(t.left)+m,C=t.right;else if("right"===a)x=b(this.left),P=t.left,C=b(t.right)-m,M=x+m,k=this.left+u;else if("x"===e){if("center"===a)x=b((t.top+t.bottom)/2+.5);else if(o(a)){const t=Object.keys(a)[0],e=a[t];x=b(this.chart.scales[t].getPixelForValue(e))}D=t.top,O=t.bottom,w=x+m,S=w+u}else if("y"===e){if("center"===a)x=b((t.left+t.right)/2);else if(o(a)){const t=Object.keys(a)[0],e=a[t];x=b(this.chart.scales[t].getPixelForValue(e))}M=x-m,k=M-u,P=t.left,C=t.right}const A=l(s.ticks.maxTicksLimit,d),T=Math.max(1,Math.ceil(d/A));for(_=0;_e.value===t));if(i>=0){return e.setContext(this.getContext(i)).lineWidth}return 0}drawGrid(t){const e=this.options.grid,i=this.ctx,s=this._gridLineItems||(this._gridLineItems=this._computeGridLineItems(t));let n,o;const a=(t,e,s)=>{s.width&&s.color&&(i.save(),i.lineWidth=s.width,i.strokeStyle=s.color,i.setLineDash(s.borderDash||[]),i.lineDashOffset=s.borderDashOffset,i.beginPath(),i.moveTo(t.x,t.y),i.lineTo(e.x,e.y),i.stroke(),i.restore())};if(e.display)for(n=0,o=s.length;n{this.drawBackground(),this.drawGrid(t),this.drawTitle()}},{z:s,draw:()=>{this.drawBorder()}},{z:e,draw:t=>{this.drawLabels(t)}}]:[{z:e,draw:t=>{this.draw(t)}}]}getMatchingVisibleMetas(t){const e=this.chart.getSortedVisibleDatasetMetas(),i=this.axis+"AxisID",s=[];let n,o;for(n=0,o=e.length;n{const s=i.split("."),n=s.pop(),o=[t].concat(s).join("."),a=e[i].split("."),r=a.pop(),l=a.join(".");ue.route(o,n,l,r)}))}(e,t.defaultRoutes);t.descriptors&&ue.describe(e,t.descriptors)}(t,o,i),this.override&&ue.override(t.id,t.overrides)),o}get(t){return this.items[t]}unregister(t){const e=this.items,i=t.id,s=this.scope;i in e&&delete e[i],s&&i in ue[s]&&(delete ue[s][i],this.override&&delete re[i])}}class tn{constructor(){this.controllers=new Qs(Ns,"datasets",!0),this.elements=new Qs(Hs,"elements"),this.plugins=new Qs(Object,"plugins"),this.scales=new Qs(Js,"scales"),this._typedRegistries=[this.controllers,this.scales,this.elements]}add(...t){this._each("register",t)}remove(...t){this._each("unregister",t)}addControllers(...t){this._each("register",t,this.controllers)}addElements(...t){this._each("register",t,this.elements)}addPlugins(...t){this._each("register",t,this.plugins)}addScales(...t){this._each("register",t,this.scales)}getController(t){return this._get(t,this.controllers,"controller")}getElement(t){return this._get(t,this.elements,"element")}getPlugin(t){return this._get(t,this.plugins,"plugin")}getScale(t){return this._get(t,this.scales,"scale")}removeControllers(...t){this._each("unregister",t,this.controllers)}removeElements(...t){this._each("unregister",t,this.elements)}removePlugins(...t){this._each("unregister",t,this.plugins)}removeScales(...t){this._each("unregister",t,this.scales)}_each(t,e,i){[...e].forEach((e=>{const s=i||this._getRegistryForType(e);i||s.isForType(e)||s===this.plugins&&e.id?this._exec(t,s,e):u(e,(e=>{const s=i||this._getRegistryForType(e);this._exec(t,s,e)}))}))}_exec(t,e,i){const s=w(t);d(i["before"+s],[],i),e[t](i),d(i["after"+s],[],i)}_getRegistryForType(t){for(let e=0;et.filter((t=>!e.some((e=>t.plugin.id===e.plugin.id))));this._notify(s(e,i),t,"stop"),this._notify(s(i,e),t,"start")}}function nn(t,e){return e||!1!==t?!0===t?{}:t:null}function on(t,{plugin:e,local:i},s,n){const o=t.pluginScopeKeys(e),a=t.getOptionScopes(s,o);return i&&e.defaults&&a.push(e.defaults),t.createResolver(a,n,[""],{scriptable:!1,indexable:!1,allKeys:!0})}function an(t,e){const i=ue.datasets[t]||{};return((e.datasets||{})[t]||{}).indexAxis||e.indexAxis||i.indexAxis||"x"}function rn(t){if("x"===t||"y"===t||"r"===t)return t}function ln(t,...e){if(rn(t))return t;for(const s of e){const e=s.axis||("top"===(i=s.position)||"bottom"===i?"x":"left"===i||"right"===i?"y":void 0)||t.length>1&&rn(t[0].toLowerCase());if(e)return e}var i;throw new Error(`Cannot determine type of '${t}' axis. Please provide 'axis' or 'position' option.`)}function hn(t,e,i){if(i[e+"AxisID"]===t)return{axis:e}}function cn(t,e){const i=re[t.type]||{scales:{}},s=e.scales||{},n=an(t.type,e),a=Object.create(null);return Object.keys(s).forEach((e=>{const r=s[e];if(!o(r))return console.error(`Invalid scale configuration for scale: ${e}`);if(r._proxy)return console.warn(`Ignoring resolver passed as options for scale: ${e}`);const l=ln(e,r,function(t,e){if(e.data&&e.data.datasets){const i=e.data.datasets.filter((e=>e.xAxisID===t||e.yAxisID===t));if(i.length)return hn(t,"x",i[0])||hn(t,"y",i[0])}return{}}(e,t),ue.scales[r.type]),h=function(t,e){return t===e?"_index_":"_value_"}(l,n),c=i.scales||{};a[e]=x(Object.create(null),[{axis:l},r,c[l],c[h]])})),t.data.datasets.forEach((i=>{const n=i.type||t.type,o=i.indexAxis||an(n,e),r=(re[n]||{}).scales||{};Object.keys(r).forEach((t=>{const e=function(t,e){let i=t;return"_index_"===t?i=e:"_value_"===t&&(i="x"===e?"y":"x"),i}(t,o),n=i[e+"AxisID"]||e;a[n]=a[n]||Object.create(null),x(a[n],[{axis:e},s[n],r[t]])}))})),Object.keys(a).forEach((t=>{const e=a[t];x(e,[ue.scales[e.type],ue.scale])})),a}function dn(t){const e=t.options||(t.options={});e.plugins=l(e.plugins,{}),e.scales=cn(t,e)}function un(t){return(t=t||{}).datasets=t.datasets||[],t.labels=t.labels||[],t}const fn=new Map,gn=new Set;function pn(t,e){let i=fn.get(t);return i||(i=e(),fn.set(t,i),gn.add(i)),i}const mn=(t,e,i)=>{const s=M(e,i);void 0!==s&&t.add(s)};class bn{constructor(t){this._config=function(t){return(t=t||{}).data=un(t.data),dn(t),t}(t),this._scopeCache=new Map,this._resolverCache=new Map}get platform(){return this._config.platform}get type(){return this._config.type}set type(t){this._config.type=t}get data(){return this._config.data}set data(t){this._config.data=un(t)}get options(){return this._config.options}set options(t){this._config.options=t}get plugins(){return this._config.plugins}update(){const t=this._config;this.clearCache(),dn(t)}clearCache(){this._scopeCache.clear(),this._resolverCache.clear()}datasetScopeKeys(t){return pn(t,(()=>[[`datasets.${t}`,""]]))}datasetAnimationScopeKeys(t,e){return pn(`${t}.transition.${e}`,(()=>[[`datasets.${t}.transitions.${e}`,`transitions.${e}`],[`datasets.${t}`,""]]))}datasetElementScopeKeys(t,e){return pn(`${t}-${e}`,(()=>[[`datasets.${t}.elements.${e}`,`datasets.${t}`,`elements.${e}`,""]]))}pluginScopeKeys(t){const e=t.id;return pn(`${this.type}-plugin-${e}`,(()=>[[`plugins.${e}`,...t.additionalOptionScopes||[]]]))}_cachedScopes(t,e){const i=this._scopeCache;let s=i.get(t);return s&&!e||(s=new Map,i.set(t,s)),s}getOptionScopes(t,e,i){const{options:s,type:n}=this,o=this._cachedScopes(t,i),a=o.get(e);if(a)return a;const r=new Set;e.forEach((e=>{t&&(r.add(t),e.forEach((e=>mn(r,t,e)))),e.forEach((t=>mn(r,s,t))),e.forEach((t=>mn(r,re[n]||{},t))),e.forEach((t=>mn(r,ue,t))),e.forEach((t=>mn(r,le,t)))}));const l=Array.from(r);return 0===l.length&&l.push(Object.create(null)),gn.has(e)&&o.set(e,l),l}chartOptionScopes(){const{options:t,type:e}=this;return[t,re[e]||{},ue.datasets[e]||{},{type:e},ue,le]}resolveNamedOptions(t,e,i,s=[""]){const o={$shared:!0},{resolver:a,subPrefixes:r}=xn(this._resolverCache,t,s);let l=a;if(function(t,e){const{isScriptable:i,isIndexable:s}=Ye(t);for(const o of e){const e=i(o),a=s(o),r=(a||e)&&t[o];if(e&&(S(r)||_n(r))||a&&n(r))return!0}return!1}(a,e)){o.$shared=!1;l=$e(a,i=S(i)?i():i,this.createResolver(t,i,r))}for(const t of e)o[t]=l[t];return o}createResolver(t,e,i=[""],s){const{resolver:n}=xn(this._resolverCache,t,i);return o(e)?$e(n,e,void 0,s):n}}function xn(t,e,i){let s=t.get(e);s||(s=new Map,t.set(e,s));const n=i.join();let o=s.get(n);if(!o){o={resolver:je(e,i),subPrefixes:i.filter((t=>!t.toLowerCase().includes("hover")))},s.set(n,o)}return o}const _n=t=>o(t)&&Object.getOwnPropertyNames(t).reduce(((e,i)=>e||S(t[i])),!1);const yn=["top","bottom","left","right","chartArea"];function vn(t,e){return"top"===t||"bottom"===t||-1===yn.indexOf(t)&&"x"===e}function Mn(t,e){return function(i,s){return i[t]===s[t]?i[e]-s[e]:i[t]-s[t]}}function wn(t){const e=t.chart,i=e.options.animation;e.notifyPlugins("afterRender"),d(i&&i.onComplete,[t],e)}function kn(t){const e=t.chart,i=e.options.animation;d(i&&i.onProgress,[t],e)}function Sn(t){return fe()&&"string"==typeof t?t=document.getElementById(t):t&&t.length&&(t=t[0]),t&&t.canvas&&(t=t.canvas),t}const Pn={},Dn=t=>{const e=Sn(t);return Object.values(Pn).filter((t=>t.canvas===e)).pop()};function Cn(t,e,i){const s=Object.keys(t);for(const n of s){const s=+n;if(s>=e){const o=t[n];delete t[n],(i>0||s>e)&&(t[s+i]=o)}}}function On(t,e,i){return t.options.clip?t[i]:e[i]}class An{static defaults=ue;static instances=Pn;static overrides=re;static registry=en;static version="4.4.0";static getChart=Dn;static register(...t){en.add(...t),Tn()}static unregister(...t){en.remove(...t),Tn()}constructor(t,e){const s=this.config=new bn(e),n=Sn(t),o=Dn(n);if(o)throw new Error("Canvas is already in use. Chart with ID '"+o.id+"' must be destroyed before the canvas with ID '"+o.canvas.id+"' can be reused.");const a=s.createResolver(s.chartOptionScopes(),this.getContext());this.platform=new(s.platform||ks(n)),this.platform.updateConfig(s);const r=this.platform.acquireContext(n,a.aspectRatio),l=r&&r.canvas,h=l&&l.height,c=l&&l.width;this.id=i(),this.ctx=r,this.canvas=l,this.width=c,this.height=h,this._options=a,this._aspectRatio=this.aspectRatio,this._layers=[],this._metasets=[],this._stacks=void 0,this.boxes=[],this.currentDevicePixelRatio=void 0,this.chartArea=void 0,this._active=[],this._lastEvent=void 0,this._listeners={},this._responsiveListeners=void 0,this._sortedMetasets=[],this.scales={},this._plugins=new sn,this.$proxies={},this._hiddenIndices={},this.attached=!1,this._animationsDisabled=void 0,this.$context=void 0,this._doResize=dt((t=>this.update(t)),a.resizeDelay||0),this._dataChanges=[],Pn[this.id]=this,r&&l?(xt.listen(this,"complete",wn),xt.listen(this,"progress",kn),this._initialize(),this.attached&&this.update()):console.error("Failed to create chart: can't acquire context from the given item")}get aspectRatio(){const{options:{aspectRatio:t,maintainAspectRatio:e},width:i,height:n,_aspectRatio:o}=this;return s(t)?e&&o?o:n?i/n:null:t}get data(){return this.config.data}set data(t){this.config.data=t}get options(){return this._options}set options(t){this.config.options=t}get registry(){return en}_initialize(){return this.notifyPlugins("beforeInit"),this.options.responsive?this.resize():ke(this,this.options.devicePixelRatio),this.bindEvents(),this.notifyPlugins("afterInit"),this}clear(){return Te(this.canvas,this.ctx),this}stop(){return xt.stop(this),this}resize(t,e){xt.running(this)?this._resizeBeforeDraw={width:t,height:e}:this._resize(t,e)}_resize(t,e){const i=this.options,s=this.canvas,n=i.maintainAspectRatio&&this.aspectRatio,o=this.platform.getMaximumSize(s,t,e,n),a=i.devicePixelRatio||this.platform.getDevicePixelRatio(),r=this.width?"resize":"attach";this.width=o.width,this.height=o.height,this._aspectRatio=this.aspectRatio,ke(this,a,!0)&&(this.notifyPlugins("resize",{size:o}),d(i.onResize,[this,o],this),this.attached&&this._doResize(r)&&this.render())}ensureScalesHaveIDs(){u(this.options.scales||{},((t,e)=>{t.id=e}))}buildOrUpdateScales(){const t=this.options,e=t.scales,i=this.scales,s=Object.keys(i).reduce(((t,e)=>(t[e]=!1,t)),{});let n=[];e&&(n=n.concat(Object.keys(e).map((t=>{const i=e[t],s=ln(t,i),n="r"===s,o="x"===s;return{options:i,dposition:n?"chartArea":o?"bottom":"left",dtype:n?"radialLinear":o?"category":"linear"}})))),u(n,(e=>{const n=e.options,o=n.id,a=ln(o,n),r=l(n.type,e.dtype);void 0!==n.position&&vn(n.position,a)===vn(e.dposition)||(n.position=e.dposition),s[o]=!0;let h=null;if(o in i&&i[o].type===r)h=i[o];else{h=new(en.getScale(r))({id:o,type:r,ctx:this.ctx,chart:this}),i[h.id]=h}h.init(n,t)})),u(s,((t,e)=>{t||delete i[e]})),u(i,(t=>{as.configure(this,t,t.options),as.addBox(this,t)}))}_updateMetasets(){const t=this._metasets,e=this.data.datasets.length,i=t.length;if(t.sort(((t,e)=>t.index-e.index)),i>e){for(let t=e;te.length&&delete this._stacks,t.forEach(((t,i)=>{0===e.filter((e=>e===t._dataset)).length&&this._destroyDatasetMeta(i)}))}buildOrUpdateControllers(){const t=[],e=this.data.datasets;let i,s;for(this._removeUnreferencedMetasets(),i=0,s=e.length;i{this.getDatasetMeta(e).controller.reset()}),this)}reset(){this._resetElements(),this.notifyPlugins("reset")}update(t){const e=this.config;e.update();const i=this._options=e.createResolver(e.chartOptionScopes(),this.getContext()),s=this._animationsDisabled=!i.animation;if(this._updateScales(),this._checkEventBindings(),this._updateHiddenIndices(),this._plugins.invalidate(),!1===this.notifyPlugins("beforeUpdate",{mode:t,cancelable:!0}))return;const n=this.buildOrUpdateControllers();this.notifyPlugins("beforeElementsUpdate");let o=0;for(let t=0,e=this.data.datasets.length;t{t.reset()})),this._updateDatasets(t),this.notifyPlugins("afterUpdate",{mode:t}),this._layers.sort(Mn("z","_idx"));const{_active:a,_lastEvent:r}=this;r?this._eventHandler(r,!0):a.length&&this._updateHoverStyles(a,a,!0),this.render()}_updateScales(){u(this.scales,(t=>{as.removeBox(this,t)})),this.ensureScalesHaveIDs(),this.buildOrUpdateScales()}_checkEventBindings(){const t=this.options,e=new Set(Object.keys(this._listeners)),i=new Set(t.events);P(e,i)&&!!this._responsiveListeners===t.responsive||(this.unbindEvents(),this.bindEvents())}_updateHiddenIndices(){const{_hiddenIndices:t}=this,e=this._getUniformDataChanges()||[];for(const{method:i,start:s,count:n}of e){Cn(t,s,"_removeElements"===i?-n:n)}}_getUniformDataChanges(){const t=this._dataChanges;if(!t||!t.length)return;this._dataChanges=[];const e=this.data.datasets.length,i=e=>new Set(t.filter((t=>t[0]===e)).map(((t,e)=>e+","+t.splice(1).join(",")))),s=i(0);for(let t=1;tt.split(","))).map((t=>({method:t[1],start:+t[2],count:+t[3]})))}_updateLayout(t){if(!1===this.notifyPlugins("beforeLayout",{cancelable:!0}))return;as.update(this,this.width,this.height,t);const e=this.chartArea,i=e.width<=0||e.height<=0;this._layers=[],u(this.boxes,(t=>{i&&"chartArea"===t.position||(t.configure&&t.configure(),this._layers.push(...t._layers()))}),this),this._layers.forEach(((t,e)=>{t._idx=e})),this.notifyPlugins("afterLayout")}_updateDatasets(t){if(!1!==this.notifyPlugins("beforeDatasetsUpdate",{mode:t,cancelable:!0})){for(let t=0,e=this.data.datasets.length;t=0;--e)this._drawDataset(t[e]);this.notifyPlugins("afterDatasetsDraw")}_drawDataset(t){const e=this.ctx,i=t._clip,s=!i.disabled,n=function(t,e){const{xScale:i,yScale:s}=t;return i&&s?{left:On(i,e,"left"),right:On(i,e,"right"),top:On(s,e,"top"),bottom:On(s,e,"bottom")}:e}(t,this.chartArea),o={meta:t,index:t.index,cancelable:!0};!1!==this.notifyPlugins("beforeDatasetDraw",o)&&(s&&Ie(e,{left:!1===i.left?0:n.left-i.left,right:!1===i.right?this.width:n.right+i.right,top:!1===i.top?0:n.top-i.top,bottom:!1===i.bottom?this.height:n.bottom+i.bottom}),t.controller.draw(),s&&ze(e),o.cancelable=!1,this.notifyPlugins("afterDatasetDraw",o))}isPointInArea(t){return Re(t,this.chartArea,this._minPadding)}getElementsAtEventForMode(t,e,i,s){const n=Xi.modes[e];return"function"==typeof n?n(this,t,i,s):[]}getDatasetMeta(t){const e=this.data.datasets[t],i=this._metasets;let s=i.filter((t=>t&&t._dataset===e)).pop();return s||(s={type:null,data:[],dataset:null,controller:null,hidden:null,xAxisID:null,yAxisID:null,order:e&&e.order||0,index:t,_dataset:e,_parsed:[],_sorted:!1},i.push(s)),s}getContext(){return this.$context||(this.$context=Ci(null,{chart:this,type:"chart"}))}getVisibleDatasetCount(){return this.getSortedVisibleDatasetMetas().length}isDatasetVisible(t){const e=this.data.datasets[t];if(!e)return!1;const i=this.getDatasetMeta(t);return"boolean"==typeof i.hidden?!i.hidden:!e.hidden}setDatasetVisibility(t,e){this.getDatasetMeta(t).hidden=!e}toggleDataVisibility(t){this._hiddenIndices[t]=!this._hiddenIndices[t]}getDataVisibility(t){return!this._hiddenIndices[t]}_updateVisibility(t,e,i){const s=i?"show":"hide",n=this.getDatasetMeta(t),o=n.controller._resolveAnimations(void 0,s);k(e)?(n.data[e].hidden=!i,this.update()):(this.setDatasetVisibility(t,i),o.update(n,{visible:i}),this.update((e=>e.datasetIndex===t?s:void 0)))}hide(t,e){this._updateVisibility(t,e,!1)}show(t,e){this._updateVisibility(t,e,!0)}_destroyDatasetMeta(t){const e=this._metasets[t];e&&e.controller&&e.controller._destroy(),delete this._metasets[t]}_stop(){let t,e;for(this.stop(),xt.remove(this),t=0,e=this.data.datasets.length;t{e.addEventListener(this,i,s),t[i]=s},s=(t,e,i)=>{t.offsetX=e,t.offsetY=i,this._eventHandler(t)};u(this.options.events,(t=>i(t,s)))}bindResponsiveEvents(){this._responsiveListeners||(this._responsiveListeners={});const t=this._responsiveListeners,e=this.platform,i=(i,s)=>{e.addEventListener(this,i,s),t[i]=s},s=(i,s)=>{t[i]&&(e.removeEventListener(this,i,s),delete t[i])},n=(t,e)=>{this.canvas&&this.resize(t,e)};let o;const a=()=>{s("attach",a),this.attached=!0,this.resize(),i("resize",n),i("detach",o)};o=()=>{this.attached=!1,s("resize",n),this._stop(),this._resize(0,0),i("attach",a)},e.isAttached(this.canvas)?a():o()}unbindEvents(){u(this._listeners,((t,e)=>{this.platform.removeEventListener(this,e,t)})),this._listeners={},u(this._responsiveListeners,((t,e)=>{this.platform.removeEventListener(this,e,t)})),this._responsiveListeners=void 0}updateHoverStyle(t,e,i){const s=i?"set":"remove";let n,o,a,r;for("dataset"===e&&(n=this.getDatasetMeta(t[0].datasetIndex),n.controller["_"+s+"DatasetHoverStyle"]()),a=0,r=t.length;a{const i=this.getDatasetMeta(t);if(!i)throw new Error("No dataset found at index "+t);return{datasetIndex:t,element:i.data[e],index:e}}));!f(i,e)&&(this._active=i,this._lastEvent=null,this._updateHoverStyles(i,e))}notifyPlugins(t,e,i){return this._plugins.notify(this,t,e,i)}isPluginEnabled(t){return 1===this._plugins._cache.filter((e=>e.plugin.id===t)).length}_updateHoverStyles(t,e,i){const s=this.options.hover,n=(t,e)=>t.filter((t=>!e.some((e=>t.datasetIndex===e.datasetIndex&&t.index===e.index)))),o=n(e,t),a=i?t:n(t,e);o.length&&this.updateHoverStyle(o,s.mode,!1),a.length&&s.mode&&this.updateHoverStyle(a,s.mode,!0)}_eventHandler(t,e){const i={event:t,replay:e,cancelable:!0,inChartArea:this.isPointInArea(t)},s=e=>(e.options.events||this.options.events).includes(t.native.type);if(!1===this.notifyPlugins("beforeEvent",i,s))return;const n=this._handleEvent(t,e,i.inChartArea);return i.cancelable=!1,this.notifyPlugins("afterEvent",i,s),(n||i.changed)&&this.render(),this}_handleEvent(t,e,i){const{_active:s=[],options:n}=this,o=e,a=this._getActiveElements(t,s,i,o),r=D(t),l=function(t,e,i,s){return i&&"mouseout"!==t.type?s?e:t:null}(t,this._lastEvent,i,r);i&&(this._lastEvent=null,d(n.onHover,[t,a,this],this),r&&d(n.onClick,[t,a,this],this));const h=!f(a,s);return(h||e)&&(this._active=a,this._updateHoverStyles(a,s,e)),this._lastEvent=l,h}_getActiveElements(t,e,i,s){if("mouseout"===t.type)return[];if(!i)return e;const n=this.options.hover;return this.getElementsAtEventForMode(t,n.mode,n,s)}}function Tn(){return u(An.instances,(t=>t._plugins.invalidate()))}function Ln(){throw new Error("This method is not implemented: Check that a complete date adapter is provided.")}class En{static override(t){Object.assign(En.prototype,t)}options;constructor(t){this.options=t||{}}init(){}formats(){return Ln()}parse(){return Ln()}format(){return Ln()}add(){return Ln()}diff(){return Ln()}startOf(){return Ln()}endOf(){return Ln()}}var Rn={_date:En};function In(t){const e=t.iScale,i=function(t,e){if(!t._cache.$bar){const i=t.getMatchingVisibleMetas(e);let s=[];for(let e=0,n=i.length;et-e)))}return t._cache.$bar}(e,t.type);let s,n,o,a,r=e._length;const l=()=>{32767!==o&&-32768!==o&&(k(a)&&(r=Math.min(r,Math.abs(o-a)||r)),a=o)};for(s=0,n=i.length;sMath.abs(r)&&(l=r,h=a),e[i.axis]=h,e._custom={barStart:l,barEnd:h,start:n,end:o,min:a,max:r}}(t,e,i,s):e[i.axis]=i.parse(t,s),e}function Fn(t,e,i,s){const n=t.iScale,o=t.vScale,a=n.getLabels(),r=n===o,l=[];let h,c,d,u;for(h=i,c=i+s;ht.x,i="left",s="right"):(e=t.base"spacing"!==t,_indexable:t=>"spacing"!==t&&!t.startsWith("borderDash")&&!t.startsWith("hoverBorderDash")};static overrides={aspectRatio:1,plugins:{legend:{labels:{generateLabels(t){const e=t.data;if(e.labels.length&&e.datasets.length){const{labels:{pointStyle:i,color:s}}=t.legend.options;return e.labels.map(((e,n)=>{const o=t.getDatasetMeta(0).controller.getStyle(n);return{text:e,fillStyle:o.backgroundColor,strokeStyle:o.borderColor,fontColor:s,lineWidth:o.borderWidth,pointStyle:i,hidden:!t.getDataVisibility(n),index:n}}))}return[]}},onClick(t,e,i){i.chart.toggleDataVisibility(e.index),i.chart.update()}}}};constructor(t,e){super(t,e),this.enableOptionSharing=!0,this.innerRadius=void 0,this.outerRadius=void 0,this.offsetX=void 0,this.offsetY=void 0}linkScales(){}parse(t,e){const i=this.getDataset().data,s=this._cachedMeta;if(!1===this._parsing)s._parsed=i;else{let n,a,r=t=>+i[t];if(o(i[t])){const{key:t="value"}=this._parsing;r=e=>+M(i[e],t)}for(n=t,a=t+e;nZ(t,r,l,!0)?1:Math.max(e,e*i,s,s*i),g=(t,e,s)=>Z(t,r,l,!0)?-1:Math.min(e,e*i,s,s*i),p=f(0,h,d),m=f(E,c,u),b=g(C,h,d),x=g(C+E,c,u);s=(p-b)/2,n=(m-x)/2,o=-(p+b)/2,a=-(m+x)/2}return{ratioX:s,ratioY:n,offsetX:o,offsetY:a}}(u,d,r),b=(i.width-o)/f,x=(i.height-o)/g,_=Math.max(Math.min(b,x)/2,0),y=c(this.options.radius,_),v=(y-Math.max(y*r,0))/this._getVisibleDatasetWeightTotal();this.offsetX=p*y,this.offsetY=m*y,s.total=this.calculateTotal(),this.outerRadius=y-v*this._getRingWeightOffset(this.index),this.innerRadius=Math.max(this.outerRadius-v*l,0),this.updateElements(n,0,n.length,t)}_circumference(t,e){const i=this.options,s=this._cachedMeta,n=this._getCircumference();return e&&i.animation.animateRotate||!this.chart.getDataVisibility(t)||null===s._parsed[t]||s.data[t].hidden?0:this.calculateCircumference(s._parsed[t]*n/O)}updateElements(t,e,i,s){const n="reset"===s,o=this.chart,a=o.chartArea,r=o.options.animation,l=(a.left+a.right)/2,h=(a.top+a.bottom)/2,c=n&&r.animateScale,d=c?0:this.innerRadius,u=c?0:this.outerRadius,{sharedOptions:f,includeOptions:g}=this._getSharedOptions(e,s);let p,m=this._getRotation();for(p=0;p0&&!isNaN(t)?O*(Math.abs(t)/e):0}getLabelAndValue(t){const e=this._cachedMeta,i=this.chart,s=i.data.labels||[],n=ne(e._parsed[t],i.options.locale);return{label:s[t]||"",value:n}}getMaxBorderWidth(t){let e=0;const i=this.chart;let s,n,o,a,r;if(!t)for(s=0,n=i.data.datasets.length;s{const o=t.getDatasetMeta(0).controller.getStyle(n);return{text:e,fillStyle:o.backgroundColor,strokeStyle:o.borderColor,fontColor:s,lineWidth:o.borderWidth,pointStyle:i,hidden:!t.getDataVisibility(n),index:n}}))}return[]}},onClick(t,e,i){i.chart.toggleDataVisibility(e.index),i.chart.update()}}},scales:{r:{type:"radialLinear",angleLines:{display:!1},beginAtZero:!0,grid:{circular:!0},pointLabels:{display:!1},startAngle:0}}};constructor(t,e){super(t,e),this.innerRadius=void 0,this.outerRadius=void 0}getLabelAndValue(t){const e=this._cachedMeta,i=this.chart,s=i.data.labels||[],n=ne(e._parsed[t].r,i.options.locale);return{label:s[t]||"",value:n}}parseObjectData(t,e,i,s){return ii.bind(this)(t,e,i,s)}update(t){const e=this._cachedMeta.data;this._updateRadius(),this.updateElements(e,0,e.length,t)}getMinMax(){const t=this._cachedMeta,e={min:Number.POSITIVE_INFINITY,max:Number.NEGATIVE_INFINITY};return t.data.forEach(((t,i)=>{const s=this.getParsed(i).r;!isNaN(s)&&this.chart.getDataVisibility(i)&&(se.max&&(e.max=s))})),e}_updateRadius(){const t=this.chart,e=t.chartArea,i=t.options,s=Math.min(e.right-e.left,e.bottom-e.top),n=Math.max(s/2,0),o=(n-Math.max(i.cutoutPercentage?n/100*i.cutoutPercentage:1,0))/t.getVisibleDatasetCount();this.outerRadius=n-o*this.index,this.innerRadius=this.outerRadius-o}updateElements(t,e,i,s){const n="reset"===s,o=this.chart,a=o.options.animation,r=this._cachedMeta.rScale,l=r.xCenter,h=r.yCenter,c=r.getIndexAngle(0)-.5*C;let d,u=c;const f=360/this.countVisibleElements();for(d=0;d{!isNaN(this.getParsed(i).r)&&this.chart.getDataVisibility(i)&&e++})),e}_computeAngle(t,e,i){return this.chart.getDataVisibility(t)?$(this.resolveDataElementOptions(t,e).angle||i):0}}var Yn=Object.freeze({__proto__:null,BarController:class extends Ns{static id="bar";static defaults={datasetElementType:!1,dataElementType:"bar",categoryPercentage:.8,barPercentage:.9,grouped:!0,animations:{numbers:{type:"number",properties:["x","y","base","width","height"]}}};static overrides={scales:{_index_:{type:"category",offset:!0,grid:{offset:!0}},_value_:{type:"linear",beginAtZero:!0}}};parsePrimitiveData(t,e,i,s){return Fn(t,e,i,s)}parseArrayData(t,e,i,s){return Fn(t,e,i,s)}parseObjectData(t,e,i,s){const{iScale:n,vScale:o}=t,{xAxisKey:a="x",yAxisKey:r="y"}=this._parsing,l="x"===n.axis?a:r,h="x"===o.axis?a:r,c=[];let d,u,f,g;for(d=i,u=i+s;dt.controller.options.grouped)),o=i.options.stacked,a=[],r=t=>{const i=t.controller.getParsed(e),n=i&&i[t.vScale.axis];if(s(n)||isNaN(n))return!0};for(const i of n)if((void 0===e||!r(i))&&((!1===o||-1===a.indexOf(i.stack)||void 0===o&&void 0===i.stack)&&a.push(i.stack),i.index===t))break;return a.length||a.push(void 0),a}_getStackCount(t){return this._getStacks(void 0,t).length}_getStackIndex(t,e,i){const s=this._getStacks(t,i),n=void 0!==e?s.indexOf(e):-1;return-1===n?s.length-1:n}_getRuler(){const t=this.options,e=this._cachedMeta,i=e.iScale,s=[];let n,o;for(n=0,o=e.data.length;n=i?1:-1)}(u,e,r)*a,f===r&&(b-=u/2);const t=e.getPixelForDecimal(0),s=e.getPixelForDecimal(1),o=Math.min(t,s),h=Math.max(t,s);b=Math.max(Math.min(b,h),o),d=b+u,i&&!c&&(l._stacks[e.axis]._visualValues[n]=e.getValueForPixel(d)-e.getValueForPixel(b))}if(b===e.getPixelForValue(r)){const t=F(u)*e.getLineWidthForValue(r)/2;b+=t,u-=t}return{size:u,base:b,head:d,center:d+u/2}}_calculateBarIndexPixels(t,e){const i=e.scale,n=this.options,o=n.skipNull,a=l(n.maxBarThickness,1/0);let r,h;if(e.grouped){const i=o?this._getStackCount(t):e.stackCount,l="flex"===n.barThickness?function(t,e,i,s){const n=e.pixels,o=n[t];let a=t>0?n[t-1]:null,r=t=0;--i)e=Math.max(e,t[i].size(this.resolveDataElementOptions(i))/2);return e>0&&e}getLabelAndValue(t){const e=this._cachedMeta,i=this.chart.data.labels||[],{xScale:s,yScale:n}=e,o=this.getParsed(t),a=s.getLabelForValue(o.x),r=n.getLabelForValue(o.y),l=o._custom;return{label:i[t]||"",value:"("+a+", "+r+(l?", "+l:"")+")"}}update(t){const e=this._cachedMeta.data;this.updateElements(e,0,e.length,t)}updateElements(t,e,i,s){const n="reset"===s,{iScale:o,vScale:a}=this._cachedMeta,{sharedOptions:r,includeOptions:l}=this._getSharedOptions(e,s),h=o.axis,c=a.axis;for(let d=e;d0&&this.getParsed(e-1);for(let i=0;i<_;++i){const g=t[i],_=b?g:{};if(i=x){_.skip=!0;continue}const v=this.getParsed(i),M=s(v[f]),w=_[u]=a.getPixelForValue(v[u],i),k=_[f]=o||M?r.getBasePixel():r.getPixelForValue(l?this.applyStack(r,v,l):v[f],i);_.skip=isNaN(w)||isNaN(k)||M,_.stop=i>0&&Math.abs(v[u]-y[u])>m,p&&(_.parsed=v,_.raw=h.data[i]),d&&(_.options=c||this.resolveDataElementOptions(i,g.active?"active":n)),b||this.updateElement(g,i,_,n),y=v}}getMaxOverflow(){const t=this._cachedMeta,e=t.dataset,i=e.options&&e.options.borderWidth||0,s=t.data||[];if(!s.length)return i;const n=s[0].size(this.resolveDataElementOptions(0)),o=s[s.length-1].size(this.resolveDataElementOptions(s.length-1));return Math.max(i,n,o)/2}draw(){const t=this._cachedMeta;t.dataset.updateControlPoints(this.chart.chartArea,t.iScale.axis),super.draw()}},PieController:class extends jn{static id="pie";static defaults={cutout:0,rotation:0,circumference:360,radius:"100%"}},PolarAreaController:$n,RadarController:class extends Ns{static id="radar";static defaults={datasetElementType:"line",dataElementType:"point",indexAxis:"r",showLine:!0,elements:{line:{fill:"start"}}};static overrides={aspectRatio:1,scales:{r:{type:"radialLinear"}}};getLabelAndValue(t){const e=this._cachedMeta.vScale,i=this.getParsed(t);return{label:e.getLabels()[t],value:""+e.getLabelForValue(i[e.axis])}}parseObjectData(t,e,i,s){return ii.bind(this)(t,e,i,s)}update(t){const e=this._cachedMeta,i=e.dataset,s=e.data||[],n=e.iScale.getLabels();if(i.points=s,"resize"!==t){const e=this.resolveDatasetElementOptions(t);this.options.showLine||(e.borderWidth=0);const o={_loop:!0,_fullLoop:n.length===s.length,options:e};this.updateElement(i,void 0,o,t)}this.updateElements(s,0,s.length,t)}updateElements(t,e,i,s){const n=this._cachedMeta.rScale,o="reset"===s;for(let a=e;a0&&this.getParsed(e-1);for(let c=e;c0&&Math.abs(i[f]-_[f])>b,m&&(p.parsed=i,p.raw=h.data[c]),u&&(p.options=d||this.resolveDataElementOptions(c,e.active?"active":n)),x||this.updateElement(e,c,p,n),_=i}this.updateSharedOptions(d,n,c)}getMaxOverflow(){const t=this._cachedMeta,e=t.data||[];if(!this.options.showLine){let t=0;for(let i=e.length-1;i>=0;--i)t=Math.max(t,e[i].size(this.resolveDataElementOptions(i))/2);return t>0&&t}const i=t.dataset,s=i.options&&i.options.borderWidth||0;if(!e.length)return s;const n=e[0].size(this.resolveDataElementOptions(0)),o=e[e.length-1].size(this.resolveDataElementOptions(e.length-1));return Math.max(s,n,o)/2}}});function Un(t,e,i,s){const n=vi(t.options.borderRadius,["outerStart","outerEnd","innerStart","innerEnd"]);const o=(i-e)/2,a=Math.min(o,s*e/2),r=t=>{const e=(i-Math.min(o,t))*s/2;return J(t,0,Math.min(o,e))};return{outerStart:r(n.outerStart),outerEnd:r(n.outerEnd),innerStart:J(n.innerStart,0,a),innerEnd:J(n.innerEnd,0,a)}}function Xn(t,e,i,s){return{x:i+t*Math.cos(e),y:s+t*Math.sin(e)}}function qn(t,e,i,s,n,o){const{x:a,y:r,startAngle:l,pixelMargin:h,innerRadius:c}=e,d=Math.max(e.outerRadius+s+i-h,0),u=c>0?c+s+i+h:0;let f=0;const g=n-l;if(s){const t=((c>0?c-s:0)+(d>0?d-s:0))/2;f=(g-(0!==t?g*t/(t+s):g))/2}const p=(g-Math.max(.001,g*d-i/C)/d)/2,m=l+p+f,b=n-p-f,{outerStart:x,outerEnd:_,innerStart:y,innerEnd:v}=Un(e,u,d,b-m),M=d-x,w=d-_,k=m+x/M,S=b-_/w,P=u+y,D=u+v,O=m+y/P,A=b-v/D;if(t.beginPath(),o){const e=(k+S)/2;if(t.arc(a,r,d,k,e),t.arc(a,r,d,e,S),_>0){const e=Xn(w,S,a,r);t.arc(e.x,e.y,_,S,b+E)}const i=Xn(D,b,a,r);if(t.lineTo(i.x,i.y),v>0){const e=Xn(D,A,a,r);t.arc(e.x,e.y,v,b+E,A+Math.PI)}const s=(b-v/u+(m+y/u))/2;if(t.arc(a,r,u,b-v/u,s,!0),t.arc(a,r,u,s,m+y/u,!0),y>0){const e=Xn(P,O,a,r);t.arc(e.x,e.y,y,O+Math.PI,m-E)}const n=Xn(M,m,a,r);if(t.lineTo(n.x,n.y),x>0){const e=Xn(M,k,a,r);t.arc(e.x,e.y,x,m-E,k)}}else{t.moveTo(a,r);const e=Math.cos(k)*d+a,i=Math.sin(k)*d+r;t.lineTo(e,i);const s=Math.cos(S)*d+a,n=Math.sin(S)*d+r;t.lineTo(s,n)}t.closePath()}function Kn(t,e,i,s,n){const{fullCircles:o,startAngle:a,circumference:r,options:l}=e,{borderWidth:h,borderJoinStyle:c,borderDash:d,borderDashOffset:u}=l,f="inner"===l.borderAlign;if(!h)return;t.setLineDash(d||[]),t.lineDashOffset=u,f?(t.lineWidth=2*h,t.lineJoin=c||"round"):(t.lineWidth=h,t.lineJoin=c||"bevel");let g=e.endAngle;if(o){qn(t,e,i,s,g,n);for(let e=0;en?(h=n/l,t.arc(o,a,l,i+h,s-h,!0)):t.arc(o,a,n,i+E,s-E),t.closePath(),t.clip()}(t,e,g),o||(qn(t,e,i,s,g,n),t.stroke())}function Gn(t,e,i=e){t.lineCap=l(i.borderCapStyle,e.borderCapStyle),t.setLineDash(l(i.borderDash,e.borderDash)),t.lineDashOffset=l(i.borderDashOffset,e.borderDashOffset),t.lineJoin=l(i.borderJoinStyle,e.borderJoinStyle),t.lineWidth=l(i.borderWidth,e.borderWidth),t.strokeStyle=l(i.borderColor,e.borderColor)}function Zn(t,e,i){t.lineTo(i.x,i.y)}function Jn(t,e,i={}){const s=t.length,{start:n=0,end:o=s-1}=i,{start:a,end:r}=e,l=Math.max(n,a),h=Math.min(o,r),c=nr&&o>r;return{count:s,start:l,loop:e.loop,ilen:h(a+(h?r-t:t))%o,_=()=>{f!==g&&(t.lineTo(m,g),t.lineTo(m,f),t.lineTo(m,p))};for(l&&(d=n[x(0)],t.moveTo(d.x,d.y)),c=0;c<=r;++c){if(d=n[x(c)],d.skip)continue;const e=d.x,i=d.y,s=0|e;s===u?(ig&&(g=i),m=(b*m+e)/++b):(_(),t.lineTo(e,i),u=s,b=0,f=g=i),p=i}_()}function eo(t){const e=t.options,i=e.borderDash&&e.borderDash.length;return!(t._decimated||t._loop||e.tension||"monotone"===e.cubicInterpolationMode||e.stepped||i)?to:Qn}const io="function"==typeof Path2D;function so(t,e,i,s){io&&!e.options.segment?function(t,e,i,s){let n=e._path;n||(n=e._path=new Path2D,e.path(n,i,s)&&n.closePath()),Gn(t,e.options),t.stroke(n)}(t,e,i,s):function(t,e,i,s){const{segments:n,options:o}=e,a=eo(e);for(const r of n)Gn(t,o,r.style),t.beginPath(),a(t,e,r,{start:i,end:i+s-1})&&t.closePath(),t.stroke()}(t,e,i,s)}class no extends Hs{static id="line";static defaults={borderCapStyle:"butt",borderDash:[],borderDashOffset:0,borderJoinStyle:"miter",borderWidth:3,capBezierPoints:!0,cubicInterpolationMode:"default",fill:!1,spanGaps:!1,stepped:!1,tension:0};static defaultRoutes={backgroundColor:"backgroundColor",borderColor:"borderColor"};static descriptors={_scriptable:!0,_indexable:t=>"borderDash"!==t&&"fill"!==t};constructor(t){super(),this.animated=!0,this.options=void 0,this._chart=void 0,this._loop=void 0,this._fullLoop=void 0,this._path=void 0,this._points=void 0,this._segments=void 0,this._decimated=!1,this._pointsUpdated=!1,this._datasetIndex=void 0,t&&Object.assign(this,t)}updateControlPoints(t,e){const i=this.options;if((i.tension||"monotone"===i.cubicInterpolationMode)&&!i.stepped&&!this._pointsUpdated){const s=i.spanGaps?this._loop:this._fullLoop;hi(this._points,i,t,s,e),this._pointsUpdated=!0}}set points(t){this._points=t,delete this._segments,delete this._path,this._pointsUpdated=!1}get points(){return this._points}get segments(){return this._segments||(this._segments=zi(this,this.options.segment))}first(){const t=this.segments,e=this.points;return t.length&&e[t[0].start]}last(){const t=this.segments,e=this.points,i=t.length;return i&&e[t[i-1].end]}interpolate(t,e){const i=this.options,s=t[e],n=this.points,o=Ii(this,{property:e,start:s,end:s});if(!o.length)return;const a=[],r=function(t){return t.stepped?pi:t.tension||"monotone"===t.cubicInterpolationMode?mi:gi}(i);let l,h;for(l=0,h=o.length;l"borderDash"!==t};circumference;endAngle;fullCircles;innerRadius;outerRadius;pixelMargin;startAngle;constructor(t){super(),this.options=void 0,this.circumference=void 0,this.startAngle=void 0,this.endAngle=void 0,this.innerRadius=void 0,this.outerRadius=void 0,this.pixelMargin=0,this.fullCircles=0,t&&Object.assign(this,t)}inRange(t,e,i){const s=this.getProps(["x","y"],i),{angle:n,distance:o}=X(s,{x:t,y:e}),{startAngle:a,endAngle:r,innerRadius:h,outerRadius:c,circumference:d}=this.getProps(["startAngle","endAngle","innerRadius","outerRadius","circumference"],i),u=(this.options.spacing+this.options.borderWidth)/2,f=l(d,r-a)>=O||Z(n,a,r),g=tt(o,h+u,c+u);return f&&g}getCenterPoint(t){const{x:e,y:i,startAngle:s,endAngle:n,innerRadius:o,outerRadius:a}=this.getProps(["x","y","startAngle","endAngle","innerRadius","outerRadius"],t),{offset:r,spacing:l}=this.options,h=(s+n)/2,c=(o+a+l+r)/2;return{x:e+Math.cos(h)*c,y:i+Math.sin(h)*c}}tooltipPosition(t){return this.getCenterPoint(t)}draw(t){const{options:e,circumference:i}=this,s=(e.offset||0)/4,n=(e.spacing||0)/2,o=e.circular;if(this.pixelMargin="inner"===e.borderAlign?.33:0,this.fullCircles=i>O?Math.floor(i/O):0,0===i||this.innerRadius<0||this.outerRadius<0)return;t.save();const a=(this.startAngle+this.endAngle)/2;t.translate(Math.cos(a)*s,Math.sin(a)*s);const r=s*(1-Math.sin(Math.min(C,i||0)));t.fillStyle=e.backgroundColor,t.strokeStyle=e.borderColor,function(t,e,i,s,n){const{fullCircles:o,startAngle:a,circumference:r}=e;let l=e.endAngle;if(o){qn(t,e,i,s,l,n);for(let e=0;e("string"==typeof e?(i=t.push(e)-1,s.unshift({index:i,label:e})):isNaN(e)&&(i=null),i))(t,e,i,s);return n!==t.lastIndexOf(e)?i:n}function po(t){const e=this.getLabels();return t>=0&&ts=e?s:t,a=t=>n=i?n:t;if(t){const t=F(s),e=F(n);t<0&&e<0?a(0):t>0&&e>0&&o(0)}if(s===n){let e=0===n?1:Math.abs(.05*n);a(n+e),t||o(s-e)}this.min=s,this.max=n}getTickLimit(){const t=this.options.ticks;let e,{maxTicksLimit:i,stepSize:s}=t;return s?(e=Math.ceil(this.max/s)-Math.floor(this.min/s)+1,e>1e3&&(console.warn(`scales.${this.id}.ticks.stepSize: ${s} would result generating up to ${e} ticks. Limiting to 1000.`),e=1e3)):(e=this.computeTickLimit(),i=i||11),i&&(e=Math.min(i,e)),e}computeTickLimit(){return Number.POSITIVE_INFINITY}buildTicks(){const t=this.options,e=t.ticks;let i=this.getTickLimit();i=Math.max(2,i);const n=function(t,e){const i=[],{bounds:n,step:o,min:a,max:r,precision:l,count:h,maxTicks:c,maxDigits:d,includeBounds:u}=t,f=o||1,g=c-1,{min:p,max:m}=e,b=!s(a),x=!s(r),_=!s(h),y=(m-p)/(d+1);let v,M,w,k,S=B((m-p)/g/f)*f;if(S<1e-14&&!b&&!x)return[{value:p},{value:m}];k=Math.ceil(m/S)-Math.floor(p/S),k>g&&(S=B(k*S/g/f)*f),s(l)||(v=Math.pow(10,l),S=Math.ceil(S*v)/v),"ticks"===n?(M=Math.floor(p/S)*S,w=Math.ceil(m/S)*S):(M=p,w=m),b&&x&&o&&H((r-a)/o,S/1e3)?(k=Math.round(Math.min((r-a)/S,c)),S=(r-a)/k,M=a,w=r):_?(M=b?a:M,w=x?r:w,k=h-1,S=(w-M)/k):(k=(w-M)/S,k=V(k,Math.round(k),S/1e3)?Math.round(k):Math.ceil(k));const P=Math.max(U(S),U(M));v=Math.pow(10,s(l)?P:l),M=Math.round(M*v)/v,w=Math.round(w*v)/v;let D=0;for(b&&(u&&M!==a?(i.push({value:a}),Mr)break;i.push({value:t})}return x&&u&&w!==r?i.length&&V(i[i.length-1].value,r,mo(r,y,t))?i[i.length-1].value=r:i.push({value:r}):x&&w!==r||i.push({value:w}),i}({maxTicks:i,bounds:t.bounds,min:t.min,max:t.max,precision:e.precision,step:e.stepSize,count:e.count,maxDigits:this._maxDigits(),horizontal:this.isHorizontal(),minRotation:e.minRotation||0,includeBounds:!1!==e.includeBounds},this._range||this);return"ticks"===t.bounds&&j(n,this,"value"),t.reverse?(n.reverse(),this.start=this.max,this.end=this.min):(this.start=this.min,this.end=this.max),n}configure(){const t=this.ticks;let e=this.min,i=this.max;if(super.configure(),this.options.offset&&t.length){const s=(i-e)/Math.max(t.length-1,1)/2;e-=s,i+=s}this._startValue=e,this._endValue=i,this._valueRange=i-e}getLabelForValue(t){return ne(t,this.chart.options.locale,this.options.ticks.format)}}class xo extends bo{static id="linear";static defaults={ticks:{callback:ae.formatters.numeric}};determineDataLimits(){const{min:t,max:e}=this.getMinMax(!0);this.min=a(t)?t:0,this.max=a(e)?e:1,this.handleTickRangeOptions()}computeTickLimit(){const t=this.isHorizontal(),e=t?this.width:this.height,i=$(this.options.ticks.minRotation),s=(t?Math.sin(i):Math.cos(i))||.001,n=this._resolveTickFontOptions(0);return Math.ceil(e/Math.min(40,n.lineHeight/s))}getPixelForValue(t){return null===t?NaN:this.getPixelForDecimal((t-this._startValue)/this._valueRange)}getValueForPixel(t){return this._startValue+this.getDecimalForPixel(t)*this._valueRange}}const _o=t=>Math.floor(z(t)),yo=(t,e)=>Math.pow(10,_o(t)+e);function vo(t){return 1===t/Math.pow(10,_o(t))}function Mo(t,e,i){const s=Math.pow(10,i),n=Math.floor(t/s);return Math.ceil(e/s)-n}function wo(t,{min:e,max:i}){e=r(t.min,e);const s=[],n=_o(e);let o=function(t,e){let i=_o(e-t);for(;Mo(t,e,i)>10;)i++;for(;Mo(t,e,i)<10;)i--;return Math.min(i,_o(t))}(e,i),a=o<0?Math.pow(10,Math.abs(o)):1;const l=Math.pow(10,o),h=n>o?Math.pow(10,n):0,c=Math.round((e-h)*a)/a,d=Math.floor((e-h)/l/10)*l*10;let u=Math.floor((c-d)/Math.pow(10,o)),f=r(t.min,Math.round((h+d+u*Math.pow(10,o))*a)/a);for(;f=10?u=u<15?15:20:u++,u>=20&&(o++,u=2,a=o>=0?1:a),f=Math.round((h+d+u*Math.pow(10,o))*a)/a;const g=r(t.max,f);return s.push({value:g,major:vo(g),significand:u}),s}class ko extends Js{static id="logarithmic";static defaults={ticks:{callback:ae.formatters.logarithmic,major:{enabled:!0}}};constructor(t){super(t),this.start=void 0,this.end=void 0,this._startValue=void 0,this._valueRange=0}parse(t,e){const i=bo.prototype.parse.apply(this,[t,e]);if(0!==i)return a(i)&&i>0?i:null;this._zero=!0}determineDataLimits(){const{min:t,max:e}=this.getMinMax(!0);this.min=a(t)?Math.max(0,t):null,this.max=a(e)?Math.max(0,e):null,this.options.beginAtZero&&(this._zero=!0),this._zero&&this.min!==this._suggestedMin&&!a(this._userMin)&&(this.min=t===yo(this.min,0)?yo(this.min,-1):yo(this.min,0)),this.handleTickRangeOptions()}handleTickRangeOptions(){const{minDefined:t,maxDefined:e}=this.getUserBounds();let i=this.min,s=this.max;const n=e=>i=t?i:e,o=t=>s=e?s:t;i===s&&(i<=0?(n(1),o(10)):(n(yo(i,-1)),o(yo(s,1)))),i<=0&&n(yo(s,-1)),s<=0&&o(yo(i,1)),this.min=i,this.max=s}buildTicks(){const t=this.options,e=wo({min:this._userMin,max:this._userMax},this);return"ticks"===t.bounds&&j(e,this,"value"),t.reverse?(e.reverse(),this.start=this.max,this.end=this.min):(this.start=this.min,this.end=this.max),e}getLabelForValue(t){return void 0===t?"0":ne(t,this.chart.options.locale,this.options.ticks.format)}configure(){const t=this.min;super.configure(),this._startValue=z(t),this._valueRange=z(this.max)-z(t)}getPixelForValue(t){return void 0!==t&&0!==t||(t=this.min),null===t||isNaN(t)?NaN:this.getPixelForDecimal(t===this.min?0:(z(t)-this._startValue)/this._valueRange)}getValueForPixel(t){const e=this.getDecimalForPixel(t);return Math.pow(10,this._startValue+e*this._valueRange)}}function So(t){const e=t.ticks;if(e.display&&t.display){const t=ki(e.backdropPadding);return l(e.font&&e.font.size,ue.font.size)+t.height}return 0}function Po(t,e,i,s,n){return t===s||t===n?{start:e-i/2,end:e+i/2}:tn?{start:e-i,end:e}:{start:e,end:e+i}}function Do(t){const e={l:t.left+t._padding.left,r:t.right-t._padding.right,t:t.top+t._padding.top,b:t.bottom-t._padding.bottom},i=Object.assign({},e),s=[],o=[],a=t._pointLabels.length,r=t.options.pointLabels,l=r.centerPointLabels?C/a:0;for(let u=0;ue.r&&(r=(s.end-e.r)/o,t.r=Math.max(t.r,e.r+r)),n.starte.b&&(l=(n.end-e.b)/a,t.b=Math.max(t.b,e.b+l))}function Oo(t,e,i){const s=t.drawingArea,{extra:n,additionalAngle:o,padding:a,size:r}=i,l=t.getPointPosition(e,s+n+a,o),h=Math.round(Y(G(l.angle+E))),c=function(t,e,i){90===i||270===i?t-=e/2:(i>270||i<90)&&(t-=e);return t}(l.y,r.h,h),d=function(t){if(0===t||180===t)return"center";if(t<180)return"left";return"right"}(h),u=function(t,e,i){"right"===i?t-=e:"center"===i&&(t-=e/2);return t}(l.x,r.w,d);return{visible:!0,x:l.x,y:c,textAlign:d,left:u,top:c,right:u+r.w,bottom:c+r.h}}function Ao(t,e){if(!e)return!0;const{left:i,top:s,right:n,bottom:o}=t;return!(Re({x:i,y:s},e)||Re({x:i,y:o},e)||Re({x:n,y:s},e)||Re({x:n,y:o},e))}function To(t,e,i){const{left:n,top:o,right:a,bottom:r}=i,{backdropColor:l}=e;if(!s(l)){const i=wi(e.borderRadius),s=ki(e.backdropPadding);t.fillStyle=l;const h=n-s.left,c=o-s.top,d=a-n+s.width,u=r-o+s.height;Object.values(i).some((t=>0!==t))?(t.beginPath(),He(t,{x:h,y:c,w:d,h:u,radius:i}),t.fill()):t.fillRect(h,c,d,u)}}function Lo(t,e,i,s){const{ctx:n}=t;if(i)n.arc(t.xCenter,t.yCenter,e,0,O);else{let i=t.getPointPosition(0,e);n.moveTo(i.x,i.y);for(let o=1;ot,padding:5,centerPointLabels:!1}};static defaultRoutes={"angleLines.color":"borderColor","pointLabels.color":"color","ticks.color":"color"};static descriptors={angleLines:{_fallback:"grid"}};constructor(t){super(t),this.xCenter=void 0,this.yCenter=void 0,this.drawingArea=void 0,this._pointLabels=[],this._pointLabelItems=[]}setDimensions(){const t=this._padding=ki(So(this.options)/2),e=this.width=this.maxWidth-t.width,i=this.height=this.maxHeight-t.height;this.xCenter=Math.floor(this.left+e/2+t.left),this.yCenter=Math.floor(this.top+i/2+t.top),this.drawingArea=Math.floor(Math.min(e,i)/2)}determineDataLimits(){const{min:t,max:e}=this.getMinMax(!1);this.min=a(t)&&!isNaN(t)?t:0,this.max=a(e)&&!isNaN(e)?e:0,this.handleTickRangeOptions()}computeTickLimit(){return Math.ceil(this.drawingArea/So(this.options))}generateTickLabels(t){bo.prototype.generateTickLabels.call(this,t),this._pointLabels=this.getLabels().map(((t,e)=>{const i=d(this.options.pointLabels.callback,[t,e],this);return i||0===i?i:""})).filter(((t,e)=>this.chart.getDataVisibility(e)))}fit(){const t=this.options;t.display&&t.pointLabels.display?Do(this):this.setCenterPoint(0,0,0,0)}setCenterPoint(t,e,i,s){this.xCenter+=Math.floor((t-e)/2),this.yCenter+=Math.floor((i-s)/2),this.drawingArea-=Math.min(this.drawingArea/2,Math.max(t,e,i,s))}getIndexAngle(t){return G(t*(O/(this._pointLabels.length||1))+$(this.options.startAngle||0))}getDistanceFromCenterForValue(t){if(s(t))return NaN;const e=this.drawingArea/(this.max-this.min);return this.options.reverse?(this.max-t)*e:(t-this.min)*e}getValueForDistanceFromCenter(t){if(s(t))return NaN;const e=t/(this.drawingArea/(this.max-this.min));return this.options.reverse?this.max-e:this.min+e}getPointLabelContext(t){const e=this._pointLabels||[];if(t>=0&&t=0;n--){const e=t._pointLabelItems[n];if(!e.visible)continue;const o=s.setContext(t.getPointLabelContext(n));To(i,o,e);const a=Si(o.font),{x:r,y:l,textAlign:h}=e;Ne(i,t._pointLabels[n],r,l+a.lineHeight/2,a,{color:o.color,textAlign:h,textBaseline:"middle"})}}(this,o),s.display&&this.ticks.forEach(((t,e)=>{if(0!==e){r=this.getDistanceFromCenterForValue(t.value);const i=this.getContext(e),a=s.setContext(i),l=n.setContext(i);!function(t,e,i,s,n){const o=t.ctx,a=e.circular,{color:r,lineWidth:l}=e;!a&&!s||!r||!l||i<0||(o.save(),o.strokeStyle=r,o.lineWidth=l,o.setLineDash(n.dash),o.lineDashOffset=n.dashOffset,o.beginPath(),Lo(t,i,a,s),o.closePath(),o.stroke(),o.restore())}(this,a,r,o,l)}})),i.display){for(t.save(),a=o-1;a>=0;a--){const s=i.setContext(this.getPointLabelContext(a)),{color:n,lineWidth:o}=s;o&&n&&(t.lineWidth=o,t.strokeStyle=n,t.setLineDash(s.borderDash),t.lineDashOffset=s.borderDashOffset,r=this.getDistanceFromCenterForValue(e.ticks.reverse?this.min:this.max),l=this.getPointPosition(a,r),t.beginPath(),t.moveTo(this.xCenter,this.yCenter),t.lineTo(l.x,l.y),t.stroke())}t.restore()}}drawBorder(){}drawLabels(){const t=this.ctx,e=this.options,i=e.ticks;if(!i.display)return;const s=this.getIndexAngle(0);let n,o;t.save(),t.translate(this.xCenter,this.yCenter),t.rotate(s),t.textAlign="center",t.textBaseline="middle",this.ticks.forEach(((s,a)=>{if(0===a&&!e.reverse)return;const r=i.setContext(this.getContext(a)),l=Si(r.font);if(n=this.getDistanceFromCenterForValue(this.ticks[a].value),r.showLabelBackdrop){t.font=l.string,o=t.measureText(s.label).width,t.fillStyle=r.backdropColor;const e=ki(r.backdropPadding);t.fillRect(-o/2-e.left,-n-l.size/2-e.top,o+e.width,l.size+e.height)}Ne(t,s.label,0,-n,l,{color:r.color,strokeColor:r.textStrokeColor,strokeWidth:r.textStrokeWidth})})),t.restore()}drawTitle(){}}const Ro={millisecond:{common:!0,size:1,steps:1e3},second:{common:!0,size:1e3,steps:60},minute:{common:!0,size:6e4,steps:60},hour:{common:!0,size:36e5,steps:24},day:{common:!0,size:864e5,steps:30},week:{common:!1,size:6048e5,steps:4},month:{common:!0,size:2628e6,steps:12},quarter:{common:!1,size:7884e6,steps:4},year:{common:!0,size:3154e7}},Io=Object.keys(Ro);function zo(t,e){return t-e}function Fo(t,e){if(s(e))return null;const i=t._adapter,{parser:n,round:o,isoWeekday:r}=t._parseOpts;let l=e;return"function"==typeof n&&(l=n(l)),a(l)||(l="string"==typeof n?i.parse(l,n):i.parse(l)),null===l?null:(o&&(l="week"!==o||!N(r)&&!0!==r?i.startOf(l,o):i.startOf(l,"isoWeek",r)),+l)}function Vo(t,e,i,s){const n=Io.length;for(let o=Io.indexOf(t);o=e?i[s]:i[n]]=!0}}else t[e]=!0}function Wo(t,e,i){const s=[],n={},o=e.length;let a,r;for(a=0;a=0&&(e[l].major=!0);return e}(t,s,n,i):s}class No extends Js{static id="time";static defaults={bounds:"data",adapters:{},time:{parser:!1,unit:!1,round:!1,isoWeekday:!1,minUnit:"millisecond",displayFormats:{}},ticks:{source:"auto",callback:!1,major:{enabled:!1}}};constructor(t){super(t),this._cache={data:[],labels:[],all:[]},this._unit="day",this._majorUnit=void 0,this._offsets={},this._normalized=!1,this._parseOpts=void 0}init(t,e={}){const i=t.time||(t.time={}),s=this._adapter=new Rn._date(t.adapters.date);s.init(e),x(i.displayFormats,s.formats()),this._parseOpts={parser:i.parser,round:i.round,isoWeekday:i.isoWeekday},super.init(t),this._normalized=e.normalized}parse(t,e){return void 0===t?null:Fo(this,t)}beforeLayout(){super.beforeLayout(),this._cache={data:[],labels:[],all:[]}}determineDataLimits(){const t=this.options,e=this._adapter,i=t.time.unit||"day";let{min:s,max:n,minDefined:o,maxDefined:r}=this.getUserBounds();function l(t){o||isNaN(t.min)||(s=Math.min(s,t.min)),r||isNaN(t.max)||(n=Math.max(n,t.max))}o&&r||(l(this._getLabelBounds()),"ticks"===t.bounds&&"labels"===t.ticks.source||l(this.getMinMax(!1))),s=a(s)&&!isNaN(s)?s:+e.startOf(Date.now(),i),n=a(n)&&!isNaN(n)?n:+e.endOf(Date.now(),i)+1,this.min=Math.min(s,n-1),this.max=Math.max(s+1,n)}_getLabelBounds(){const t=this.getLabelTimestamps();let e=Number.POSITIVE_INFINITY,i=Number.NEGATIVE_INFINITY;return t.length&&(e=t[0],i=t[t.length-1]),{min:e,max:i}}buildTicks(){const t=this.options,e=t.time,i=t.ticks,s="labels"===i.source?this.getLabelTimestamps():this._generate();"ticks"===t.bounds&&s.length&&(this.min=this._userMin||s[0],this.max=this._userMax||s[s.length-1]);const n=this.min,o=nt(s,n,this.max);return this._unit=e.unit||(i.autoSkip?Vo(e.minUnit,this.min,this.max,this._getLabelCapacity(n)):function(t,e,i,s,n){for(let o=Io.length-1;o>=Io.indexOf(i);o--){const i=Io[o];if(Ro[i].common&&t._adapter.diff(n,s,i)>=e-1)return i}return Io[i?Io.indexOf(i):0]}(this,o.length,e.minUnit,this.min,this.max)),this._majorUnit=i.major.enabled&&"year"!==this._unit?function(t){for(let e=Io.indexOf(t)+1,i=Io.length;e+t.value)))}initOffsets(t=[]){let e,i,s=0,n=0;this.options.offset&&t.length&&(e=this.getDecimalForValue(t[0]),s=1===t.length?1-e:(this.getDecimalForValue(t[1])-e)/2,i=this.getDecimalForValue(t[t.length-1]),n=1===t.length?i:(i-this.getDecimalForValue(t[t.length-2]))/2);const o=t.length<3?.5:.25;s=J(s,0,o),n=J(n,0,o),this._offsets={start:s,end:n,factor:1/(s+1+n)}}_generate(){const t=this._adapter,e=this.min,i=this.max,s=this.options,n=s.time,o=n.unit||Vo(n.minUnit,e,i,this._getLabelCapacity(e)),a=l(s.ticks.stepSize,1),r="week"===o&&n.isoWeekday,h=N(r)||!0===r,c={};let d,u,f=e;if(h&&(f=+t.startOf(f,"isoWeek",r)),f=+t.startOf(f,h?"day":o),t.diff(i,e,o)>1e5*a)throw new Error(e+" and "+i+" are too far apart with stepSize of "+a+" "+o);const g="data"===s.ticks.source&&this.getDataTimestamps();for(d=f,u=0;d+t))}getLabelForValue(t){const e=this._adapter,i=this.options.time;return i.tooltipFormat?e.format(t,i.tooltipFormat):e.format(t,i.displayFormats.datetime)}format(t,e){const i=this.options.time.displayFormats,s=this._unit,n=e||i[s];return this._adapter.format(t,n)}_tickFormatFunction(t,e,i,s){const n=this.options,o=n.ticks.callback;if(o)return d(o,[t,e,i],this);const a=n.time.displayFormats,r=this._unit,l=this._majorUnit,h=r&&a[r],c=l&&a[l],u=i[e],f=l&&c&&u&&u.major;return this._adapter.format(t,s||(f?c:h))}generateTickLabels(t){let e,i,s;for(e=0,i=t.length;e0?a:1}getDataTimestamps(){let t,e,i=this._cache.data||[];if(i.length)return i;const s=this.getMatchingVisibleMetas();if(this._normalized&&s.length)return this._cache.data=s[0].controller.getAllParsedValues(this);for(t=0,e=s.length;t=t[r].pos&&e<=t[l].pos&&({lo:r,hi:l}=it(t,"pos",e)),({pos:s,time:o}=t[r]),({pos:n,time:a}=t[l])):(e>=t[r].time&&e<=t[l].time&&({lo:r,hi:l}=it(t,"time",e)),({time:s,pos:o}=t[r]),({time:n,pos:a}=t[l]));const h=n-s;return h?o+(a-o)*(e-s)/h:o}var jo=Object.freeze({__proto__:null,CategoryScale:class extends Js{static id="category";static defaults={ticks:{callback:po}};constructor(t){super(t),this._startValue=void 0,this._valueRange=0,this._addedLabels=[]}init(t){const e=this._addedLabels;if(e.length){const t=this.getLabels();for(const{index:i,label:s}of e)t[i]===s&&t.splice(i,1);this._addedLabels=[]}super.init(t)}parse(t,e){if(s(t))return null;const i=this.getLabels();return((t,e)=>null===t?null:J(Math.round(t),0,e))(e=isFinite(e)&&i[e]===t?e:go(i,t,l(e,t),this._addedLabels),i.length-1)}determineDataLimits(){const{minDefined:t,maxDefined:e}=this.getUserBounds();let{min:i,max:s}=this.getMinMax(!0);"ticks"===this.options.bounds&&(t||(i=0),e||(s=this.getLabels().length-1)),this.min=i,this.max=s}buildTicks(){const t=this.min,e=this.max,i=this.options.offset,s=[];let n=this.getLabels();n=0===t&&e===n.length-1?n:n.slice(t,e+1),this._valueRange=Math.max(n.length-(i?0:1),1),this._startValue=this.min-(i?.5:0);for(let i=t;i<=e;i++)s.push({value:i});return s}getLabelForValue(t){return po.call(this,t)}configure(){super.configure(),this.isHorizontal()||(this._reversePixels=!this._reversePixels)}getPixelForValue(t){return"number"!=typeof t&&(t=this.parse(t)),null===t?NaN:this.getPixelForDecimal((t-this._startValue)/this._valueRange)}getPixelForTick(t){const e=this.ticks;return t<0||t>e.length-1?null:this.getPixelForValue(e[t].value)}getValueForPixel(t){return Math.round(this._startValue+this.getDecimalForPixel(t)*this._valueRange)}getBasePixel(){return this.bottom}},LinearScale:xo,LogarithmicScale:ko,RadialLinearScale:Eo,TimeScale:No,TimeSeriesScale:class extends No{static id="timeseries";static defaults=No.defaults;constructor(t){super(t),this._table=[],this._minPos=void 0,this._tableRange=void 0}initOffsets(){const t=this._getTimestampsForTable(),e=this._table=this.buildLookupTable(t);this._minPos=Ho(e,this.min),this._tableRange=Ho(e,this.max)-this._minPos,super.initOffsets(t)}buildLookupTable(t){const{min:e,max:i}=this,s=[],n=[];let o,a,r,l,h;for(o=0,a=t.length;o=e&&l<=i&&s.push(l);if(s.length<2)return[{time:e,pos:0},{time:i,pos:1}];for(o=0,a=s.length;ot-e))}_getTimestampsForTable(){let t=this._cache.all||[];if(t.length)return t;const e=this.getDataTimestamps(),i=this.getLabelTimestamps();return t=e.length&&i.length?this.normalize(e.concat(i)):e.length?e:i,t=this._cache.all=t,t}getDecimalForValue(t){return(Ho(this._table,t)-this._minPos)/this._tableRange}getValueForPixel(t){const e=this._offsets,i=this.getDecimalForPixel(t)/e.factor-e.end;return Ho(this._table,i*this._tableRange+this._minPos,!0)}}});const $o=["rgb(54, 162, 235)","rgb(255, 99, 132)","rgb(255, 159, 64)","rgb(255, 205, 86)","rgb(75, 192, 192)","rgb(153, 102, 255)","rgb(201, 203, 207)"],Yo=$o.map((t=>t.replace("rgb(","rgba(").replace(")",", 0.5)")));function Uo(t){return $o[t%$o.length]}function Xo(t){return Yo[t%Yo.length]}function qo(t){let e=0;return(i,s)=>{const n=t.getDatasetMeta(s).controller;n instanceof jn?e=function(t,e){return t.backgroundColor=t.data.map((()=>Uo(e++))),e}(i,e):n instanceof $n?e=function(t,e){return t.backgroundColor=t.data.map((()=>Xo(e++))),e}(i,e):n&&(e=function(t,e){return t.borderColor=Uo(e),t.backgroundColor=Xo(e),++e}(i,e))}}function Ko(t){let e;for(e in t)if(t[e].borderColor||t[e].backgroundColor)return!0;return!1}var Go={id:"colors",defaults:{enabled:!0,forceOverride:!1},beforeLayout(t,e,i){if(!i.enabled)return;const{data:{datasets:s},options:n}=t.config,{elements:o}=n;if(!i.forceOverride&&(Ko(s)||(a=n)&&(a.borderColor||a.backgroundColor)||o&&Ko(o)))return;var a;const r=qo(t);s.forEach(r)}};function Zo(t){if(t._decimated){const e=t._data;delete t._decimated,delete t._data,Object.defineProperty(t,"data",{configurable:!0,enumerable:!0,writable:!0,value:e})}}function Jo(t){t.data.datasets.forEach((t=>{Zo(t)}))}var Qo={id:"decimation",defaults:{algorithm:"min-max",enabled:!1},beforeElementsUpdate:(t,e,i)=>{if(!i.enabled)return void Jo(t);const n=t.width;t.data.datasets.forEach(((e,o)=>{const{_data:a,indexAxis:r}=e,l=t.getDatasetMeta(o),h=a||e.data;if("y"===Pi([r,t.options.indexAxis]))return;if(!l.controller.supportsDecimation)return;const c=t.scales[l.xAxisID];if("linear"!==c.type&&"time"!==c.type)return;if(t.options.parsing)return;let{start:d,count:u}=function(t,e){const i=e.length;let s,n=0;const{iScale:o}=t,{min:a,max:r,minDefined:l,maxDefined:h}=o.getUserBounds();return l&&(n=J(it(e,o.axis,a).lo,0,i-1)),s=h?J(it(e,o.axis,r).hi+1,n,i)-n:i-n,{start:n,count:s}}(l,h);if(u<=(i.threshold||4*n))return void Zo(e);let f;switch(s(a)&&(e._data=h,delete e.data,Object.defineProperty(e,"data",{configurable:!0,enumerable:!0,get:function(){return this._decimated},set:function(t){this._data=t}})),i.algorithm){case"lttb":f=function(t,e,i,s,n){const o=n.samples||s;if(o>=i)return t.slice(e,e+i);const a=[],r=(i-2)/(o-2);let l=0;const h=e+i-1;let c,d,u,f,g,p=e;for(a[l++]=t[p],c=0;cu&&(u=f,d=t[s],g=s);a[l++]=d,p=g}return a[l++]=t[h],a}(h,d,u,n,i);break;case"min-max":f=function(t,e,i,n){let o,a,r,l,h,c,d,u,f,g,p=0,m=0;const b=[],x=e+i-1,_=t[e].x,y=t[x].x-_;for(o=e;og&&(g=l,d=o),p=(m*p+a.x)/++m;else{const i=o-1;if(!s(c)&&!s(d)){const e=Math.min(c,d),s=Math.max(c,d);e!==u&&e!==i&&b.push({...t[e],x:p}),s!==u&&s!==i&&b.push({...t[s],x:p})}o>0&&i!==u&&b.push(t[i]),b.push(a),h=e,m=0,f=g=l,c=d=u=o}}return b}(h,d,u,n);break;default:throw new Error(`Unsupported decimation algorithm '${i.algorithm}'`)}e._decimated=f}))},destroy(t){Jo(t)}};function ta(t,e,i,s){if(s)return;let n=e[t],o=i[t];return"angle"===t&&(n=G(n),o=G(o)),{property:t,start:n,end:o}}function ea(t,e,i){for(;e>t;e--){const t=i[e];if(!isNaN(t.x)&&!isNaN(t.y))break}return e}function ia(t,e,i,s){return t&&e?s(t[i],e[i]):t?t[i]:e?e[i]:0}function sa(t,e){let i=[],s=!1;return n(t)?(s=!0,i=t):i=function(t,e){const{x:i=null,y:s=null}=t||{},n=e.points,o=[];return e.segments.forEach((({start:t,end:e})=>{e=ea(t,e,n);const a=n[t],r=n[e];null!==s?(o.push({x:a.x,y:s}),o.push({x:r.x,y:s})):null!==i&&(o.push({x:i,y:a.y}),o.push({x:i,y:r.y}))})),o}(t,e),i.length?new no({points:i,options:{tension:0},_loop:s,_fullLoop:s}):null}function na(t){return t&&!1!==t.fill}function oa(t,e,i){let s=t[e].fill;const n=[e];let o;if(!i)return s;for(;!1!==s&&-1===n.indexOf(s);){if(!a(s))return s;if(o=t[s],!o)return!1;if(o.visible)return s;n.push(s),s=o.fill}return!1}function aa(t,e,i){const s=function(t){const e=t.options,i=e.fill;let s=l(i&&i.target,i);void 0===s&&(s=!!e.backgroundColor);if(!1===s||null===s)return!1;if(!0===s)return"origin";return s}(t);if(o(s))return!isNaN(s.value)&&s;let n=parseFloat(s);return a(n)&&Math.floor(n)===n?function(t,e,i,s){"-"!==t&&"+"!==t||(i=e+i);if(i===e||i<0||i>=s)return!1;return i}(s[0],e,n,i):["origin","start","end","stack","shape"].indexOf(s)>=0&&s}function ra(t,e,i){const s=[];for(let n=0;n=0;--e){const i=n[e].$filler;i&&(i.line.updateControlPoints(o,i.axis),s&&i.fill&&da(t.ctx,i,o))}},beforeDatasetsDraw(t,e,i){if("beforeDatasetsDraw"!==i.drawTime)return;const s=t.getSortedVisibleDatasetMetas();for(let e=s.length-1;e>=0;--e){const i=s[e].$filler;na(i)&&da(t.ctx,i,t.chartArea)}},beforeDatasetDraw(t,e,i){const s=e.meta.$filler;na(s)&&"beforeDatasetDraw"===i.drawTime&&da(t.ctx,s,t.chartArea)},defaults:{propagate:!0,drawTime:"beforeDatasetDraw"}};const ba=(t,e)=>{let{boxHeight:i=e,boxWidth:s=e}=t;return t.usePointStyle&&(i=Math.min(i,e),s=t.pointStyleWidth||Math.min(s,e)),{boxWidth:s,boxHeight:i,itemHeight:Math.max(e,i)}};class xa extends Hs{constructor(t){super(),this._added=!1,this.legendHitBoxes=[],this._hoveredItem=null,this.doughnutMode=!1,this.chart=t.chart,this.options=t.options,this.ctx=t.ctx,this.legendItems=void 0,this.columnSizes=void 0,this.lineWidths=void 0,this.maxHeight=void 0,this.maxWidth=void 0,this.top=void 0,this.bottom=void 0,this.left=void 0,this.right=void 0,this.height=void 0,this.width=void 0,this._margins=void 0,this.position=void 0,this.weight=void 0,this.fullSize=void 0}update(t,e,i){this.maxWidth=t,this.maxHeight=e,this._margins=i,this.setDimensions(),this.buildLabels(),this.fit()}setDimensions(){this.isHorizontal()?(this.width=this.maxWidth,this.left=this._margins.left,this.right=this.width):(this.height=this.maxHeight,this.top=this._margins.top,this.bottom=this.height)}buildLabels(){const t=this.options.labels||{};let e=d(t.generateLabels,[this.chart],this)||[];t.filter&&(e=e.filter((e=>t.filter(e,this.chart.data)))),t.sort&&(e=e.sort(((e,i)=>t.sort(e,i,this.chart.data)))),this.options.reverse&&e.reverse(),this.legendItems=e}fit(){const{options:t,ctx:e}=this;if(!t.display)return void(this.width=this.height=0);const i=t.labels,s=Si(i.font),n=s.size,o=this._computeTitleHeight(),{boxWidth:a,itemHeight:r}=ba(i,n);let l,h;e.font=s.string,this.isHorizontal()?(l=this.maxWidth,h=this._fitRows(o,n,a,r)+10):(h=this.maxHeight,l=this._fitCols(o,s,a,r)+10),this.width=Math.min(l,t.maxWidth||this.maxWidth),this.height=Math.min(h,t.maxHeight||this.maxHeight)}_fitRows(t,e,i,s){const{ctx:n,maxWidth:o,options:{labels:{padding:a}}}=this,r=this.legendHitBoxes=[],l=this.lineWidths=[0],h=s+a;let c=t;n.textAlign="left",n.textBaseline="middle";let d=-1,u=-h;return this.legendItems.forEach(((t,f)=>{const g=i+e/2+n.measureText(t.text).width;(0===f||l[l.length-1]+g+2*a>o)&&(c+=h,l[l.length-(f>0?0:1)]=0,u+=h,d++),r[f]={left:0,top:u,row:d,width:g,height:s},l[l.length-1]+=g+a})),c}_fitCols(t,e,i,s){const{ctx:n,maxHeight:o,options:{labels:{padding:a}}}=this,r=this.legendHitBoxes=[],l=this.columnSizes=[],h=o-t;let c=a,d=0,u=0,f=0,g=0;return this.legendItems.forEach(((t,o)=>{const{itemWidth:p,itemHeight:m}=function(t,e,i,s,n){const o=function(t,e,i,s){let n=t.text;n&&"string"!=typeof n&&(n=n.reduce(((t,e)=>t.length>e.length?t:e)));return e+i.size/2+s.measureText(n).width}(s,t,e,i),a=function(t,e,i){let s=t;"string"!=typeof e.text&&(s=_a(e,i));return s}(n,s,e.lineHeight);return{itemWidth:o,itemHeight:a}}(i,e,n,t,s);o>0&&u+m+2*a>h&&(c+=d+a,l.push({width:d,height:u}),f+=d+a,g++,d=u=0),r[o]={left:f,top:u,col:g,width:p,height:m},d=Math.max(d,p),u+=m+a})),c+=d,l.push({width:d,height:u}),c}adjustHitBoxes(){if(!this.options.display)return;const t=this._computeTitleHeight(),{legendHitBoxes:e,options:{align:i,labels:{padding:s},rtl:n}}=this,o=Oi(n,this.left,this.width);if(this.isHorizontal()){let n=0,a=ft(i,this.left+s,this.right-this.lineWidths[n]);for(const r of e)n!==r.row&&(n=r.row,a=ft(i,this.left+s,this.right-this.lineWidths[n])),r.top+=this.top+t+s,r.left=o.leftForLtr(o.x(a),r.width),a+=r.width+s}else{let n=0,a=ft(i,this.top+t+s,this.bottom-this.columnSizes[n].height);for(const r of e)r.col!==n&&(n=r.col,a=ft(i,this.top+t+s,this.bottom-this.columnSizes[n].height)),r.top=a,r.left+=this.left+s,r.left=o.leftForLtr(o.x(r.left),r.width),a+=r.height+s}}isHorizontal(){return"top"===this.options.position||"bottom"===this.options.position}draw(){if(this.options.display){const t=this.ctx;Ie(t,this),this._draw(),ze(t)}}_draw(){const{options:t,columnSizes:e,lineWidths:i,ctx:s}=this,{align:n,labels:o}=t,a=ue.color,r=Oi(t.rtl,this.left,this.width),h=Si(o.font),{padding:c}=o,d=h.size,u=d/2;let f;this.drawTitle(),s.textAlign=r.textAlign("left"),s.textBaseline="middle",s.lineWidth=.5,s.font=h.string;const{boxWidth:g,boxHeight:p,itemHeight:m}=ba(o,d),b=this.isHorizontal(),x=this._computeTitleHeight();f=b?{x:ft(n,this.left+c,this.right-i[0]),y:this.top+c+x,line:0}:{x:this.left+c,y:ft(n,this.top+x+c,this.bottom-e[0].height),line:0},Ai(this.ctx,t.textDirection);const _=m+c;this.legendItems.forEach(((y,v)=>{s.strokeStyle=y.fontColor,s.fillStyle=y.fontColor;const M=s.measureText(y.text).width,w=r.textAlign(y.textAlign||(y.textAlign=o.textAlign)),k=g+u+M;let S=f.x,P=f.y;r.setWidth(this.width),b?v>0&&S+k+c>this.right&&(P=f.y+=_,f.line++,S=f.x=ft(n,this.left+c,this.right-i[f.line])):v>0&&P+_>this.bottom&&(S=f.x=S+e[f.line].width+c,f.line++,P=f.y=ft(n,this.top+x+c,this.bottom-e[f.line].height));if(function(t,e,i){if(isNaN(g)||g<=0||isNaN(p)||p<0)return;s.save();const n=l(i.lineWidth,1);if(s.fillStyle=l(i.fillStyle,a),s.lineCap=l(i.lineCap,"butt"),s.lineDashOffset=l(i.lineDashOffset,0),s.lineJoin=l(i.lineJoin,"miter"),s.lineWidth=n,s.strokeStyle=l(i.strokeStyle,a),s.setLineDash(l(i.lineDash,[])),o.usePointStyle){const a={radius:p*Math.SQRT2/2,pointStyle:i.pointStyle,rotation:i.rotation,borderWidth:n},l=r.xPlus(t,g/2);Ee(s,a,l,e+u,o.pointStyleWidth&&g)}else{const o=e+Math.max((d-p)/2,0),a=r.leftForLtr(t,g),l=wi(i.borderRadius);s.beginPath(),Object.values(l).some((t=>0!==t))?He(s,{x:a,y:o,w:g,h:p,radius:l}):s.rect(a,o,g,p),s.fill(),0!==n&&s.stroke()}s.restore()}(r.x(S),P,y),S=gt(w,S+g+u,b?S+k:this.right,t.rtl),function(t,e,i){Ne(s,i.text,t,e+m/2,h,{strikethrough:i.hidden,textAlign:r.textAlign(i.textAlign)})}(r.x(S),P,y),b)f.x+=k+c;else if("string"!=typeof y.text){const t=h.lineHeight;f.y+=_a(y,t)+c}else f.y+=_})),Ti(this.ctx,t.textDirection)}drawTitle(){const t=this.options,e=t.title,i=Si(e.font),s=ki(e.padding);if(!e.display)return;const n=Oi(t.rtl,this.left,this.width),o=this.ctx,a=e.position,r=i.size/2,l=s.top+r;let h,c=this.left,d=this.width;if(this.isHorizontal())d=Math.max(...this.lineWidths),h=this.top+l,c=ft(t.align,c,this.right-d);else{const e=this.columnSizes.reduce(((t,e)=>Math.max(t,e.height)),0);h=l+ft(t.align,this.top,this.bottom-e-t.labels.padding-this._computeTitleHeight())}const u=ft(a,c,c+d);o.textAlign=n.textAlign(ut(a)),o.textBaseline="middle",o.strokeStyle=e.color,o.fillStyle=e.color,o.font=i.string,Ne(o,e.text,u,h,i)}_computeTitleHeight(){const t=this.options.title,e=Si(t.font),i=ki(t.padding);return t.display?e.lineHeight+i.height:0}_getLegendItemAt(t,e){let i,s,n;if(tt(t,this.left,this.right)&&tt(e,this.top,this.bottom))for(n=this.legendHitBoxes,i=0;it.chart.options.color,boxWidth:40,padding:10,generateLabels(t){const e=t.data.datasets,{labels:{usePointStyle:i,pointStyle:s,textAlign:n,color:o,useBorderRadius:a,borderRadius:r}}=t.legend.options;return t._getSortedDatasetMetas().map((t=>{const l=t.controller.getStyle(i?0:void 0),h=ki(l.borderWidth);return{text:e[t.index].label,fillStyle:l.backgroundColor,fontColor:o,hidden:!t.visible,lineCap:l.borderCapStyle,lineDash:l.borderDash,lineDashOffset:l.borderDashOffset,lineJoin:l.borderJoinStyle,lineWidth:(h.width+h.height)/4,strokeStyle:l.borderColor,pointStyle:s||l.pointStyle,rotation:l.rotation,textAlign:n||l.textAlign,borderRadius:a&&(r||l.borderRadius),datasetIndex:t.index}}),this)}},title:{color:t=>t.chart.options.color,display:!1,position:"center",text:""}},descriptors:{_scriptable:t=>!t.startsWith("on"),labels:{_scriptable:t=>!["generateLabels","filter","sort"].includes(t)}}};class va extends Hs{constructor(t){super(),this.chart=t.chart,this.options=t.options,this.ctx=t.ctx,this._padding=void 0,this.top=void 0,this.bottom=void 0,this.left=void 0,this.right=void 0,this.width=void 0,this.height=void 0,this.position=void 0,this.weight=void 0,this.fullSize=void 0}update(t,e){const i=this.options;if(this.left=0,this.top=0,!i.display)return void(this.width=this.height=this.right=this.bottom=0);this.width=this.right=t,this.height=this.bottom=e;const s=n(i.text)?i.text.length:1;this._padding=ki(i.padding);const o=s*Si(i.font).lineHeight+this._padding.height;this.isHorizontal()?this.height=o:this.width=o}isHorizontal(){const t=this.options.position;return"top"===t||"bottom"===t}_drawArgs(t){const{top:e,left:i,bottom:s,right:n,options:o}=this,a=o.align;let r,l,h,c=0;return this.isHorizontal()?(l=ft(a,i,n),h=e+t,r=n-i):("left"===o.position?(l=i+t,h=ft(a,s,e),c=-.5*C):(l=n-t,h=ft(a,e,s),c=.5*C),r=s-e),{titleX:l,titleY:h,maxWidth:r,rotation:c}}draw(){const t=this.ctx,e=this.options;if(!e.display)return;const i=Si(e.font),s=i.lineHeight/2+this._padding.top,{titleX:n,titleY:o,maxWidth:a,rotation:r}=this._drawArgs(s);Ne(t,e.text,0,0,i,{color:e.color,maxWidth:a,rotation:r,textAlign:ut(e.align),textBaseline:"middle",translation:[n,o]})}}var Ma={id:"title",_element:va,start(t,e,i){!function(t,e){const i=new va({ctx:t.ctx,options:e,chart:t});as.configure(t,i,e),as.addBox(t,i),t.titleBlock=i}(t,i)},stop(t){const e=t.titleBlock;as.removeBox(t,e),delete t.titleBlock},beforeUpdate(t,e,i){const s=t.titleBlock;as.configure(t,s,i),s.options=i},defaults:{align:"center",display:!1,font:{weight:"bold"},fullSize:!0,padding:10,position:"top",text:"",weight:2e3},defaultRoutes:{color:"color"},descriptors:{_scriptable:!0,_indexable:!1}};const wa=new WeakMap;var ka={id:"subtitle",start(t,e,i){const s=new va({ctx:t.ctx,options:i,chart:t});as.configure(t,s,i),as.addBox(t,s),wa.set(t,s)},stop(t){as.removeBox(t,wa.get(t)),wa.delete(t)},beforeUpdate(t,e,i){const s=wa.get(t);as.configure(t,s,i),s.options=i},defaults:{align:"center",display:!1,font:{weight:"normal"},fullSize:!0,padding:0,position:"top",text:"",weight:1500},defaultRoutes:{color:"color"},descriptors:{_scriptable:!0,_indexable:!1}};const Sa={average(t){if(!t.length)return!1;let e,i,s=0,n=0,o=0;for(e=0,i=t.length;e-1?t.split("\n"):t}function Ca(t,e){const{element:i,datasetIndex:s,index:n}=e,o=t.getDatasetMeta(s).controller,{label:a,value:r}=o.getLabelAndValue(n);return{chart:t,label:a,parsed:o.getParsed(n),raw:t.data.datasets[s].data[n],formattedValue:r,dataset:o.getDataset(),dataIndex:n,datasetIndex:s,element:i}}function Oa(t,e){const i=t.chart.ctx,{body:s,footer:n,title:o}=t,{boxWidth:a,boxHeight:r}=e,l=Si(e.bodyFont),h=Si(e.titleFont),c=Si(e.footerFont),d=o.length,f=n.length,g=s.length,p=ki(e.padding);let m=p.height,b=0,x=s.reduce(((t,e)=>t+e.before.length+e.lines.length+e.after.length),0);if(x+=t.beforeBody.length+t.afterBody.length,d&&(m+=d*h.lineHeight+(d-1)*e.titleSpacing+e.titleMarginBottom),x){m+=g*(e.displayColors?Math.max(r,l.lineHeight):l.lineHeight)+(x-g)*l.lineHeight+(x-1)*e.bodySpacing}f&&(m+=e.footerMarginTop+f*c.lineHeight+(f-1)*e.footerSpacing);let _=0;const y=function(t){b=Math.max(b,i.measureText(t).width+_)};return i.save(),i.font=h.string,u(t.title,y),i.font=l.string,u(t.beforeBody.concat(t.afterBody),y),_=e.displayColors?a+2+e.boxPadding:0,u(s,(t=>{u(t.before,y),u(t.lines,y),u(t.after,y)})),_=0,i.font=c.string,u(t.footer,y),i.restore(),b+=p.width,{width:b,height:m}}function Aa(t,e,i,s){const{x:n,width:o}=i,{width:a,chartArea:{left:r,right:l}}=t;let h="center";return"center"===s?h=n<=(r+l)/2?"left":"right":n<=o/2?h="left":n>=a-o/2&&(h="right"),function(t,e,i,s){const{x:n,width:o}=s,a=i.caretSize+i.caretPadding;return"left"===t&&n+o+a>e.width||"right"===t&&n-o-a<0||void 0}(h,t,e,i)&&(h="center"),h}function Ta(t,e,i){const s=i.yAlign||e.yAlign||function(t,e){const{y:i,height:s}=e;return it.height-s/2?"bottom":"center"}(t,i);return{xAlign:i.xAlign||e.xAlign||Aa(t,e,i,s),yAlign:s}}function La(t,e,i,s){const{caretSize:n,caretPadding:o,cornerRadius:a}=t,{xAlign:r,yAlign:l}=i,h=n+o,{topLeft:c,topRight:d,bottomLeft:u,bottomRight:f}=wi(a);let g=function(t,e){let{x:i,width:s}=t;return"right"===e?i-=s:"center"===e&&(i-=s/2),i}(e,r);const p=function(t,e,i){let{y:s,height:n}=t;return"top"===e?s+=i:s-="bottom"===e?n+i:n/2,s}(e,l,h);return"center"===l?"left"===r?g+=h:"right"===r&&(g-=h):"left"===r?g-=Math.max(c,u)+n:"right"===r&&(g+=Math.max(d,f)+n),{x:J(g,0,s.width-e.width),y:J(p,0,s.height-e.height)}}function Ea(t,e,i){const s=ki(i.padding);return"center"===e?t.x+t.width/2:"right"===e?t.x+t.width-s.right:t.x+s.left}function Ra(t){return Pa([],Da(t))}function Ia(t,e){const i=e&&e.dataset&&e.dataset.tooltip&&e.dataset.tooltip.callbacks;return i?t.override(i):t}const za={beforeTitle:e,title(t){if(t.length>0){const e=t[0],i=e.chart.data.labels,s=i?i.length:0;if(this&&this.options&&"dataset"===this.options.mode)return e.dataset.label||"";if(e.label)return e.label;if(s>0&&e.dataIndex{const e={before:[],lines:[],after:[]},n=Ia(i,t);Pa(e.before,Da(Fa(n,"beforeLabel",this,t))),Pa(e.lines,Fa(n,"label",this,t)),Pa(e.after,Da(Fa(n,"afterLabel",this,t))),s.push(e)})),s}getAfterBody(t,e){return Ra(Fa(e.callbacks,"afterBody",this,t))}getFooter(t,e){const{callbacks:i}=e,s=Fa(i,"beforeFooter",this,t),n=Fa(i,"footer",this,t),o=Fa(i,"afterFooter",this,t);let a=[];return a=Pa(a,Da(s)),a=Pa(a,Da(n)),a=Pa(a,Da(o)),a}_createItems(t){const e=this._active,i=this.chart.data,s=[],n=[],o=[];let a,r,l=[];for(a=0,r=e.length;at.filter(e,s,n,i)))),t.itemSort&&(l=l.sort(((e,s)=>t.itemSort(e,s,i)))),u(l,(e=>{const i=Ia(t.callbacks,e);s.push(Fa(i,"labelColor",this,e)),n.push(Fa(i,"labelPointStyle",this,e)),o.push(Fa(i,"labelTextColor",this,e))})),this.labelColors=s,this.labelPointStyles=n,this.labelTextColors=o,this.dataPoints=l,l}update(t,e){const i=this.options.setContext(this.getContext()),s=this._active;let n,o=[];if(s.length){const t=Sa[i.position].call(this,s,this._eventPosition);o=this._createItems(i),this.title=this.getTitle(o,i),this.beforeBody=this.getBeforeBody(o,i),this.body=this.getBody(o,i),this.afterBody=this.getAfterBody(o,i),this.footer=this.getFooter(o,i);const e=this._size=Oa(this,i),a=Object.assign({},t,e),r=Ta(this.chart,i,a),l=La(i,a,r,this.chart);this.xAlign=r.xAlign,this.yAlign=r.yAlign,n={opacity:1,x:l.x,y:l.y,width:e.width,height:e.height,caretX:t.x,caretY:t.y}}else 0!==this.opacity&&(n={opacity:0});this._tooltipItems=o,this.$context=void 0,n&&this._resolveAnimations().update(this,n),t&&i.external&&i.external.call(this,{chart:this.chart,tooltip:this,replay:e})}drawCaret(t,e,i,s){const n=this.getCaretPosition(t,i,s);e.lineTo(n.x1,n.y1),e.lineTo(n.x2,n.y2),e.lineTo(n.x3,n.y3)}getCaretPosition(t,e,i){const{xAlign:s,yAlign:n}=this,{caretSize:o,cornerRadius:a}=i,{topLeft:r,topRight:l,bottomLeft:h,bottomRight:c}=wi(a),{x:d,y:u}=t,{width:f,height:g}=e;let p,m,b,x,_,y;return"center"===n?(_=u+g/2,"left"===s?(p=d,m=p-o,x=_+o,y=_-o):(p=d+f,m=p+o,x=_-o,y=_+o),b=p):(m="left"===s?d+Math.max(r,h)+o:"right"===s?d+f-Math.max(l,c)-o:this.caretX,"top"===n?(x=u,_=x-o,p=m-o,b=m+o):(x=u+g,_=x+o,p=m+o,b=m-o),y=x),{x1:p,x2:m,x3:b,y1:x,y2:_,y3:y}}drawTitle(t,e,i){const s=this.title,n=s.length;let o,a,r;if(n){const l=Oi(i.rtl,this.x,this.width);for(t.x=Ea(this,i.titleAlign,i),e.textAlign=l.textAlign(i.titleAlign),e.textBaseline="middle",o=Si(i.titleFont),a=i.titleSpacing,e.fillStyle=i.titleColor,e.font=o.string,r=0;r0!==t))?(t.beginPath(),t.fillStyle=n.multiKeyBackground,He(t,{x:e,y:g,w:h,h:l,radius:r}),t.fill(),t.stroke(),t.fillStyle=a.backgroundColor,t.beginPath(),He(t,{x:i,y:g+1,w:h-2,h:l-2,radius:r}),t.fill()):(t.fillStyle=n.multiKeyBackground,t.fillRect(e,g,h,l),t.strokeRect(e,g,h,l),t.fillStyle=a.backgroundColor,t.fillRect(i,g+1,h-2,l-2))}t.fillStyle=this.labelTextColors[i]}drawBody(t,e,i){const{body:s}=this,{bodySpacing:n,bodyAlign:o,displayColors:a,boxHeight:r,boxWidth:l,boxPadding:h}=i,c=Si(i.bodyFont);let d=c.lineHeight,f=0;const g=Oi(i.rtl,this.x,this.width),p=function(i){e.fillText(i,g.x(t.x+f),t.y+d/2),t.y+=d+n},m=g.textAlign(o);let b,x,_,y,v,M,w;for(e.textAlign=o,e.textBaseline="middle",e.font=c.string,t.x=Ea(this,m,i),e.fillStyle=i.bodyColor,u(this.beforeBody,p),f=a&&"right"!==m?"center"===o?l/2+h:l+2+h:0,y=0,M=s.length;y0&&e.stroke()}_updateAnimationTarget(t){const e=this.chart,i=this.$animations,s=i&&i.x,n=i&&i.y;if(s||n){const i=Sa[t.position].call(this,this._active,this._eventPosition);if(!i)return;const o=this._size=Oa(this,t),a=Object.assign({},i,this._size),r=Ta(e,t,a),l=La(t,a,r,e);s._to===l.x&&n._to===l.y||(this.xAlign=r.xAlign,this.yAlign=r.yAlign,this.width=o.width,this.height=o.height,this.caretX=i.x,this.caretY=i.y,this._resolveAnimations().update(this,l))}}_willRender(){return!!this.opacity}draw(t){const e=this.options.setContext(this.getContext());let i=this.opacity;if(!i)return;this._updateAnimationTarget(e);const s={width:this.width,height:this.height},n={x:this.x,y:this.y};i=Math.abs(i)<.001?0:i;const o=ki(e.padding),a=this.title.length||this.beforeBody.length||this.body.length||this.afterBody.length||this.footer.length;e.enabled&&a&&(t.save(),t.globalAlpha=i,this.drawBackground(n,t,s,e),Ai(t,e.textDirection),n.y+=o.top,this.drawTitle(n,t,e),this.drawBody(n,t,e),this.drawFooter(n,t,e),Ti(t,e.textDirection),t.restore())}getActiveElements(){return this._active||[]}setActiveElements(t,e){const i=this._active,s=t.map((({datasetIndex:t,index:e})=>{const i=this.chart.getDatasetMeta(t);if(!i)throw new Error("Cannot find a dataset at index "+t);return{datasetIndex:t,element:i.data[e],index:e}})),n=!f(i,s),o=this._positionChanged(s,e);(n||o)&&(this._active=s,this._eventPosition=e,this._ignoreReplayEvents=!0,this.update(!0))}handleEvent(t,e,i=!0){if(e&&this._ignoreReplayEvents)return!1;this._ignoreReplayEvents=!1;const s=this.options,n=this._active||[],o=this._getActiveElements(t,n,e,i),a=this._positionChanged(o,t),r=e||!f(o,n)||a;return r&&(this._active=o,(s.enabled||s.external)&&(this._eventPosition={x:t.x,y:t.y},this.update(!0,e))),r}_getActiveElements(t,e,i,s){const n=this.options;if("mouseout"===t.type)return[];if(!s)return e;const o=this.chart.getElementsAtEventForMode(t,n.mode,n,i);return n.reverse&&o.reverse(),o}_positionChanged(t,e){const{caretX:i,caretY:s,options:n}=this,o=Sa[n.position].call(this,t,e);return!1!==o&&(i!==o.x||s!==o.y)}}var Ba={id:"tooltip",_element:Va,positioners:Sa,afterInit(t,e,i){i&&(t.tooltip=new Va({chart:t,options:i}))},beforeUpdate(t,e,i){t.tooltip&&t.tooltip.initialize(i)},reset(t,e,i){t.tooltip&&t.tooltip.initialize(i)},afterDraw(t){const e=t.tooltip;if(e&&e._willRender()){const i={tooltip:e};if(!1===t.notifyPlugins("beforeTooltipDraw",{...i,cancelable:!0}))return;e.draw(t.ctx),t.notifyPlugins("afterTooltipDraw",i)}},afterEvent(t,e){if(t.tooltip){const i=e.replay;t.tooltip.handleEvent(e.event,i,e.inChartArea)&&(e.changed=!0)}},defaults:{enabled:!0,external:null,position:"average",backgroundColor:"rgba(0,0,0,0.8)",titleColor:"#fff",titleFont:{weight:"bold"},titleSpacing:2,titleMarginBottom:6,titleAlign:"left",bodyColor:"#fff",bodySpacing:2,bodyFont:{},bodyAlign:"left",footerColor:"#fff",footerSpacing:2,footerMarginTop:6,footerFont:{weight:"bold"},footerAlign:"left",padding:6,caretPadding:2,caretSize:5,cornerRadius:6,boxHeight:(t,e)=>e.bodyFont.size,boxWidth:(t,e)=>e.bodyFont.size,multiKeyBackground:"#fff",displayColors:!0,boxPadding:0,borderColor:"rgba(0,0,0,0)",borderWidth:0,animation:{duration:400,easing:"easeOutQuart"},animations:{numbers:{type:"number",properties:["x","y","width","height","caretX","caretY"]},opacity:{easing:"linear",duration:200}},callbacks:za},defaultRoutes:{bodyFont:"font",footerFont:"font",titleFont:"font"},descriptors:{_scriptable:t=>"filter"!==t&&"itemSort"!==t&&"external"!==t,_indexable:!1,callbacks:{_scriptable:!1,_indexable:!1},animation:{_fallback:!1},animations:{_fallback:"animation"}},additionalOptionScopes:["interaction"]};return An.register(Yn,jo,fo,t),An.helpers={...Wi},An._adapters=Rn,An.Animation=Cs,An.Animations=Os,An.animator=xt,An.controllers=en.controllers.items,An.DatasetController=Ns,An.Element=Hs,An.elements=fo,An.Interaction=Xi,An.layouts=as,An.platforms=Ss,An.Scale=Js,An.Ticks=ae,Object.assign(An,Yn,jo,fo,t,Ss),An.Chart=An,"undefined"!=typeof window&&(window.Chart=An),An}));
+//# sourceMappingURL=chart.umd.js.map
diff --git a/web/js/charts_modal.js b/web/js/charts_modal.js
index e5df464..c3d0112 100644
--- a/web/js/charts_modal.js
+++ b/web/js/charts_modal.js
@@ -12,14 +12,14 @@ function updateModalChart(history) {
     }
 
     const labels = history.map(h => `E${h.epoch}`);
-    const scoreData = history.map(h => h.score !== undefined ? h.score : h.dice);
-    const lossData = history.map(h => h.loss !== undefined ? h.loss : h.bce);
+    const scoreData = history.map(h => h.score);
+    const lossData = history.map(h => h.loss);
 
     const hasScore = scoreData.some(d => d !== null && d !== undefined);
     const hasLoss = lossData.some(d => d !== null && d !== undefined);
 
-    const lossLabel = window.HPOState.data.hpoConfig?.metric_loss_label || "BCE";
-    const scoreLabel = window.HPOState.data.hpoConfig?.metric_score_label || "Dice";
+    const lossLabel = window.HPOState.data.hpoConfig?.metric_loss_label || "Loss";
+    const scoreLabel = window.HPOState.data.hpoConfig?.metric_score_label || "Score";
     const lossText = lossLabel.toLowerCase().includes("loss") ? lossLabel : `${lossLabel} Loss`;
     const scoreText = scoreLabel.toLowerCase().includes("score") || scoreLabel.toLowerCase().includes("accuracy") ? scoreLabel : `${scoreLabel} Score`;
 
diff --git a/web/js/charts_pareto.js b/web/js/charts_pareto.js
index f012734..85a24c0 100644
--- a/web/js/charts_pareto.js
+++ b/web/js/charts_pareto.js
@@ -16,8 +16,8 @@ function updateChart(trials, paretoSet, directions) {
     const isSingleObj = !directions || directions.length === 1;
     const isMinimize = directions && directions[0] === "MINIMIZE";
     
-    const lossLabel = window.HPOState.data.hpoConfig?.metric_loss_label || "BCE";
-    const scoreLabel = window.HPOState.data.hpoConfig?.metric_score_label || "Dice";
+    const lossLabel = window.HPOState.data.hpoConfig?.metric_loss_label || "Loss";
+    const scoreLabel = window.HPOState.data.hpoConfig?.metric_score_label || "Score";
     const metricLabel = isMinimize ? lossLabel : scoreLabel;
     const accentColor = window.HPOState.ui.accentColorHex;
 
diff --git a/web/js/charts_pathways_math.js b/web/js/charts_pathways_math.js
index 205d060..d0c10ad 100644
--- a/web/js/charts_pathways_math.js
+++ b/web/js/charts_pathways_math.js
@@ -13,8 +13,8 @@ function computeAxesBounds(trials) {
     paramKeys.forEach(k => {
         axes.push({ key: k, label: k.replace(/_/g, " "), type: "param" });
     });
-    axes.push({ key: "dice", label: window.HPOState.data.hpoConfig?.metric_score_label || "Dice", type: "metric" });
-    axes.push({ key: "bce", label: window.HPOState.data.hpoConfig?.metric_loss_label || "BCE", type: "metric" });
+    axes.push({ key: "score", label: window.HPOState.data.hpoConfig?.metric_score_label || "Score", type: "metric" });
+    axes.push({ key: "loss", label: window.HPOState.data.hpoConfig?.metric_loss_label || "Loss", type: "metric" });
 
     axes.forEach(axis => {
         if (axis.type === "param") {
diff --git a/web/js/charts_pathways_render.js b/web/js/charts_pathways_render.js
index 9fca7b9..920e891 100644
--- a/web/js/charts_pathways_render.js
+++ b/web/js/charts_pathways_render.js
@@ -155,7 +155,7 @@ function updateParallelCoordinates(trials, paretoSet) {
             ctx.strokeStyle = "rgba(100, 116, 139, 0.15)";
             ctx.lineWidth = 1;
         } else {
-            const score = t.dice !== null ? t.dice : 0;
+            const score = t.score !== null ? t.score : 0;
             const ratio = Math.max(0, Math.min(1, score));
             const grad = ctx.createLinearGradient(xPad, 0, W - xPad, 0);
             grad.addColorStop(0, `hsla(260, 80%, 60%, ${0.15 + ratio * 0.3})`);
@@ -177,7 +177,7 @@ function updateParallelCoordinates(trials, paretoSet) {
         
         const lossLabel = window.HPOState.data.hpoConfig?.metric_loss_label || "Loss";
         const scoreLabel = window.HPOState.data.hpoConfig?.metric_score_label || "Score";
-        const tooltipText = `Trial #${t.number} [${t.state}] — ${scoreLabel}: ${t.dice !== null ? t.dice.toFixed(4) : "N/A"}, ${lossLabel}: ${t.bce !== null ? t.bce.toFixed(4) : "N/A"}`;
+        const tooltipText = `Trial #${t.number} [${t.state}] — ${scoreLabel}: ${t.score !== null ? t.score.toFixed(4) : "N/A"}, ${lossLabel}: ${t.loss !== null ? t.loss.toFixed(4) : "N/A"}`;
         ctx.font = "600 0.75rem Inter, system-ui, sans-serif";
         const textWidth = ctx.measureText(tooltipText).width;
         const boxW = textWidth + 20, boxH = 28;
diff --git a/web/js/charts_timeline.js b/web/js/charts_timeline.js
index 52d3464..43e66e3 100644
--- a/web/js/charts_timeline.js
+++ b/web/js/charts_timeline.js
@@ -11,8 +11,8 @@ function updateAshaTimeline(trials) {
         return;
     }
 
-    const scoreLabel = window.HPOState.data.hpoConfig?.metric_score_label || "Dice";
-    const lossLabel = window.HPOState.data.hpoConfig?.metric_loss_label || "BCE";
+    const scoreLabel = window.HPOState.data.hpoConfig?.metric_score_label || "Score";
+    const lossLabel = window.HPOState.data.hpoConfig?.metric_loss_label || "Loss";
 
     let html = "";
     validTrials.forEach(t => {
@@ -77,8 +77,8 @@ function updateAshaTimeline(trials) {
                         ${paramsStr}
                     
                     

-                        ${t.dice !== null ? `${scoreLabel}: ${t.dice.toFixed(4)}` : ""}
-                        ${t.bce !== null ? `  (${lossLabel}: ${t.bce.toFixed(4)})` : ""}
+                        ${t.score !== null ? `${scoreLabel}: ${t.score.toFixed(4)}` : ""}
+                        ${t.loss !== null ? `  (${lossLabel}: ${t.loss.toFixed(4)})` : ""}
                     

                 
                 

diff --git a/web/js/charts_utils.js b/web/js/charts_utils.js
index 3d451fc..2cc733a 100644
--- a/web/js/charts_utils.js
+++ b/web/js/charts_utils.js
@@ -13,14 +13,14 @@ function prepareParetoDatasets(trials, paretoSet, directions, isSingleObj, isMin
 
     if (isSingleObj) {
         const sortedCompleted = trials
-            .filter(t => t.state === "COMPLETE" && (isMinimize ? t.bce !== null : t.dice !== null))
+            .filter(t => t.state === "COMPLETE" && (isMinimize ? t.loss !== null : t.score !== null))
             .sort((a, b) => a.number - b.number);
             
         let currentBest = null;
         const bestLinePoints = [];
         
         sortedCompleted.forEach(t => {
-            const val = isMinimize ? t.bce : t.dice;
+            const val = isMinimize ? t.loss : t.score;
             if (currentBest === null) {
                 currentBest = val;
             } else {
@@ -31,13 +31,13 @@ function prepareParetoDatasets(trials, paretoSet, directions, isSingleObj, isMin
         
         const completedPoints = sortedCompleted.map(t => ({
             x: t.number,
-            y: isMinimize ? t.bce : t.dice,
+            y: isMinimize ? t.loss : t.score,
             label: `Trial ${t.number}`
         }));
         
         const runningTrials = trials.filter(t => t.state === "RUNNING");
         runningTrials.forEach(t => {
-            const val = isMinimize ? t.bce : t.dice;
+            const val = isMinimize ? t.loss : t.score;
             if (val !== null && val !== undefined) {
                 runningPoints.push({ x: t.number, y: val, label: `Trial ${t.number} (running)` });
             }
@@ -84,8 +84,8 @@ function prepareParetoDatasets(trials, paretoSet, directions, isSingleObj, isMin
         const scatterPoints = [];
         const paretoPoints = [];
         trials.forEach(t => {
-            if (t.bce === null || t.dice === null) return;
-            const point = { x: t.bce, y: t.dice, label: `Trial ${t.number}` };
+            if (t.loss === null || t.score === null) return;
+            const point = { x: t.loss, y: t.score, label: `Trial ${t.number}` };
             if (t.state === "RUNNING") {
                 runningPoints.push(point);
             } else if (paretoSet.has(t.number)) {
@@ -137,7 +137,7 @@ function prepareParetoDatasets(trials, paretoSet, directions, isSingleObj, isMin
         }
     }
 
-    const totalPoints = trials.filter(t => (isSingleObj ? (isMinimize ? t.bce !== null : t.dice !== null) : (t.bce !== null && t.dice !== null))).length;
+    const totalPoints = trials.filter(t => (isSingleObj ? (isMinimize ? t.loss !== null : t.score !== null) : (t.loss !== null && t.score !== null))).length;
     return { datasets, totalPoints };
 }
 
diff --git a/web/js/export.js b/web/js/export.js
index 735d334..aa10734 100644
--- a/web/js/export.js
+++ b/web/js/export.js
@@ -23,11 +23,11 @@ function exportTrialsToCsv() {
                 const row = [
                     `#${t.number}`,
                     t.state,
-                    t.bce !== null && t.bce !== undefined ? t.bce.toFixed(6) : "",
-                    t.dice !== null && t.dice !== undefined ? t.dice.toFixed(6) : ""
+                    t.loss !== null && t.loss !== undefined ? t.loss.toFixed(6) : "",
+                    t.score !== null && t.score !== undefined ? t.score.toFixed(6) : ""
                 ];
                 if (ev.enabled) {
-                    row.push(t.dice_eval_fixed !== null && t.dice_eval_fixed !== undefined ? t.dice_eval_fixed.toFixed(6) : "");
+                    row.push(t.score_eval_fixed !== null && t.score_eval_fixed !== undefined ? t.score_eval_fixed.toFixed(6) : "");
                 }
                 paramKeys.forEach(k => {
                     const val = t.params[k];
@@ -69,8 +69,8 @@ async function exportParetoFront() {
                 data.pareto_front.forEach(t => {
                     const row = [
                         `#${t.number}`,
-                        t.bce !== null && t.bce !== undefined ? t.bce.toFixed(6) : "",
-                        t.dice !== null && t.dice !== undefined ? t.dice.toFixed(6) : ""
+                        t.loss !== null && t.loss !== undefined ? t.loss.toFixed(6) : "",
+                        t.score !== null && t.score !== undefined ? t.score.toFixed(6) : ""
                     ];
                     paramKeys.forEach(k => {
                         const val = t.params[k];
diff --git a/web/js/health.js b/web/js/health.js
index 3b20f38..b14ed29 100644
--- a/web/js/health.js
+++ b/web/js/health.js
@@ -156,7 +156,7 @@ function copyDiagnosticPrompt() {
         anomaly: window.HPOState.data.studyHealthReason || "unknown anomaly",
         active_search_space: window.HPOState.data.activeSearchSpace || {},
         recent_trials: window.HPOState.data.trials.slice(0, 3).map(t => ({
-            trial_id: t.number, state: t.state, params: t.params, score: t.dice, loss: t.bce
+            trial_id: t.number, state: t.state, params: t.params, score: t.score, loss: t.loss
         }))
     };
     const promptText = `I need help debugging my Pathfinder study because it is failing health checks:
diff --git a/web/js/main.js b/web/js/main.js
index 6db3d43..8735d1c 100644
--- a/web/js/main.js
+++ b/web/js/main.js
@@ -12,25 +12,24 @@ document.addEventListener("DOMContentLoaded", () => {
 
     window.fetchSearchSpace();
     window.fetchHpoConfig();
+
+    window.populateStudyList();
     
-    const reviewPill = document.getElementById("dashboard-review-pill");
-    if (reviewPill) {
-        reviewPill.addEventListener("click", window.copyReviewPrompt);
-        reviewPill.addEventListener("keydown", (e) => {
-            if (e.key === "Enter" || e.key === " ") {
-                e.preventDefault();
-                window.copyReviewPrompt();
+    async function checkWelcomeState() {
+        const overlay = document.getElementById("welcome-overlay");
+        if (!overlay) return;
+        try {
+            const res = await fetch("/api/studies");
+            const data = await res.json();
+            if (data.success && (!data.studies || data.studies.length === 0)) {
+                overlay.style.display = "flex";
             }
-        });
-    }
-    const reviewCloseBtn = document.getElementById("dashboard-review-close");
-    if (reviewCloseBtn) {
-        reviewCloseBtn.addEventListener("click", (e) => {
-            e.stopPropagation();
-            window.dismissReviewPill();
-        });
+        } catch (e) {
+            overlay.style.display = "flex";
+        }
     }
-    window.populateStudyList();
+    checkWelcomeState();
+
     const studySelect = document.getElementById("study-select");
     if (studySelect) {
         studySelect.addEventListener("change", (e) => {
@@ -88,7 +87,6 @@ async function pollData() {
     try {
         await Promise.all([
             fetchStudyDetails(true),
-            fetchPendingChanges(true),
             checkStudyHealth(true)
         ]);
         
@@ -122,4 +120,25 @@ async function pollData() {
 // Window exports
 window.pollData = pollData;
 
+async function launchDemo() {
+    const status = document.getElementById("welcome-status");
+    const btn = document.getElementById("welcome-launch-btn");
+    if (status) status.textContent = "Initializing demo study...";
+    if (btn) btn.disabled = true;
+    try {
+        const res = await fetch("/api/quickstart_demo", { method: "POST" });
+        const data = await res.json();
+        if (data.success) {
+            window.location.href = `/?study=${data.study_name}`;
+        } else {
+            if (status) status.textContent = "Failed to launch demo. Try again.";
+            if (btn) btn.disabled = false;
+        }
+    } catch (e) {
+        if (status) status.textContent = "Connection error. Is the broker running?";
+        if (btn) btn.disabled = false;
+    }
+}
+window.launchDemo = launchDemo;
+
 window.addEventListener("hashchange", () => window.handleRouting());
diff --git a/web/js/modal.js b/web/js/modal.js
deleted file mode 100644
index 5d7ba87..0000000
--- a/web/js/modal.js
+++ /dev/null
@@ -1,185 +0,0 @@
-function openTrialDetails(trialNum) {
-    const trial = window.HPOState.data.trials.find(t => t.number === trialNum);
-    if (!trial) return;
-
-    const titleEl = document.getElementById("modal-trial-title");
-    if (titleEl) titleEl.textContent = `Trial ${trial.number} Inspector [ID: ${trial.trial_id}]`;
-
-    const pGrid = document.getElementById("modal-params");
-    if (pGrid) {
-        pGrid.replaceChildren();
-        const displayParams = trial.params_display || trial.params;
-        Object.entries(displayParams).forEach(([k, v]) => {
-            const displayVal = typeof v === "number" ? (v % 1 === 0 ? String(v) : v.toExponential(2)) : String(v);
-            const item = document.createElement("div");
-            item.className = "param-item";
-            const label = document.createElement("div");
-            label.className = "param-label";
-            label.textContent = k.replace(/_/g, " ");
-            const value = document.createElement("div");
-            value.className = "param-value";
-            value.textContent = displayVal;
-            item.appendChild(label);
-            item.appendChild(value);
-            pGrid.appendChild(item);
-        });
-        if (trial.dice_eval_fixed != null) {
-            const item = document.createElement("div");
-            item.className = "param-item";
-            const label = document.createElement("div");
-            label.className = "param-label";
-            label.textContent = window.HPOState.data.hpoConfig?.eval_protocol?.dice_fixed_label || "Dice (eval)";
-            const value = document.createElement("div");
-            value.className = "param-value";
-            value.textContent = trial.dice_eval_fixed.toFixed(4);
-            item.appendChild(label);
-            item.appendChild(value);
-            pGrid.appendChild(item);
-        }
-        let ledClass = "complete";
-        let stateLabel = trial.state;
-
-        if (trial.state === "COMPLETE") {
-            ledClass = "complete";
-            stateLabel = "DONE";
-        } else if (trial.oom_triggered || trial.failure_tag === "OOM" || trial.state === "FAIL") {
-            ledClass = "fail";
-            stateLabel = (trial.oom_triggered || trial.failure_tag === "OOM") ? "FAILED (OOM)" : "FAILED";
-        } else if (trial.state === "PRUNED") {
-            ledClass = "pruned";
-            stateLabel = "PRUNED";
-        } else if (trial.state === "RUNNING") {
-            ledClass = "running";
-            stateLabel = "RUNNING";
-        }
-
-        const statusItem = document.createElement("div");
-        statusItem.className = `param-item status-param-item status-cell-${ledClass}`;
-        const statusLabel = document.createElement("div");
-        statusLabel.className = "param-label";
-        statusLabel.textContent = "State";
-        const statusValue = document.createElement("div");
-        statusValue.className = "param-value status-cell-wrapper";
-        statusValue.style.marginTop = "4px";
-        const led = document.createElement("span");
-        led.className = `led-light ${ledClass}`;
-        const statusText = document.createElement("span");
-        statusText.className = "status-text";
-        statusText.textContent = stateLabel;
-        statusValue.appendChild(led);
-        statusValue.appendChild(statusText);
-        statusItem.appendChild(statusLabel);
-        statusItem.appendChild(statusValue);
-        pGrid.appendChild(statusItem);
-    }
-
-    const oomSection = document.getElementById("modal-oom-section");
-    const oomDetail = document.getElementById("modal-oom-detail");
-    const isOom = trial.oom_triggered || trial.failure_tag === 'OOM';
-    if (oomSection && oomDetail) {
-        if (isOom) {
-            oomSection.style.display = "block";
-            let oomText = "This trial crashed with an out-of-memory error before completing.";
-            if (trial.gpu_model) oomText += ` GPU: ${trial.gpu_model}.`;
-            if (trial.max_vram_gb != null) oomText += ` Peak VRAM: ${trial.max_vram_gb.toFixed(1)} GB.`;
-            oomText += " TPE will automatically steer future suggestions away from this configuration.";
-            oomDetail.innerText = oomText;
-        } else {
-            oomSection.style.display = "none";
-        }
-    }
-
-    // Metric validation health warning
-    const warnSection = document.getElementById("modal-health-warning-section");
-    const warnDetail = document.getElementById("modal-health-warning-detail");
-    if (warnSection && warnDetail) {
-        if (trial.health_reason) {
-            warnSection.style.display = "block";
-            warnDetail.textContent = trial.health_reason;
-        } else {
-            warnSection.style.display = "none";
-        }
-    }
-
-    // Environment info section
-    const envSection = document.getElementById("modal-env-section");
-    const envFields = ["python_version", "platform", "hostname", "git_commit", "cuda_version", "dataset_version", "pip_freeze"];
-    const hasEnvData = envFields.some(f => trial[f] !== undefined && trial[f] !== null && trial[f] !== "");
-    if (envSection) {
-        if (hasEnvData) {
-            envSection.style.display = "block";
-            document.getElementById("modal-env-hostname").textContent = trial.hostname || "—";
-            document.getElementById("modal-env-platform").textContent = trial.platform || "—";
-            document.getElementById("modal-env-python").textContent = trial.python_version || "—";
-            document.getElementById("modal-env-cuda").textContent = trial.cuda_version || "—";
-            document.getElementById("modal-env-git").textContent = trial.git_commit || "—";
-            document.getElementById("modal-env-dataset").textContent = trial.dataset_version || "—";
-            
-            const pipTextarea = document.getElementById("modal-env-pip");
-            if (pipTextarea) {
-                pipTextarea.value = trial.pip_freeze || "No pip freeze recorded.";
-                pipTextarea.parentElement.style.display = trial.pip_freeze ? "flex" : "none";
-            }
-            
-            // Start collapsed
-            const content = document.getElementById("modal-env-content");
-            const chevron = document.getElementById("modal-env-chevron");
-            if (content) content.style.display = "none";
-            if (chevron) chevron.style.transform = "rotate(180deg)";
-        } else {
-            envSection.style.display = "none";
-        }
-    }
-
-    const reasoningEl = document.getElementById("modal-reasoning");
-    const rationaleTitle = document.getElementById("modal-rationale-title");
-    const logic = window.HPOState.data.thoughtLogs.find(l => l.trial_id === trial.trial_id);
-    if (reasoningEl && rationaleTitle) {
-        if (logic) {
-            rationaleTitle.innerText = "Coordinator Rationale";
-            reasoningEl.innerText = `"${logic.predicted_outcome_rationale}"`;
-        } else {
-            rationaleTitle.innerText = "TPE Suggestion";
-            reasoningEl.innerText = `"Optuna TPE sampled this configuration based on the observed score landscape. No coordinator override was active for this trial."`;
-        }
-    }
-
-    window.HPOState.ui.isModalOpen = true;
-    document.getElementById("detail-modal")?.classList.add("active");
-    requestAnimationFrame(() => window.updateModalChart(trial.history));
-}
-
-function closeModal(event) {
-    if (event.target.id === "detail-modal") {
-        closeModalDirect();
-    }
-}
-
-function closeModalDirect() {
-    document.getElementById("detail-modal")?.classList.remove("active");
-    window.HPOState.ui.isModalOpen = false;
-    if (window.HPOState.render.pendingRender) {
-        window.HPOState.render.pendingRender = false;
-        window.renderStudyDetails(window.HPOState.data.latestStudyData);
-    }
-}
-
-function toggleModalEnvCard() {
-    const content = document.getElementById("modal-env-content");
-    const chevron = document.getElementById("modal-env-chevron");
-    if (!content || !chevron) return;
-    
-    const isHidden = content.style.display === "none";
-    if (isHidden) {
-        content.style.display = "flex";
-        chevron.style.transform = "rotate(0deg)";
-    } else {
-        content.style.display = "none";
-        chevron.style.transform = "rotate(180deg)";
-    }
-}
-
-window.openTrialDetails = openTrialDetails;
-window.closeModal = closeModal;
-window.closeModalDirect = closeModalDirect;
-window.toggleModalEnvCard = toggleModalEnvCard;
diff --git a/web/js/onboarding.js b/web/js/onboarding.js
deleted file mode 100644
index 3e17885..0000000
--- a/web/js/onboarding.js
+++ /dev/null
@@ -1,313 +0,0 @@
-// web/js/onboarding.js — Handles manifest-based onboarding modal lifecycle and validation.
-
-(function () {
-    const DEFAULT_TEMPLATE = `study_name: segment_hpo
-metrics:
-  primary_score: dice
-  objectives:
-    - name: dice
-      direction: maximize
-      label: Dice Score
-    - name: loss
-      direction: minimize
-      label: BCE Loss
-params:
-  - name: lr
-    type: float_log
-    min: 0.0001
-    max: 0.1
-  - name: batch_size
-    type: categorical
-    options: [16, 32, 64]
-  - name: optimizer
-    type: categorical
-    options: ["adam", "sgd"]
-  - name: num_epochs
-    type: fixed
-    value: 10
-eval_protocol:
-  enabled: true
-  fixed_resolution: 512
-  train_resolution_param: resolution
-worker:
-  entrypoint: python train.py --lr {lr} --batch_size {batch_size}
-  env:
-    CUDA_VISIBLE_DEVICES: "0"`;
-
-    let validationTimeout = null;
-
-    function showNewStudyModal() {
-        const modal = document.getElementById("new-study-modal");
-        const editor = document.getElementById("manifest-yaml-editor");
-        const consoleEl = document.getElementById("manifest-validation-console");
-        const forceCheck = document.getElementById("manifest-force-check");
-        const submitBtn = document.getElementById("btn-submit-manifest");
-        const fileInput = document.getElementById("manifest-file-input");
-
-        if (fileInput) fileInput.value = "";
-        if (forceCheck) forceCheck.checked = false;
-        if (submitBtn) submitBtn.disabled = true;
-
-        if (editor && (!editor.value || editor.value.trim() === "")) {
-            editor.value = DEFAULT_TEMPLATE;
-        }
-
-        if (modal) {
-            modal.classList.add("active");
-            validateManifestOnClient();
-        }
-    }
-
-    function closeNewStudyModal(event) {
-        if (event && event.target.id === "new-study-modal") {
-            closeNewStudyModalDirect();
-        }
-    }
-
-    function closeNewStudyModalDirect() {
-        const modal = document.getElementById("new-study-modal");
-        if (modal) {
-            modal.classList.remove("active");
-        }
-    }
-
-    async function validateManifestOnClient() {
-        const editor = document.getElementById("manifest-yaml-editor");
-        const consoleEl = document.getElementById("manifest-validation-console");
-        const submitBtn = document.getElementById("btn-submit-manifest");
-
-        if (!editor || !consoleEl) return;
-
-        const yamlContent = editor.value;
-        if (!yamlContent || yamlContent.trim() === "") {
-            consoleEl.innerHTML = `
No manifest loaded yet. Drag-and-drop or select a file to begin.
`;
-            if (submitBtn) submitBtn.disabled = true;
-            return;
-        }
-
-        consoleEl.innerHTML = `
 Running validation check...
`;
-
-        try {
-            const res = await fetch("/api/validate_manifest", {
-                method: "POST",
-                headers: { "Content-Type": "application/json" },
-                body: JSON.stringify({ yaml: yamlContent })
-            });
-
-            if (!res.ok) {
-                const data = await res.json();
-                throw new Error(data.detail || "Validation request failed");
-            }
-
-            const result = await res.json();
-
-            consoleEl.innerHTML = "";
-
-            if (!result.success && result.errors && result.errors.length > 0) {
-                if (submitBtn) submitBtn.disabled = true;
-                
-                const errTitle = document.createElement("div");
-                errTitle.style.color = "var(--status-failed, #ef4444)";
-                errTitle.style.fontWeight = "bold";
-                errTitle.style.marginBottom = "6px";
-                errTitle.textContent = `✗ Validation Failed (${result.errors.length} error${result.errors.length > 1 ? 's' : ''}):`;
-                consoleEl.appendChild(errTitle);
-
-                result.errors.forEach(err => {
-                    const line = document.createElement("div");
-                    line.style.color = "#f87171";
-                    line.style.paddingLeft = "12px";
-                    line.style.textIndent = "-12px";
-                    line.style.marginBottom = "4px";
-                    line.textContent = `• ${err}`;
-                    consoleEl.appendChild(line);
-                });
-            } else {
-                if (submitBtn) submitBtn.disabled = false;
-                
-                const okLine = document.createElement("div");
-                okLine.style.color = "var(--status-complete, #10b981)";
-                okLine.style.fontWeight = "bold";
-                okLine.style.marginBottom = "6px";
-                okLine.textContent = `✓ Manifest validation passed! Ready to initialize.`;
-                consoleEl.appendChild(okLine);
-            }
-
-            if (result.warnings && result.warnings.length > 0) {
-                const warnTitle = document.createElement("div");
-                warnTitle.style.color = "var(--status-pruned, #f59e0b)";
-                warnTitle.style.fontWeight = "bold";
-                warnTitle.style.marginTop = "10px";
-                warnTitle.style.marginBottom = "6px";
-                warnTitle.textContent = `⚠ Warnings (${result.warnings.length}):`;
-                consoleEl.appendChild(warnTitle);
-
-                result.warnings.forEach(warn => {
-                    const line = document.createElement("div");
-                    line.style.color = "#fbbf24";
-                    line.style.paddingLeft = "12px";
-                    line.style.textIndent = "-12px";
-                    line.style.marginBottom = "4px";
-                    line.textContent = `• ${warn}`;
-                    consoleEl.appendChild(line);
-                });
-            }
-        } catch (err) {
-            if (submitBtn) submitBtn.disabled = true;
-            consoleEl.innerHTML = `
✗ Validation Error:

-            
${err.message}
`;
-        }
-    }
-
-    function debounceValidation() {
-        if (validationTimeout) {
-            clearTimeout(validationTimeout);
-        }
-        validationTimeout = setTimeout(validateManifestOnClient, 300);
-    }
-
-    async function submitManifest() {
-        const editor = document.getElementById("manifest-yaml-editor");
-        const consoleEl = document.getElementById("manifest-validation-console");
-        const forceCheck = document.getElementById("manifest-force-check");
-        const submitBtn = document.getElementById("btn-submit-manifest");
-
-        if (!editor) return;
-
-        const yamlContent = editor.value;
-        const force = forceCheck ? forceCheck.checked : false;
-
-        if (submitBtn) {
-            submitBtn.disabled = true;
-            submitBtn.textContent = "Initializing...";
-        }
-
-        consoleEl.innerHTML = `
Registering study...
`;
-
-        try {
-            const res = await fetch(`/api/init_from_manifest?force=${force}`, {
-                method: "POST",
-                headers: { "Content-Type": "application/json" },
-                body: JSON.stringify({ yaml: yamlContent })
-            });
-
-            const result = await res.json();
-
-            if (!res.ok) {
-                throw new Error(result.detail || "Server failed to register the study.");
-            }
-
-            if (!result.success) {
-                if (result.errors) {
-                    throw new Error("Validation errors returned: " + result.errors.join("; "));
-                } else {
-                    throw new Error(result.message || "Initialization failed");
-                }
-            }
-
-            consoleEl.innerHTML = `
✓ Study '${result.study_name}' registered successfully!
`;
-            
-            // Refresh study list and switch to the new study
-            if (window.populateStudyList) {
-                await window.populateStudyList();
-            }
-
-            // Select and redirect to the new study
-            const select = document.getElementById("study-select");
-            if (select) {
-                select.value = result.study_name;
-                const url = new URL(window.location);
-                url.searchParams.set('study', result.study_name);
-                window.history.pushState({}, '', url);
-                
-                window.HPOState.session.studyName = result.study_name;
-                window.HPOState.tables.dashboard = { sort: { col: null, dir: null }, filters: {} };
-                window.HPOState.tables.analysis = { sort: { col: null, dir: null }, filters: {} };
-                window.HPOState.render.lastDashboardHeaderSnapshot = "";
-                window.HPOState.render.lastAnalysisHeaderSnapshot = "";
-
-                if (window.fetchStudyDetails) window.fetchStudyDetails();
-                if (window.fetchFanova) window.fetchFanova();
-                if (window.fetchSearchSpace) window.fetchSearchSpace();
-                if (window.fetchHpoConfig) window.fetchHpoConfig();
-            }
-
-            setTimeout(() => {
-                closeNewStudyModalDirect();
-                if (typeof window.showToast === 'function') {
-                    window.showToast("Study registered successfully! Go to the 'Worker Setup' tab to run your model.");
-                }
-            }, 1000);
-
-        } catch (err) {
-            consoleEl.innerHTML = `
✗ Initialization Failed:

-            
${err.message}
`;
-        } finally {
-            if (submitBtn) {
-                submitBtn.disabled = false;
-                submitBtn.textContent = "Initialize Study";
-            }
-        }
-    }
-
-    // Set up drag and drop + file input listener once DOM loads
-    document.addEventListener("DOMContentLoaded", () => {
-        const uploadZone = document.getElementById("manifest-upload-zone");
-        const fileInput = document.getElementById("manifest-file-input");
-        const editor = document.getElementById("manifest-yaml-editor");
-
-        if (uploadZone && fileInput) {
-            uploadZone.addEventListener("click", () => {
-                fileInput.click();
-            });
-
-            uploadZone.addEventListener("dragover", (e) => {
-                e.preventDefault();
-                uploadZone.classList.add("dragover");
-            });
-
-            uploadZone.addEventListener("dragleave", (e) => {
-                e.preventDefault();
-                uploadZone.classList.remove("dragover");
-            });
-
-            uploadZone.addEventListener("drop", (e) => {
-                e.preventDefault();
-                uploadZone.classList.remove("dragover");
-                if (e.dataTransfer.files && e.dataTransfer.files.length > 0) {
-                    const file = e.dataTransfer.files[0];
-                    readManifestFile(file);
-                }
-            });
-
-            fileInput.addEventListener("change", (e) => {
-                if (e.target.files && e.target.files.length > 0) {
-                    const file = e.target.files[0];
-                    readManifestFile(file);
-                }
-            });
-        }
-
-        if (editor) {
-            editor.addEventListener("input", debounceValidation);
-        }
-
-        function readManifestFile(file) {
-            const reader = new FileReader();
-            reader.onload = (event) => {
-                if (editor) {
-                    editor.value = event.target.result;
-                    validateManifestOnClient();
-                }
-            };
-            reader.readAsText(file);
-        }
-    });
-
-    // Expose functions globally
-    window.showNewStudyModal = showNewStudyModal;
-    window.closeNewStudyModal = closeNewStudyModal;
-    window.closeNewStudyModalDirect = closeNewStudyModalDirect;
-    window.submitManifest = submitManifest;
-    window.validateManifestOnClient = validateManifestOnClient;
-})();
diff --git a/web/js/settings.js b/web/js/settings.js
index 3d98a13..1e6a519 100644
--- a/web/js/settings.js
+++ b/web/js/settings.js
@@ -1,48 +1,3 @@
-async function fetchPendingChanges(throwOnError = false) {
-    try {
-        const res = await fetch(`/api/pending_changes?study_name=${window.HPOState.session.studyName}`);
-        if (!res.ok) { if (throwOnError) throw new Error("HTTP " + res.status); return; }
-        const data = await res.json();
-        const banner = document.getElementById("pending-changes-banner");
-        const diffContainer = document.getElementById("pending-changes-diff");
-        if (!banner || !diffContainer) return;
-        if (data && data.proposed_changes) {
-            window.HPOState.data.pendingChanges = data.proposed_changes;
-            const current = window.HPOState.data.activeSearchSpace || {};
-            diffContainer.replaceChildren();
-            for (const [key, val] of Object.entries(data.proposed_changes)) {
-                const curVal = current[key];
-                if (!curVal) continue;
-                const infoLine = document.createElement("span");
-                infoLine.className = "diff-line info";
-                infoLine.textContent = `# Parameter: ${paramLabel(key)} (${key})`;
-                diffContainer.appendChild(infoLine);
-                if (curVal.type === "categorical") {
-                    const curActive = curVal.active || [], newActive = val.active || [];
-                    diffContainer.appendChild(Object.assign(document.createElement("span"), { className: "diff-line removed", textContent: `- active: [${curActive.join(", ")}]` }));
-                    diffContainer.appendChild(Object.assign(document.createElement("span"), { className: "diff-line added", textContent: `+ active: [${newActive.join(", ")}]` }));
-                } else {
-                    const curMin = curVal.min, curMax = curVal.max;
-                    const newMin = val.min !== undefined ? val.min : curMin;
-                    const newMax = val.max !== undefined ? val.max : curMax;
-                    if (curMin !== newMin || curMax !== newMax) {
-                        diffContainer.appendChild(Object.assign(document.createElement("span"), { className: "diff-line removed", textContent: `- range: [${curMin}, ${curMax}]` }));
-                        diffContainer.appendChild(Object.assign(document.createElement("span"), { className: "diff-line added", textContent: `+ range: [${newMin}, ${newMax}]` }));
-                    }
-                }
-            }
-            banner.classList.toggle("hidden", diffContainer.childNodes.length === 0);
-        } else {
-            window.HPOState.data.pendingChanges = null;
-            banner.classList.add("hidden");
-        }
-        renderHealthLatestAction();
-    } catch (err) {
-        console.error("Error fetching pending changes:", err);
-        if (throwOnError) throw err;
-    }
-}
-
 function populateEvalProtocolForm(config) {
     const ev = config?.eval_protocol || {};
     const vr = config?.validation_rules || {};
@@ -50,13 +5,12 @@ function populateEvalProtocolForm(config) {
     const setCheck = (id, val) => { const el = document.getElementById(id); if (el) el.checked = !!val; };
     set("eval-metric-loss-label", config?.metric_loss_label || "Loss");
     set("eval-metric-score-label", config?.metric_score_label || "Score");
-    setCheck("desktop-notifs-enabled", !!config?.desktop_notifications_enabled);
     setCheck("eval-enabled", ev.enabled);
     set("eval-fixed-resolution", ev.fixed_resolution ?? "");
     set("eval-train-res-param", ev.train_resolution_param || "resolution");
     set("eval-low-warn", ev.low_train_res_warning ?? "");
-    set("eval-score-train-label", ev.dice_train_label || "Score (train)");
-    set("eval-score-fixed-label", ev.dice_fixed_label || "Score (eval)");
+    set("eval-score-train-label", ev.score_train_label || "Score (train)");
+    set("eval-score-fixed-label", ev.score_fixed_label || "Score (eval)");
     setCheck("eval-prune-on-fixed", ev.use_fixed_metric_for_pruning);
     // Validation rules
     setCheck("validation-rules-enabled", !!vr.enabled);
@@ -85,8 +39,8 @@ async function saveEvalProtocol() {
             fixed_resolution: fixedRaw === "" || fixedRaw == null ? null : Number(fixedRaw),
             train_resolution_param: document.getElementById("eval-train-res-param")?.value || "resolution",
             low_train_res_warning: lowRaw === "" || lowRaw == null ? null : Number(lowRaw),
-            dice_train_label: document.getElementById("eval-score-train-label")?.value || "Score (train)",
-            dice_fixed_label: document.getElementById("eval-score-fixed-label")?.value || "Score (eval)",
+            score_train_label: document.getElementById("eval-score-train-label")?.value || "Score (train)",
+            score_fixed_label: document.getElementById("eval-score-fixed-label")?.value || "Score (eval)",
             use_fixed_metric_for_pruning: document.getElementById("eval-prune-on-fixed")?.checked || false,
         },
         validation_rules: {
@@ -115,49 +69,6 @@ async function saveEvalProtocol() {
     } catch (err) { alert("Error saving eval protocol: " + err); }
 }
 
-async function saveIdeSettings() {
-    const payload = {
-        ...window.HPOState.data.hpoConfig,
-        desktop_notifications_enabled: !!document.getElementById("desktop-notifs-enabled")?.checked,
-    };
-    try {
-        const studyName = window.HPOState.session.studyName || '';
-        const res = await fetch(`/api/hpo_config?study_name=${encodeURIComponent(studyName)}`, {
-            method: "POST",
-            headers: { "Content-Type": "application/json" },
-            body: JSON.stringify(payload),
-        });
-        if (!res.ok) { alert("Failed to save IDE settings"); return; }
-        const data = await res.json();
-        window.HPOState.data.hpoConfig = data.config || data;
-        populateEvalProtocolForm(window.HPOState.data.hpoConfig);
-        alert("IDE integration settings saved successfully!");
-    } catch (err) { alert("Error saving IDE settings: " + err); }
-}
-
-async function applyPendingChanges() {
-    try {
-        const res = await fetch(`/api/apply_pending_changes?study_name=${window.HPOState.session.studyName}`, { method: "POST" });
-        if (!res.ok) { const errData = await res.json(); alert("Error: " + (errData.detail || "Failed to apply changes")); return; }
-        const data = await res.json();
-        if (data.success) {
-            window.HPOState.data.activeSearchSpace = data.space;
-            renderSearchSpace();
-            renderDashboardSearchSpaceSummary();
-            fetchPendingChanges();
-        }
-    } catch (err) { console.error("Error applying pending changes:", err); }
-}
-
-async function discardPendingChanges() {
-    try {
-        const res = await fetch(`/api/discard_pending_changes?study_name=${window.HPOState.session.studyName}`, { method: "POST" });
-        if (!res.ok) { alert("Failed to discard changes"); return; }
-        const data = await res.json();
-        if (data.success) { fetchPendingChanges(); }
-    } catch (err) { console.error("Error discarding pending changes:", err); }
-}
-
 function togglePill(param, option, btn) {
     const list = window.HPOState.data.activeSearchSpace[param].active;
     const val = isNaN(option) ? option : Number(option);
@@ -306,12 +217,8 @@ function switchSettingsTab(tabId) {
     if (panel) panel.classList.add("active");
 }
 
-window.fetchPendingChanges = fetchPendingChanges;
 window.populateEvalProtocolForm = populateEvalProtocolForm;
 window.saveEvalProtocol = saveEvalProtocol;
-window.saveIdeSettings = saveIdeSettings;
-window.applyPendingChanges = applyPendingChanges;
-window.discardPendingChanges = discardPendingChanges;
 window.togglePill = togglePill;
 window.applySingleParamConstraints = applySingleParamConstraints;
 window.renderSearchSpace = renderSearchSpace;
diff --git a/web/js/table_filters.js b/web/js/table_filters.js
index 75e0254..c1a9aba 100644
--- a/web/js/table_filters.js
+++ b/web/js/table_filters.js
@@ -1,5 +1,5 @@
 function getPrimaryScoreKey() {
-    return window.HPOState.data.hpoConfig?.primary_score_key || "dice";
+    return window.HPOState.data.hpoConfig?.primary_score_key || "score";
 }
 
 function filterTrialsForDisplay(trials, paretoSet) {
@@ -74,9 +74,9 @@ function applyTablePipeline(trials, tableId) {
 
 function cycleTableSort(tableId, colKey) {
     const sort = window.HPOState.tables[tableId].sort;
-    // Loss (bce) defaults to ascending (lower is better).
-    // Others (number/Trial, score/dice/eval, parameters) default to descending (higher/latest is better).
-    const defaultDir = (colKey === "bce") ? "asc" : "desc";
+    // Loss defaults to ascending (lower is better).
+    // Others (number/Trial, score/eval, parameters) default to descending (higher/latest is better).
+    const defaultDir = (colKey === "loss") ? "asc" : "desc";
     
     if (sort.col !== colKey) {
         sort.col = colKey;
diff --git a/web/js/table_infra.js b/web/js/table_infra.js
index 6c929cb..1992394 100644
--- a/web/js/table_infra.js
+++ b/web/js/table_infra.js
@@ -110,9 +110,9 @@ function makeTableResizable(table, force = false) {
 function getTrialCellValue(trial, colKey) {
     if (colKey === "number") return trial.number;
     if (colKey === "state") return trial.state;
-    if (colKey === "bce") return trial.bce;
-    if (colKey === "dice") return trial[getPrimaryScoreKey()] ?? trial.dice;
-    if (colKey === "dice_eval_fixed") return trial.dice_eval_fixed;
+    if (colKey === "loss") return trial.loss;
+    if (colKey === "score") return trial[getPrimaryScoreKey()] ?? trial.score;
+    if (colKey === "score_eval_fixed") return trial.score_eval_fixed;
     if (colKey.startsWith("param:")) return trial.params?.[colKey.slice(6)];
     return null;
 }
diff --git a/web/js/table_render.js b/web/js/table_render.js
index d7cad78..0725fb6 100644
--- a/web/js/table_render.js
+++ b/web/js/table_render.js
@@ -31,7 +31,7 @@ function buildTrialsSnapshot(trials, paretoSet) {
     const base = trials
         .map(
             (t) =>
-                `${t.number}|${t.state}|${t.bce ?? ""}|${t.dice ?? ""}|${paretoSet.has(t.number) ? 1 : 0}`
+                `${t.number}|${t.state}|${t.loss ?? ""}|${t.score ?? ""}|${paretoSet.has(t.number) ? 1 : 0}`
         )
         .sort((a, b) => Number(a.split("|")[0]) - Number(b.split("|")[0]))
         .join(";");
@@ -41,7 +41,7 @@ function buildTrialsSnapshot(trials, paretoSet) {
 function buildFanovaSnapshot(trials) {
     return trials
         .filter((t) => t.state === "COMPLETE")
-        .map((t) => `${t.number}|${t.bce}|${t.dice}`)
+        .map((t) => `${t.number}|${t.loss}|${t.score}`)
         .join(";");
 }
 
@@ -65,8 +65,8 @@ function renderDashboardTableBody(displayTrials, paretoSet, data) {
     let rowsHtml = "";
     piped.forEach((t) => {
         const isPareto = paretoSet.has(t.number);
-        const bceStr = formatMetric(t.bce);
-        const diceStr = formatMetric(t.dice);
+        const lossStr = formatMetric(t.loss);
+        const scoreStr = formatMetric(t.score);
         let cellsHtml = "";
         paramKeys.forEach((k) => {
             const val = t.params[k];
@@ -82,12 +82,12 @@ function renderDashboardTableBody(displayTrials, paretoSet, data) {
         });
         let trClass = t.state === "RUNNING" ? "running" : (isPareto ? "pareto" : (t.state === "PRUNED" ? "pruned" : (t.state === "FAIL" ? "failed" : "")));
         const statusTdHtml = buildStatusTd(t.state, t.latest_epoch);
-        const fixedStr = formatMetric(t.dice_eval_fixed);
+        const fixedStr = formatMetric(t.score_eval_fixed);
         let rowHtml = `
             #${t.number}
             ${statusTdHtml}
-            ${bceStr}
-            ${diceStr}`;
+            ${lossStr}
+            ${scoreStr}`;
         if (ev.enabled) rowHtml += `${fixedStr}`;
         rowHtml += `${cellsHtml}
             ${isPareto ? "★" : ""}
@@ -131,9 +131,9 @@ function renderAnalysisTableBody(displayTrials) {
     }
     let html = "";
     piped.forEach((t) => {
-        const bceStr = formatMetric(t.bce, "—");
-        const diceStr = formatMetric(t.dice, "—");
-        const fixedStr = formatMetric(t.dice_eval_fixed, "—");
+        const lossStr = formatMetric(t.loss, "—");
+        const scoreStr = formatMetric(t.score, "—");
+        const fixedStr = formatMetric(t.score_eval_fixed, "—");
         let cellsHtml = "";
         paramKeys.forEach((k) => {
             const val = t.params[k];
@@ -151,8 +151,8 @@ function renderAnalysisTableBody(displayTrials) {
         html += `
             #${t.number}
             ${statusTdHtml}
-            ${bceStr}
-            ${diceStr}
+            ${lossStr}
+            ${scoreStr}
             ${ev.enabled ? `${fixedStr}` : ""}
             ${cellsHtml}
             
@@ -176,7 +176,7 @@ function applyAnalysisTableHeaders() {
 
     const lossLabel = window.HPOState.data.hpoConfig?.metric_loss_label || "Loss";
     const scoreLabel = window.HPOState.data.hpoConfig?.metric_score_label || "Score";
-    const evalLabel = ev.dice_fixed_label || "Score (eval)";
+    const evalLabel = ev.score_fixed_label || "Score (eval)";
 
     const headerSnapshot = `${lossLabel}|${scoreLabel}|${evalLabel}|${ev.enabled}|${paramKeys.join(",")}`;
     if (headerSnapshot === window.HPOState.render.lastAnalysisHeaderSnapshot) {
@@ -188,10 +188,10 @@ function applyAnalysisTableHeaders() {
     if (tableHeader) {
         let thHtml = buildSortableTh("Trial", "number", "analysis");
         thHtml += buildSortableTh("State", "state", "analysis");
-        thHtml += buildSortableTh(lossLabel, "bce", "analysis");
-        thHtml += buildSortableTh(scoreLabel, "dice", "analysis");
+        thHtml += buildSortableTh(lossLabel, "loss", "analysis");
+        thHtml += buildSortableTh(scoreLabel, "score", "analysis");
         if (ev.enabled) {
-            thHtml += buildSortableTh(evalLabel, "dice_eval_fixed", "analysis");
+            thHtml += buildSortableTh(evalLabel, "score_eval_fixed", "analysis");
         }
         paramKeys.forEach(k => {
             const label = paramLabels[k] || k.replace(/_/g, " ");
@@ -238,7 +238,7 @@ function renderStudyDetails(data) {
     if (tableHeader) {
         const lossLabel = data.hpo_config?.metric_loss_label || "Loss";
         const scoreLabel = data.hpo_config?.metric_score_label || "Score";
-        const evalLabel = ev.dice_fixed_label || "Score (eval)";
+        const evalLabel = ev.score_fixed_label || "Score (eval)";
 
         const headerSnapshot = `${lossLabel}|${scoreLabel}|${evalLabel}|${ev.enabled}|${paramKeys.join(",")}`;
         if (headerSnapshot !== window.HPOState.render.lastDashboardHeaderSnapshot) {
@@ -246,10 +246,10 @@ function renderStudyDetails(data) {
 
             let thHtml = buildSortableTh("Trial", "number", "dashboard");
             thHtml += buildSortableTh("State", "state", "dashboard");
-            thHtml += buildSortableTh(lossLabel, "bce", "dashboard");
-            thHtml += buildSortableTh(scoreLabel, "dice", "dashboard");
+            thHtml += buildSortableTh(lossLabel, "loss", "dashboard");
+            thHtml += buildSortableTh(scoreLabel, "score", "dashboard");
             if (ev.enabled) {
-                thHtml += buildSortableTh(evalLabel, "dice_eval_fixed", "dashboard");
+                thHtml += buildSortableTh(evalLabel, "score_eval_fixed", "dashboard");
             }
             paramKeys.forEach(k => {
                 const label = paramLabels[k] || k.replace(/_/g, " ");
@@ -369,10 +369,10 @@ function applyEvalInsightsUi() {
         grid.innerHTML = keys
             .map((res) => {
                 const s = summary[res];
-                const trainBestVal = s.best_score_train !== undefined ? s.best_score_train : s.best_dice_train;
+                const trainBestVal = s.best_score_train;
                 const trainBest = trainBestVal != null ? trainBestVal.toFixed(4) : "—";
                 
-                const fixedBestVal = s.best_score_fixed !== undefined ? s.best_score_fixed : s.best_dice_fixed;
+                const fixedBestVal = s.best_score_fixed;
                 const fixedBest = fixedBestVal != null ? fixedBestVal.toFixed(4) : "—";
                 
                 // Capitalize parameter label for individual chips
@@ -386,9 +386,7 @@ function applyEvalInsightsUi() {
 
     const scoreLabel = window.HPOState.data.hpoConfig?.metric_score_label || "Score";
     if (deployLabel && window.HPOState.data.evalInsights?.best_deploy_trial_number != null) {
-        const d = window.HPOState.data.evalInsights.best_deploy_score_fixed !== undefined
-            ? window.HPOState.data.evalInsights.best_deploy_score_fixed
-            : window.HPOState.data.evalInsights.best_deploy_dice_fixed;
+        const d = window.HPOState.data.evalInsights.best_deploy_score_fixed;
         deployLabel.textContent = `Best fixed-eval: Trial #${window.HPOState.data.evalInsights.best_deploy_trial_number}${d != null ? ` · ${scoreLabel} ${d.toFixed(4)}` : ""}`;
     } else if (deployLabel) {
         deployLabel.textContent = "";
diff --git a/web/js/theme.js b/web/js/theme.js
index 5e25dfc..ee4ba79 100644
--- a/web/js/theme.js
+++ b/web/js/theme.js
@@ -80,7 +80,7 @@ function changeAccent(accentName, colorHex, glowStyle) {
     
     if (window.HPOState.charts.modalHistory.instance && window.HPOState.charts.modalHistory.instance.data && window.HPOState.charts.modalHistory.instance.data.datasets) {
         window.HPOState.charts.modalHistory.instance.data.datasets.forEach(ds => {
-            if (ds.yAxisID === "y-dice") {
+            if (ds.yAxisID === "y-score") {
                 ds.borderColor = colorHex;
             }
         });
diff --git a/web/js/utils.js b/web/js/utils.js
index a35bcf4..1f896f9 100644
--- a/web/js/utils.js
+++ b/web/js/utils.js
@@ -244,6 +244,15 @@ function hideEmptyState(containerId, recreateCanvas = false) {
     }
 }
 
+function initHpoMarks() {
+    const svg = window.HPOState?.constants?.HPO_MARK_SVG;
+    if (!svg) return;
+    document.querySelectorAll(".brand-mark").forEach((el) => {
+        el.innerHTML = svg;
+    });
+}
+
 window.showEmptyState = showEmptyState;
 window.hideEmptyState = hideEmptyState;
+window.initHpoMarks = initHpoMarks;
 
diff --git a/web/js/worker_setup.js b/web/js/worker_setup.js
deleted file mode 100644
index cf13658..0000000
--- a/web/js/worker_setup.js
+++ /dev/null
@@ -1,231 +0,0 @@
-/* === Worker Setup & Colab Integration === */
-
-async function updateColabSnippet() {
-    // Determine current host base URL
-    const protocol = window.location.protocol;
-    const host = window.location.host;
-    let baseUrl = `${protocol}//${host}`;
-    
-    // Handle custom inputs
-    const inputEl = document.getElementById("ngrok-url-input");
-    const colabInputEl = document.getElementById("ngrok-url-input-colab");
-    const customInput = inputEl ? inputEl.value.trim() : "";
-    
-    if (customInput) {
-        baseUrl = customInput;
-        if (baseUrl.endsWith('/')) {
-            baseUrl = baseUrl.slice(0, -1);
-        }
-    } else {
-        // Pre-fill the input with the current browser origin if empty
-        if (inputEl) {
-            inputEl.value = baseUrl;
-        }
-    }
-    
-    // Keep the Colab input field in sync
-    if (colabInputEl && colabInputEl.value !== baseUrl) {
-        colabInputEl.value = baseUrl;
-    }
-    
-    // Toggle Colab warning if the active broker URL is localhost / 127.0.0.1 / [::1]
-    const isLocalhost = baseUrl.includes("localhost") || baseUrl.includes("127.0.0.1") || baseUrl.includes("[::1]");
-    const warningBanner = document.getElementById("colab-warning-banner");
-    const warningUrlSpan = document.getElementById("colab-warning-url");
-    
-    if (warningBanner) {
-        if (isLocalhost) {
-            warningBanner.style.display = "block";
-            if (warningUrlSpan) {
-                warningUrlSpan.textContent = baseUrl;
-            }
-        } else {
-            warningBanner.style.display = "none";
-        }
-    }
-    
-    const tokenLine = isLocalhost
-        ? ""
-        : `os.environ["HPO_SECRET_TOKEN"] = "paste-same-token-as-broker"\n`;
-
-    const studyName = window.HPOState.session.studyName || 'bridge_crack_study';
-
-    let isReference = true;
-    let workerEntrypoint = null;
-    let workerEnv = null;
-    let colabSnippet = null;
-
-    try {
-        const response = await fetch(`/api/study_setup?study_name=${encodeURIComponent(studyName)}`);
-        if (response.ok) {
-            const data = await response.json();
-            if (data.success) {
-                isReference = data.is_reference;
-                workerEntrypoint = data.worker_entrypoint;
-                workerEnv = data.worker_env;
-                colabSnippet = data.colab_snippet;
-            }
-        }
-    } catch (err) {
-        console.error("Error fetching study setup:", err);
-    }
-
-    // Render the Colab snippet text
-    let snippet = "";
-    if (colabSnippet) {
-        snippet = colabSnippet
-            .replace(/\$\{baseUrl\}/g, baseUrl)
-            .replace(/\{baseUrl\}/g, baseUrl)
-            .replace(/\$\{studyName\}/g, studyName)
-            .replace(/\{studyName\}/g, studyName);
-    } else if (isReference) {
-        snippet = `# 1. Install required packages
-!pip install -q albumentations opencv-python optuna requests sqlalchemy
-
-# 2. Download worker + client (token header required when broker uses --tunnel)
-import requests
-import os
-
-broker_url = "${baseUrl}"
-os.environ["HPO_BROKER_URL"] = broker_url
-${tokenLine}
-def _broker_headers():
-    h = {}
-    if "ngrok-free.app" in broker_url or "ngrok.io" in broker_url:
-        h["ngrok-skip-browser-warning"] = "1"
-    if os.environ.get("HPO_SECRET_TOKEN"):
-        h["X-HPO-Token"] = os.environ["HPO_SECRET_TOKEN"]
-    return h
-
-for _name in ("hpo_client.py", "colab_worker.py"):
-    _r = requests.get(f"{broker_url}/{_name}", headers=_broker_headers(), timeout=60)
-    _r.raise_for_status()
-    with open(_name, "w") as _f:
-        _f.write(_r.text)
-
-# 3. Import and run the reference worker loop
-from colab_worker import train_colab_trial_loop
-train_colab_trial_loop("${studyName}", n_trials=12, epochs=15)`;
-    } else {
-        const envExports = [];
-        if (workerEnv && typeof workerEnv === 'object') {
-            for (const [k, v] of Object.entries(workerEnv)) {
-                envExports.push(`os.environ["${k}"] = "${v}"`);
-            }
-        }
-        const envLines = envExports.length ? envExports.join("\n") + "\n" : "";
-
-        if (workerEntrypoint) {
-            snippet = `# 1. Install required packages
-!pip install -q optuna requests sqlalchemy
-
-# 2. Download HPO client (token header required when broker uses --tunnel)
-import requests
-import os
-
-broker_url = "${baseUrl}"
-os.environ["HPO_BROKER_URL"] = broker_url
-os.environ["HPO_STUDY_NAME"] = "${studyName}"
-${tokenLine}${envLines}
-def _broker_headers():
-    h = {}
-    if "ngrok-free.app" in broker_url or "ngrok.io" in broker_url:
-        h["ngrok-skip-browser-warning"] = "1"
-    if os.environ.get("HPO_SECRET_TOKEN"):
-        h["X-HPO-Token"] = os.environ["HPO_SECRET_TOKEN"]
-    return h
-
-for _name in ("hpo_client.py",):
-    _r = requests.get(f"{broker_url}/{_name}", headers=_broker_headers(), timeout=60)
-    _r.raise_for_status()
-    with open(_name, "w") as _f:
-        _f.write(_r.text)
-
-# 3. Run your training entrypoint (make sure your training script is uploaded to Colab)
-!${workerEntrypoint}`;
-        } else {
-            snippet = `# 1. Install required packages
-!pip install -q optuna requests sqlalchemy
-
-# 2. Download worker template + client (token header required when broker uses --tunnel)
-import requests
-import os
-
-broker_url = "${baseUrl}"
-os.environ["HPO_BROKER_URL"] = broker_url
-os.environ["HPO_STUDY_NAME"] = "${studyName}"
-${tokenLine}${envLines}
-def _broker_headers():
-    h = {}
-    if "ngrok-free.app" in broker_url or "ngrok.io" in broker_url:
-        h["ngrok-skip-browser-warning"] = "1"
-    if os.environ.get("HPO_SECRET_TOKEN"):
-        h["X-HPO-Token"] = os.environ["HPO_SECRET_TOKEN"]
-    return h
-
-for _name in ("hpo_client.py", "worker_minimal.py"):
-    _r = requests.get(f"{broker_url}/{_name}", headers=_broker_headers(), timeout=60)
-    _r.raise_for_status()
-    with open(_name, "w") as _f:
-        _f.write(_r.text)
-
-# 3. Fill in train_one_epoch inside worker_minimal.py and run
-# !python worker_minimal.py`;
-        }
-    }
-    
-    const snippetTextEl = document.getElementById("colab-snippet-text");
-    if (snippetTextEl) {
-        snippetTextEl.textContent = snippet;
-    }
-
-    let customSnippet = `export HPO_BROKER_URL="${baseUrl}"
-export HPO_STUDY_NAME="${studyName}"\n`;
-
-    if (workerEnv && typeof workerEnv === 'object') {
-        for (const [k, v] of Object.entries(workerEnv)) {
-            customSnippet += `export ${k}="${v}"\n`;
-        }
-    }
-    if (workerEntrypoint) {
-        customSnippet += `${workerEntrypoint}`;
-    } else {
-        customSnippet += `python worker_minimal.py`;
-    }
-
-    const customSnippetEl = document.getElementById("custom-worker-snippet");
-    if (customSnippetEl) {
-        customSnippetEl.textContent = customSnippet;
-    }
-}
-
-function syncColabUrl(val) {
-    const mainInputEl = document.getElementById("ngrok-url-input");
-    if (mainInputEl) {
-        mainInputEl.value = val;
-    }
-    updateColabSnippet();
-}
-
-function copyColabSnippet(btn) {
-    const snippetTextEl = document.getElementById("colab-snippet-text");
-    if (!snippetTextEl) return;
-    const snippetText = snippetTextEl.textContent;
-    navigator.clipboard.writeText(snippetText).then(() => {
-        const targetBtn = btn || document.querySelector("button[onclick^='copyColabSnippet']");
-        if (targetBtn) {
-            const originalText = targetBtn.textContent;
-            targetBtn.textContent = "Copied!";
-            setTimeout(() => {
-                targetBtn.textContent = originalText;
-            }, 2000);
-        }
-    }).catch(err => {
-        console.error("Could not copy text: ", err);
-    });
-}
-
-// Window exports
-window.updateColabSnippet = updateColabSnippet;
-window.syncColabUrl = syncColabUrl;
-window.copyColabSnippet = copyColabSnippet;
diff --git a/web/styles.css b/web/styles.css
index 748d2fd..325edd8 100644
--- a/web/styles.css
+++ b/web/styles.css
@@ -1278,7 +1278,7 @@
         .trial-table .col-trial { width: 60px; min-width: 60px; }
         .trial-table .col-state { width: 100px; }
         .trial-table .col-metric { width: 72px; }
-        .trial-table .col-dice-fixed { width: 72px; }
+        .trial-table .col-score-fixed { width: 72px; }
         .trial-table .col-lr { width: 64px; }
         .trial-table .col-bs { width: 44px; }
         .trial-table .col-res { width: 44px; }