diff --git a/.claude/skills/finish-branch/SKILL.md b/.claude/skills/finish-branch/SKILL.md new file mode 100644 index 0000000..0b32a7f --- /dev/null +++ b/.claude/skills/finish-branch/SKILL.md @@ -0,0 +1,69 @@ +--- +name: finish-branch +description: Use when wrapping up, finalizing, or merging a feature branch in this repo. Distils the branch's agent-generated plans/specs into a durable Architecture Decision Record when warranted (with human approval), then reminds the user to remove the plans/specs in a cleanup commit before the usual tests/merge/cleanup steps. +--- + +# Finish branch + +Finalize a feature branch in this repo. The distinctive step is **distilling the +branch's agent-generated plans/specs into a durable ADR** when warranted, then +removing those plans/specs in a cleanup commit before merge — see `AGENTS.md` +and `docs/adr/README.md` for the policy. This is advisory: surface it to the +user, don't enforce it. + +Do not start until the feature work itself is complete and the user is wrapping +up the branch. + +## Checklist + +Create a TodoWrite item per step and work them in order. + +1. **Confirm scope.** Identify the branch and its diff against `main` + (`git log --oneline main..HEAD`, `git diff main...HEAD --stat`). Confirm with + the user that the feature is done and ready to finalize. + +2. **Gather the working artifacts.** Find the agent-generated plans/specs for + this branch (typically under `docs/superpowers/plans/` and + `docs/superpowers/specs/`; they may or may not be committed). Read them with + the commit messages and the diff. You now have the full "what and why." + +3. **Decide if an ADR is warranted.** Apply the bar in `docs/adr/README.md`: + write one only if a future maintainer would ask "why is it like this?" and + the code won't answer — an architectural choice, a reversal of a prior + decision, a cross-cutting convention, or a non-obvious trade-off. + **Most branches warrant none.** If none: say so, with a one-line reason, and + skip to step 6. + +4. **Draft the ADR.** Copy `docs/adr/template.md` to the next number + `docs/adr/NNNN-short-title.md`. Fill in Context / Decision / Consequences / + Alternatives considered. Keep it short; link to the key commit(s)/PR and to + any superseded ADR. Add a row to the index table in `docs/adr/README.md`. + +5. **Get human approval of the ADR.** Show the user the drafted ADR and ask them + to confirm or edit before proceeding. Judgment about what is architecturally + relevant stays with the human. Revise until approved. + +6. **Verify green.** Run the checks CI runs and the branch touched: + `python -m pytest`, `flake8 . --max-line-length=127`, and `mypy optimizerapi`. + Report results honestly; do not finalize over failures without the user's + say-so (a known, documented pre-existing failure is the user's call). + +7. **Clean up the plans/specs.** If they were committed on the branch, remove + them in a dedicated cleanup commit so they don't land on `main` — their + durable content (if any) is now in the ADR, the rest stays in git history. + Keep them through review; the natural time to remove is just before merge. + **Don't enforce this** — surface it: remind the user and confirm, rather than + blocking the merge if they'd rather keep them. + +8. **Finalize.** Commit the ADR (and code, if uncommitted) on the branch, then + open a PR or merge per the user's preference and delete the branch. If the + `superpowers:finishing-a-development-branch` skill is available, use it for + this git mechanics step. End git commit messages with the repo's required + `Co-Authored-By` trailer. + +## Notes + +- One ADR per branch is a smell — recording nothing is the common, healthy case. +- Don't invent a timeframe or follow-up the branch didn't actually create. +- Respect the repo's commit-signing setup; if it was temporarily disabled for a + history rewrite, restore it before the user signs. diff --git a/.devcontainer-allowlist b/.devcontainer-allowlist new file mode 100644 index 0000000..a3be5d8 --- /dev/null +++ b/.devcontainer-allowlist @@ -0,0 +1,13 @@ +registry-1.docker.io +auth.docker.io +production.cloudfront.docker.com +ghcr.io +pkg-containers.githubusercontent.com +deb.debian.org +security.debian.org +pypi.org +files.pythonhosted.org +download.pytorch.org +download-r2.pytorch.org +github.com +codeload.github.com diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..f265479 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,25 @@ +# Keep the build context small and reproducible: the image installs deps from +# pyproject.toml via uv, so local virtualenvs, VCS, tests, and docs are noise. +.git +.github +.venv +env +*.egg-info +__pycache__ +**/__pycache__ +*.pyc +.mypy_cache +.pytest_cache +.ruff_cache +tests +docs +scripts +.claude +.sisyphus +*.md +!README.md +mise.toml +.flake8 +pytest.ini +Dockerfile +.dockerignore diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..bb82ebc --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +max-line-length = 127 +# Keep flake8's built-in excludes and skip the local virtualenv and build +# artifacts so `flake8 .` lints only project sources (matches CI). +extend-exclude = .venv,env,build,dist,*.egg-info diff --git a/.github/workflows/github-actions-docker.yaml b/.github/workflows/github-actions-docker.yaml index 88360db..076c1b2 100644 --- a/.github/workflows/github-actions-docker.yaml +++ b/.github/workflows/github-actions-docker.yaml @@ -20,11 +20,11 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Docker meta id: meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v5 with: # list of Docker images to use as base name for tags images: | @@ -39,30 +39,30 @@ jobs: type=semver,pattern={{major}}.{{minor}} type=semver,pattern={{major}} type=sha - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - with: - platforms: all - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v6 with: context: . push: true - platforms: linux/amd64,linux/arm64 + # amd64 only — arm64 was emulated under QEMU and dominated build time. + platforms: linux/amd64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + # Reuse layers across runs so the heavy dependency layer is not rebuilt + # when only application code changes. + cache-from: type=gha + cache-to: type=gha,mode=max build-args: | GITHUB_REF_NAME GITHUB_SHA diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index c75e6ee..80f522e 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -1,6 +1,5 @@ -# This workflow will install Python dependencies, run tests and lint with a single version of Python -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - +# Installs dependencies, lints, type-checks, and tests using the same +# toolchain as local development (mise-managed Python + uv; see mise.toml). name: Python application on: @@ -15,22 +14,23 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.9 - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - name: Set up toolchain with mise (Python + uv, matches dev env) + uses: jdx/mise-action@v2 with: - python-version: 3.9 + install: true + cache: true - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 pytest - if [ -f requirements-freeze.txt ]; then pip install -r requirements-freeze.txt; fi + # mise.toml provisions Python 3.13 + uv and creates .venv; install the + # project with its dev extras (pytest, flake8, mypy) into that venv. + run: mise exec -- uv pip install -e ".[dev]" - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + mise exec -- flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + mise exec -- flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Type-check with mypy + run: mise exec -- mypy optimizerapi - name: Test with pytest - run: | - pytest + run: mise exec -- python -m pytest diff --git a/.gitignore b/.gitignore index 6036b2a..97d118f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,15 @@ .vscode /tmp version.txt -.env \ No newline at end of file +.env + +*.egg-info/ +dist/ +build/ +*.egg +.sisyphus +# Ignore local Claude Code settings, but keep shared skills tracked. +/.claude/* +!/.claude/skills/ +/.claude/skills/* +!/.claude/skills/finish-branch/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..cd23747 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,68 @@ +# AGENTS + +- Before large refactors, run `python -m pytest`, `flake8 . --max-line-length=127`, and `mypy optimizerapi` to match CI. +- Consult `README.md` alongside this file for end-user setup and deployment details. +- Dev server runs on port 9090; Swagger UI at `/v1.0/ui/`. +- Follow existing style: double quotes for JSON-like payload keys, snake_case for functions/variables, CapWords for classes. +- For development dependencies: `uv pip install -e ".[dev]"` +- Handle errors explicitly; avoid bare `except:` and log or re-raise with context where appropriate. +- Keep functions small and focused; share logic between API handlers and tests instead of duplicating. +- Keep module-level constants UPPER_SNAKE_CASE; avoid one-letter names except simple indices. +- Lint locally with `flake8 . --max-line-length=127` (matches CI workflow). +- No Cursor or GitHub Copilot instruction files are present; if added later, update this AGENTS file to reference them. +- Prefer pure functions where practical; avoid side effects in import time except for necessary configuration. +- Prefer standard-library imports, then third-party, then local; keep tests mirroring src layout. +- Preserve current behavior around environment variables (e.g. `FLASK_ENV`, `CORS_ORIGIN`, `USE_WORKER`). +- Run a single test by node id, e.g. `python -m pytest tests/test_optimizer.py::test_can_be_run_without_data`. +- Run a single test module via `python -m pytest tests/test_optimizer.py`. +- Run the full test suite with `python -m pytest` (watch mode: `ptw`). +- Run the worker (requires Redis) with `python -m optimizerapi.worker`. +- Start the dev server with `python -m optimizerapi.server` (see README.md). +- Type-check locally with `mypy optimizerapi` (matches CI workflow); see `[tool.mypy]` in `pyproject.toml`. +- Type hints are required on new or changed public functions; keep signatures simple and ensure `mypy optimizerapi` passes. +- Use Python 3.13 (pinned in `mise.toml`); with [mise](https://mise.jdx.dev/) installed, `mise install` provisions Python, `uv`, and `.venv` automatically. Without mise, install Python 3.13 and `uv` manually, then `uv pip install -e .`. +- When adding dependencies, update `pyproject.toml` in the `dependencies` array + (or `dev` in `optional-dependencies` for dev tools) and verify with `git diff` + (see README.md). +- When modifying the API, update `optimizerapi/openapi/specification.yml` and keep handler names consistent. +- Write tests under `tests/` using `pytest` style; use `unittest.mock.patch` for external effects as in `tests/test_optimizer.py`. + +## Decisions and finishing a branch + +- **ADRs are the only durable design doc on `main`.** Significant decisions live + as Architecture Decision Records under `docs/adr/` (Nygard format); see + `docs/adr/README.md` for the bar and the workflow. Write one only when a future + maintainer would ask "why is it like this?" and the code won't answer. Most + branches need none — one ADR per branch is a smell. +- **Agent-generated plans/specs are branch-only working artifacts.** The + spec/plan files produced while building (e.g. under `docs/superpowers/`) may + stay on the branch and in the PR for review, but are removed in a cleanup + commit before merge so they don't accumulate on `main`. Their durable content, + if any, is distilled into an ADR first. +- **Branch-finalize flow** (advisory — the agent surfaces it, nothing enforces + it): when wrapping up a branch, (1) decide whether the work warrants an ADR and + draft it with human approval, (2) run the CI checks (`python -m pytest`, + `flake8 . --max-line-length=127`, `mypy optimizerapi`), (3) remove the + branch's plans/specs in a cleanup commit, then (4) open the PR / merge. The + Claude Code `finish-branch` skill (`.claude/skills/finish-branch/`) encodes + this; this policy is vendor-neutral and applies under any agent framework. + +## Architecture + +This is an OpenAPI-first REST API wrapping [ProcessOptimizer](https://github.com/novonordisk-research/ProcessOptimizer) (Bayesian optimization). The API has a single main endpoint: `POST /optimizer`. + +**Request flow:** Connexion validates requests against `optimizerapi/openapi/specification.yml`, routes to handler via `operationId`, handler dispatches to optimizer core logic. + +Key modules: +- `optimizerapi/server.py` — Flask/Connexion app init, CORS, Waitress (prod) vs Flask dev server +- `optimizerapi/openapi/specification.yml` — OpenAPI 3.0 spec; defines all schemas and wires handlers via `operationId` +- `optimizerapi/optimizer_handler.py` — HTTP handler; optionally routes work through Redis/RQ job queue (`USE_WORKER=true`) with SHA256-based dedup +- `optimizerapi/optimizer.py` — Core logic: runs optimizer, generates plots (base64 PNG or JSON), handles pickled model reuse, multi-objective support +- `optimizerapi/securepickle/` — Fernet encryption for pickled optimizer state (`PICKLE_KEY` env var) +- `optimizerapi/auth.py` — Optional Keycloak OIDC or static API key auth + +**Multi-objective:** When `yi` arrays have >1 element, the optimizer runs in multi-objective mode with separate per-objective plots (`objective_1_*`, `objective_2_*`) and pareto front data. + +**Pickled model flow:** Client sends `extras.pickled` to skip expensive GP retraining. If unpickling fails (corrupt, wrong key, schema change), falls back to full run with a warning log. + +**Plot formats:** `extras.graphFormat` controls output — `"png"` returns base64 PNG, `"json"` returns structured plot data. JSON mode uses `_get_brownie_bee_1d_plot_safe()` which works around a ProcessOptimizer bug with categorical dimensions. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..cbd4978 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,10 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +Source of truth for setup, workflow, style, and architecture: @AGENTS.md. + +When wrapping up or merging a feature branch, use the `finish-branch` skill +(`.claude/skills/finish-branch/`): distil any durable decision into an ADR under +`docs/adr/`, then clean up the branch's plans/specs before merge. See the +"Decisions and finishing a branch" section of @AGENTS.md. diff --git a/Dockerfile b/Dockerfile index 79d40a4..20cae2d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,43 +1,57 @@ +# syntax=docker/dockerfile:1 ARG GITHUB_REF_NAME=develop ARG GITHUB_SHA=local -# First stage -FROM python:3.9-bullseye AS builder -RUN python3 -m venv /opt/venv -ENV PATH="/opt/venv/bin:${PATH}" - -RUN pip install --upgrade pip && pip install pip-tools && pip install --upgrade pip - -COPY requirements-freeze.txt . -RUN cat requirements-freeze.txt | grep --invert-match pkg_resources > requirements-fixed.txt -RUN pip install -r requirements-fixed.txt - -# Second stage - -FROM python:3.9-bullseye +# ---- Builder: resolve and install dependencies with uv (matches dev/CI) ---- +FROM python:3.13-slim-bookworm AS builder + +# git is required to install ProcessOptimizer from its git ref (see pyproject.toml). +RUN apt-get update \ + && apt-get install -y --no-install-recommends git \ + && rm -rf /var/lib/apt/lists/* + +# uv: fast, parallel resolver — the same tool used locally and in CI. +COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv + +ENV VIRTUAL_ENV=/opt/venv \ + UV_COMPILE_BYTECODE=1 \ + UV_LINK_MODE=copy +RUN uv venv "$VIRTUAL_ENV" +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +WORKDIR /src +# setuptools needs the readme (project metadata) and the package source +# (tool.setuptools.packages.find) to build the project; copy both before install. +COPY pyproject.toml README.md ./ +COPY optimizerapi/ ./optimizerapi/ +# Cache mount keeps uv's download/build cache across builds even when this layer +# is rebuilt, so the ProcessOptimizer git dependency isn't re-fetched every time. +RUN --mount=type=cache,target=/root/.cache/uv uv pip install . + +# ---- Runtime ---- +FROM python:3.13-slim-bookworm ARG GITHUB_REF_NAME ARG GITHUB_SHA -COPY --from=builder /opt/venv /opt/venv -WORKDIR /code -ENV VERSION=${GITHUB_REF_NAME} -ENV SHA=${GITHUB_SHA} -# add non-root user -RUN addgroup --system user && adduser --system --no-create-home --group user -RUN chown -R user:user /code && chmod -R 755 /code -RUN mkdir -p /code/mapplotlib - -USER user - -COPY --from=builder /requirements-fixed.txt /code/requirements-freeze.txt -#COPY version.txt /code -RUN echo "${VERSION}-${SHA}" > /code/version.txt +WORKDIR /code +ENV VERSION=${GITHUB_REF_NAME} \ + SHA=${GITHUB_SHA} \ + FLASK_ENV=production \ + MPLCONFIGDIR=/tmp/matplotlib \ + PATH=/opt/venv/bin:${PATH} + +# Dependencies come from the builder venv; the app itself runs from /code where +# the full source tree (incl. optimizerapi/openapi/specification.yml) is present. +COPY --from=builder /opt/venv /opt/venv +COPY pyproject.toml /code/pyproject.toml COPY optimizerapi/ /code/optimizerapi -ENV FLASK_ENV=production -ENV MPLCONFIGDIR=/tmp/mapplotlib +# Run as a non-root user with a version stamp baked in. +RUN addgroup --system user \ + && adduser --system --no-create-home --group user \ + && echo "${VERSION}-${SHA}" > /code/version.txt \ + && chown -R user:user /code -ENV PATH=/opt/venv/bin:${PATH} -VOLUME /code/matplotlib +USER user -CMD [ "python", "-m", "optimizerapi.server" ] \ No newline at end of file +CMD [ "python", "-m", "optimizerapi.server" ] diff --git a/README.md b/README.md index c2dec66..c871591 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,20 @@ This project expose a REST based API for [ProcessOptimizer](https://github.com/n If you have Docker installed the API can be started locally, in development mode, by running the script `build-and-run.sh` -Alternatively the project can be build and run with the following commands: +Alternatively the project can be built and run locally. - python3 -m venv env - source env/bin/activate - pip install --upgrade pip +The toolchain (Python 3.13 + [`uv`](https://github.com/astral-sh/uv)) is +pinned in `mise.toml`. With [mise](https://mise.jdx.dev/) installed +(see the [getting started guide](https://mise.jdx.dev/getting-started.html)), +`cd` into the repo provisions Python, `uv`, and a `.venv` automatically — +just run `mise install` once. Without mise, install Python 3.13 and `uv` +yourself (`brew install uv` or `pip install uv`) and create the venv +manually: - pip install -r requirements-freeze.txt + python3 -m venv .venv + source .venv/bin/activate + + uv pip install -e . python -m optimizerapi.server Now open [http://localhost:9090/v1.0/ui/](http://localhost:9090/v1.0/ui/) in a browser to explore the API through Swagger UI @@ -108,20 +115,21 @@ Keycloak is configured using the following environement variables # Adding or updating dependencies -When adding a new dependency, you should manually add it to `requirements.txt` and then run the following commands: +This project uses `pyproject.toml` for dependency management following PEP 621 standards. - pip install -r requirements.txt - pip freeze | grep --invert-match pkg_resources > requirements-freeze.txt +When adding a new dependency: -Now you should check if the freeze operation resulted in unwanted upates by running: +1. Add it to the `dependencies` array in `pyproject.toml` with a version constraint +2. For development dependencies, add to `[project.optional-dependencies]` under `dev` +3. Install the updated dependencies: - git diff requirements-freeze.txt + uv pip install -e . -After manually fixing any dependencies, you should run: + Or for development dependencies: - pip install -r requirements-freeze.txt + uv pip install -e ".[dev]" -Remember to commit both the changed `requirements.txt` and `requirements-freeze.txt` files. +4. Commit the changed `pyproject.toml` file # Updating the change log diff --git a/docs/adr/0001-pickled-model-cache-contract.md b/docs/adr/0001-pickled-model-cache-contract.md new file mode 100644 index 0000000..cd32565 --- /dev/null +++ b/docs/adr/0001-pickled-model-cache-contract.md @@ -0,0 +1,71 @@ +# 0001. Pickled model is an advisory, fingerprint-validated cache hint + +- **Status:** accepted +- **Date:** 2026-05-30 +- **Deciders:** Jakob Langdal + +## Context + +The pareto-point UI flow re-renders single plots whenever a user clicks a point +on the front. Re-running the Bayesian optimizer (GP `tell` / retraining) on +every such request is expensive, so the API lets a client round-trip an opaque +`extras.pickled` blob produced by a previous response to skip that work. + +The risk is that `pickled` carries optimizer state that could silently diverge +from the request's authoritative inputs (`data`, `optimizerConfig`) — a corrupt +blob, a stale one from a different dataset, or one encrypted under a rotated +`PICKLE_KEY`. If the server trusted that state, two requests with identical +inputs could return different results depending on whether a cache hint was +attached, which is impossible to debug from the client side. + +## Decision + +We will treat the pickled model as an **efficiency feature only**, never as +authoritative state. See `optimizerapi/pickled_state.py` and its use in +`optimizerapi/optimizer.py:run`. + +- **Equivalence guarantee.** For any request `R`, the responses to `R` and to + `R ∪ {extras.pickled: P}` are observably equivalent (ignoring `result.pickled` + and timing), provided `P` came from an earlier response whose `data` and + `optimizerConfig` matched `R`. The full round-trip MUST work without `pickled`, + just slower. The client always sends `data`; the server never reads inputs out + of the blob. +- **Fingerprint validation.** The blob stores + `sha256_hex(canonical_json({data, optimizerConfig}))`. On an incoming + `pickled`, the server decrypts → checks structure → compares the fingerprint + to the current request, and only then takes the fast path. Any failure falls + through to a full run with a single `warning` tagged `decrypt_failed`, + `bad_structure`, or `fingerprint_mismatch`. +- **Observability.** `result.extras.pickledUsed` (boolean) reports whether the + fast path was taken, so the UI need not infer it from timing. +- **No in-tree payload versioning.** Rotating `PICKLE_KEY` makes pre-existing + blobs undecryptable, and the decrypt-failure fall-through handles them — so no + migration code is carried. +- **Adjacent request semantics** fixed at the same time: `selectedPoint` is + honored only on the `graphFormat: "json"` path (logged-and-ignored on PNG), + and `includeModel: "false"` with `pickled` is honored verbatim (fast path, + empty `result.pickled`, warning logged) rather than auto-overridden. + +## Consequences + +- Identical inputs always produce identical results; `pickled` can only make a + response faster, never different. This is enforceable by an equivalence test. +- Fall-through is safe by construction, so a bad or stale blob degrades to a + correct (slower) answer instead of an error or wrong output. +- Fingerprinting on the raw request fields keeps the contract stable across + internal refactors of `space` / `hyperparams` / `constraints`. +- Cost: every fast path pays a decrypt + sha256 + structural check, and the blob + is opaque and bound to the current key — clients cannot reuse it across a key + rotation. Accepted as cheap relative to a GP refit. + +## Alternatives considered + +- **Trust optimizer state inside `pickled`.** Rejected: breaks the equivalence + guarantee and makes client-visible behavior depend on an opaque blob. +- **`selectedPoint` as an index into the prior response's `front_x_data`.** + Rejected in favor of raw X-space coordinates: stateless, no coupling to prior + response shape, no dependency on deterministically re-deriving the front. +- **A payload `version` field with in-tree migration.** Rejected: key rotation + plus fall-through already covers stale blobs without migration code. +- **Server-side session cache keyed by a short `pickleHandle`.** Deferred (out + of scope); the opaque-blob contract leaves a clean swap path if needed later. diff --git a/docs/adr/0002-adrs-and-ephemeral-agent-docs.md b/docs/adr/0002-adrs-and-ephemeral-agent-docs.md new file mode 100644 index 0000000..7f9bc97 --- /dev/null +++ b/docs/adr/0002-adrs-and-ephemeral-agent-docs.md @@ -0,0 +1,50 @@ +# 0002. Record decisions in ADRs; treat agent plans/specs as branch-only + +- **Status:** accepted +- **Date:** 2026-05-30 +- **Deciders:** Jakob Langdal + +## Context + +Agent-driven development (the superpowers brainstorming / writing-plans +workflow, and similar tools) produces a spec and a plan for essentially every +feature — this repo already carries several under `docs/superpowers/`. Committing +all of them to `main` accumulates ephemeral, quickly-stale documents that bury +the few decisions that actually matter. We wanted a durable record of _why_ the +code is shaped as it is, without the noise of _how_ each feature was built — and +the convention had to work across agent frameworks, not just one tool. + +## Decision + +We will keep **one** durable design document on `main`: Architecture Decision +Records under `docs/adr/` (Nygard format). + +- **Agent-generated plans/specs are working artifacts.** They may live on the + branch — and in the PR — during development and review, but are removed in a + cleanup commit before merge. They are not kept on `main`. +- **At branch finalization, distil any durable decision into an ADR**, with a + human approving it. Don't force one per branch; most branches record none. +- **This is advisory, not enforced.** Plans/specs are not git-ignored and there + is no CI gate; the agent surfaces the cleanup/distillation and the human + decides. +- Instructions live in `AGENTS.md` (vendor-neutral), imported by `CLAUDE.md`. + The Claude Code `finish-branch` skill encodes the procedure. + +## Consequences + +- `main` carries only high-signal decision records; plans/specs don't rot there. +- The convention is portable across agent frameworks via `AGENTS.md`. +- Reviewers still see the plans/specs in the PR, before the cleanup commit. +- Because it's advisory, a branch could merge with stale specs still committed, + or without an ADR that arguably should exist. Accepted: we preferred low + ceremony over enforcement for a small team. The `finish-branch` reminder is + the mitigation. + +## Alternatives considered + +- **Keep specs/plans committed permanently** (the original workflow default). + Rejected: accumulates noise and stale docs, and buries the "why". +- **Git-ignore specs/plans entirely** (never committed). Rejected: reviewers + wouldn't see them in the PR. We chose branch-visible + cleanup-before-merge. +- **Mechanically enforce the cleanup** (git-ignore, or a CI/hook gate). + Rejected: too much ceremony for a small team; an advisory reminder is enough. diff --git a/docs/adr/0003-cpu-only-torch.md b/docs/adr/0003-cpu-only-torch.md new file mode 100644 index 0000000..521861f --- /dev/null +++ b/docs/adr/0003-cpu-only-torch.md @@ -0,0 +1,51 @@ +# 0003. Pin torch to the CPU-only wheel + +- **Status:** accepted +- **Date:** 2026-05-30 +- **Deciders:** Jakob Langdal + +## Context + +ProcessOptimizer (a core dependency, installed from a git ref) pulls `torch` in +transitively. On Linux, the default `torch` wheel on PyPI is the CUDA build, +which drags in the full NVIDIA CUDA stack (`nvidia-cublas`, `cudnn`, `nccl`, +`cusolver`, `cusparse`, ...) — roughly 5 GB. The API only ever runs CPU +inference, so none of that is used. It was the dominant cost behind slow Docker +builds and produced a ~6 GB image. + +## Decision + +We will pin `torch` to PyTorch's CPU-only index. See `pyproject.toml` +(`[[tool.uv.index]]` `pytorch-cpu` + `[tool.uv.sources]`) and commit `aa490b0`. + +A subtlety drove the shape of this: **uv's `[tool.uv.sources]` index pins apply +only to _direct_ dependencies, not transitive ones** (verified empirically). So +we also declare `torch` explicitly in `[project.dependencies]` (unconstrained, +so uv unifies it with ProcessOptimizer's requirement) and map it to the +`pytorch-cpu` index via `[tool.uv.sources]`. Resolution then yields +`torch==2.12.0+cpu` with zero CUDA packages. + +## Consequences + +- Image drops from ~6 GB to ~1.42 GB; builds and CI are markedly faster. The fix + applies uniformly to the Docker image, CI, and local dev because all three + install via uv from the same `pyproject.toml`. +- The build now depends on `download.pytorch.org` **and** `download-r2.pytorch.org` + (PyTorch redirects the actual wheel/metadata blobs to its R2 backend) being + reachable — relevant only where egress is filtered. +- `torch` is now a direct dependency we must keep loosely in step with what + ProcessOptimizer expects; leaving it unconstrained lets uv resolve the unified + version. +- If GPU inference is ever wanted, this pin must be revisited. + +## Alternatives considered + +- **Accept the default CUDA `torch`.** Rejected: ~5 GB of libraries we never use, + for a CPU-only workload. +- **`[tool.uv.sources]` pin without declaring `torch` directly.** Rejected: source + pins don't apply to transitive dependencies, so it had no effect on resolution + (confirmed by testing). +- **Install CPU `torch` only inside the Dockerfile** (e.g. `uv pip install torch + --index-url .../cpu` before the project). Rejected: it would slim the image but + leave dev and CI installs on the CUDA build. The `pyproject.toml` approach fixes + all three in one place. diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 0000000..264e26d --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,40 @@ +# Architecture Decision Records + +This folder is the durable record of significant decisions in this repo. Each +ADR captures one decision: the context that forced it, what was decided, the +consequences, and the alternatives that were rejected. + +ADRs are the **only** design document we keep on `main` long-term. The +agent-generated plans and specs produced while building (e.g. under +`docs/superpowers/`) may live on the branch for review, but are removed in a +cleanup commit before merge (see `AGENTS.md`). + +## When to write one + +Write an ADR when a future maintainer would ask _"why is it like this?"_ and the +code alone won't answer. Typical triggers: + +- An architectural choice with viable alternatives. +- Reversing or superseding an earlier decision. +- A cross-cutting convention or constraint. +- A trade-off that is non-obvious from the code. + +**Don't** write one for routine features, bug fixes, or refactors. Most branches +record no ADR — that's expected. One ADR per branch is a smell. + +## How + +1. Copy `template.md` to `NNNN-short-title.md` — `NNNN` is the next zero-padded + number. +2. Fill it in. Keep it short; link to code, PRs, or superseded ADRs. +3. Set `Status` (`accepted` once agreed). When a later ADR overrides this one, + set this one's status to `superseded by ADR-XXXX` and reference it. +4. Add a line to the index below. + +## Index + +| ADR | Title | Status | +| -------------------------------------------- | ---------------------------------------------------------------- | -------- | +| [0001](0001-pickled-model-cache-contract.md) | Pickled model is an advisory, fingerprint-validated cache hint | accepted | +| [0002](0002-adrs-and-ephemeral-agent-docs.md) | Record decisions in ADRs; treat agent plans/specs as branch-only | accepted | +| [0003](0003-cpu-only-torch.md) | Pin torch to the CPU-only wheel (slim image, faster builds) | accepted | diff --git a/docs/adr/template.md b/docs/adr/template.md new file mode 100644 index 0000000..05d69b3 --- /dev/null +++ b/docs/adr/template.md @@ -0,0 +1,25 @@ +# NNNN. + +- **Status:** proposed | accepted | superseded by ADR-XXXX | deprecated +- **Date:** YYYY-MM-DD +- **Deciders:** + +## Context + +What forced this decision? The problem, constraints, and relevant facts. Enough +that a reader who wasn't there understands the pressure behind the choice. + +## Decision + +What we decided, stated plainly in the active voice ("We will …"). Link to the +key code, PR, or commit. + +## Consequences + +What becomes easier and what becomes harder as a result. Include the costs and +risks, not just the upside — breaking changes, follow-ups, things we accept. + +## Alternatives considered + +The options we rejected and, briefly, why. This is often the most valuable part +of the record. diff --git a/docs/api-usage.md b/docs/api-usage.md new file mode 100644 index 0000000..046b143 --- /dev/null +++ b/docs/api-usage.md @@ -0,0 +1,344 @@ +# Using the Process Optimizer API + +This document walks through the end-to-end process of driving an experiment with the Process Optimizer API. It covers the request shape, the response shape, the suggested experiment loop, multi-objective workflows with pareto-point exploration, and the performance contract around cached model state. + +The API is OpenAPI-first. For machine-readable details and an interactive sandbox, see the Swagger UI at `http://localhost:9090/v1.0/ui/`. This document describes the *process* of using the API — what to send, what comes back, and how successive calls fit together. + +## 1. What the API does + +The Process Optimizer API wraps [ProcessOptimizer](https://github.com/novonordisk-research/ProcessOptimizer) — a Bayesian optimization library — behind a single REST endpoint: + +``` +POST /v1.0/optimizer +``` + +You describe an experiment space and the data you've collected so far, and the server returns the next experiment(s) to run, plus diagnostic plots of the model's current beliefs. Each call is stateless: the full history is sent in the request body. A cache hint (`extras.pickled`) lets you skip retraining on repeat calls without affecting correctness. + +The endpoint is the same for single- and multi-objective experiments — multi-objective is inferred from the shape of `yi`. + +## 2. Running the server + +For development: + +```bash +uv pip install -e . +python -m optimizerapi.server +``` + +The server listens on port `9090` with Swagger UI at `/v1.0/ui/`. See `README.md` for production deployment, Redis-backed job queueing, CORS, and auth configuration. + +## 3. Authentication + +The endpoint accepts either of: + +- **Static API key** as the `apikey` query parameter. Set `AUTH_API_KEY` on the server; clients add `?apikey=` to the URL. +- **Keycloak OIDC** bearer token in `Authorization: Bearer `. See `README.md` for the relevant `AUTH_*` env vars. + +If neither is configured, the server still requires the `apikey` query parameter — pass any value (e.g. `?apikey=none`) for local development. + +## 4. The request body + +The endpoint accepts a JSON object with three top-level fields: + +```json +{ + "data": [ ... measurement history ... ], + "optimizerConfig": { ... search space and BO hyperparameters ... }, + "extras": { ... output and caching options ... } +} +``` + +`data` and `optimizerConfig` are **authoritative inputs** — together they define what the server computes. `extras` shapes the output (which plots, which format) and provides a cache hint to make repeat calls faster; it never affects correctness. + +### 4.1 `optimizerConfig` — the search space and BO hyperparameters + +```jsonc +{ + "baseEstimator": "GP", // "GP" or another ProcessOptimizer base estimator + "acqFunc": "EI", // "EI" | "PI" | "LCB" | "gp_hedge" ... + "initialPoints": 3, // # of random samples before model takes over + "kappa": 1.96, // exploration weight for LCB + "xi": 0.01, // improvement threshold for EI / PI + "space": [ ... ], // list of dimensions, see below + "constraints": [ ... ] // optional, see below +} +``` + +Each dimension in `space` has one of three types: + +```jsonc +// continuous (float) +{ "type": "continuous", "name": "Sugar", "from": 0, "to": 100 } + +// discrete (integer) +{ "type": "discrete", "name": "Temperature", "from": 0, "to": 300 } + +// categorical +{ "type": "category", "name": "Finish", "categories": ["None", "Frosting", "Whipped cream"] } +``` + +`constraints` is an optional list of sum-constraints across dimensions: + +```jsonc +[ + { "type": "sum", "dimensions": [0, 1], "value": 200 } // dim[0] + dim[1] == 200 +] +``` + +### 4.2 `data` — the measurement history + +A list of `{xi, yi}` pairs. `xi` is one experiment's coordinates in the same order as `space`. `yi` is a **list** of measured outcomes — length 1 for single-objective, length ≥ 2 for multi-objective. The optimizer minimizes each `yi` component. + +```jsonc +{ + "data": [ + { "xi": [50, 833, 150, 60, "Whipped cream"], "yi": [-3, -6] }, + { "xi": [16.7, 500, 250, 20, "None"], "yi": [-2, -17] } + ] +} +``` + +For the very first call you have no measurements yet — send `"data": []`. The server returns initial-point suggestions instead. + +> **Maximize instead of minimize?** Negate the value client-side. The optimizer always minimizes. + +### 4.3 `extras` — output and cache options + +All fields are optional. The most useful ones: + +| Field | Meaning | Default | +| ----------------------------- | --------------------------------------------------------------------------------------------- | --------------------------------------------- | +| `experimentSuggestionCount` | How many next-experiments to return | `1` | +| `graphFormat` | `"png"` (base64-encoded images) or `"json"` (structured plot data) | `"png"` | +| `graphs` | Which plots to compute: subset of `["objective", "convergence", "pareto", "single"]` | `["objective", "convergence", "pareto", "single"]` | +| `maxQuality` | Render quality for PNG plots | `5` | +| `objectivePars` | `"result"` or `"expected_minimum"` — where to evaluate the objective plot | `"result"` | +| `includeModel` | String `"true"` / `"false"` — whether to populate `result.pickled` in the response | `"true"` | +| `selectedPoint` | Override the highlight point in single plots with explicit X-space coordinates. JSON only. | none | +| `pickled` | Cache hint from a previous response. The server validates it against the current request. | none | + +Notes on a few less-obvious knobs: + +- `graphFormat` controls the plot encoding. `"png"` produces classic image plots and is intended for human-facing UIs that display PNGs. `"json"` returns structured per-dimension series for clients that render plots themselves (e.g. a React-based UI). +- `selectedPoint` is honored **only** on the JSON path. On the PNG path it is logged-and-ignored. +- `includeModel: "false"` skips serializing the model into the response, saving bandwidth. The next call then has to do a full retrain (no cache available) — the server logs a warning when it sees `includeModel: "false"` together with an incoming `pickled`, since this combination silently breaks the cache chain. + +## 5. The response body + +```jsonc +{ + "plots": [ + { "id": "single_0_0", "plot": "" }, + { "id": "convergence_0", "plot": "<...>" }, + { "id": "pareto_data", "plot": "<...>" } + // ... more + ], + "result": { + "next": [ [50, 833, 150, 60, "Whipped cream"] ], // suggested experiments + "models": [ { "expected_minimum": [...], "extras": {} } ], + "expected_minimum": [...], + "pickled": "", + "extras": { + "pickledUsed": false, // fast path engaged? + "parameters": { ... echo of the request inputs ... }, + "version": "..." + } + } +} +``` + +Plot ids by mode: + +| Mode | Plot ids | +| ------------------------------- | ----------------------------------------------------------------------------- | +| Single-objective, `png` | `single_0`, `convergence_0`, `objective` plots (when available) | +| Single-objective, `json` | `single_0_0`, `single_0_1`, ..., `single_0_` (per-dim series), `single_0_` (histogram entry) | +| Multi-objective | `pareto_data` plus per-objective single/objective plots (`objective_1_*`, `objective_2_*`) | + +The `next` field is what you feed back into the next iteration: run those experiments, record the results, append to `data`, and call again. + +## 6. The optimization loop + +A typical sequence: + +1. **Round 1 — no data yet.** POST with `"data": []` and your `optimizerConfig`. The server returns one or more initial-point suggestions in `result.next`. +2. **Run the experiments.** Measure `yi` for each suggested `xi`. +3. **Round 2..N.** POST again with the accumulated `data` (all prior `{xi, yi}` pairs). The server fits a model, returns plots, and proposes the next experiment in `result.next`. +4. **(Optional) Pass the cache hint.** On rounds 2..N, include the previous response's `result.pickled` as `extras.pickled`. The server validates that the data and config still match what produced that cache; if they match, model fitting is skipped and the response comes back faster. +5. **Stop when you're satisfied** — when convergence flattens, your budget is exhausted, or `result.expected_minimum` is good enough. + +A minimal curl call for round 1: + +```bash +curl 'http://localhost:9090/v1.0/optimizer?apikey=none' \ + -X POST \ + -H 'Content-Type: application/json' \ + --data-raw '{ + "extras": {"experimentSuggestionCount": 1}, + "data": [], + "optimizerConfig": { + "baseEstimator": "GP", + "acqFunc": "gp_hedge", + "initialPoints": 3, + "kappa": 1.96, + "xi": 0.01, + "space": [ + {"type": "continuous", "name": "Red", "from": 0, "to": 255}, + {"type": "continuous", "name": "Green", "from": 0, "to": 255}, + {"type": "discrete", "name": "Blue", "from": 0, "to": 255} + ] + } + }' +``` + +Pre-built sample requests live in `scripts/`: + +- `sample.curl` — minimal single-objective, JSON plot format +- `sample-multi.curl` — multi-objective with multiple data points +- `sample-multi-with-selection.curl` — multi-objective with `selectedPoint` + `pickled` (the pareto-exploration flow) + +## 7. Multi-objective and pareto-point exploration + +When `yi` is a list of length ≥ 2, the optimizer runs in multi-objective mode. The response then includes: + +- A `pareto_data` plot entry with the pareto front in objective space (`front_y_data`), the corresponding points in X-space (`front_x_data`), per-objective uncertainty, and a `best_idx` highlighting a recommended compromise. +- Per-objective single plots (`objective_1_*`, `objective_2_*`). + +A common UI flow: + +1. Render the pareto front from `pareto_data`. +2. The user clicks a point on the pareto front. The UI looks up the corresponding `front_x_data[i]`. +3. The UI re-requests the optimizer endpoint with that coordinate list as `extras.selectedPoint` (and `graphFormat: "json"`). The server re-renders the single plots highlighting that point. +4. To keep this responsive, the UI also sends back the previous response's `result.pickled` as `extras.pickled` — the server skips the GP retraining step. + +The full process, end-to-end, is the curl pair shown in `scripts/sample-multi-with-selection.curl`. + +## 8. The pickled cache contract + +The `pickled` field is a performance optimization. Two guarantees: + +- **Equivalence.** For any request `R`, the response to `R` and to `R ∪ {extras.pickled: P}` are observably equivalent — same plots, same `next`, same `expected_minimum` — provided `P` was produced by an earlier response whose `data` and `optimizerConfig` matched `R`. Only `result.pickled` itself and the timing differ. +- **Fingerprint-validated.** The server embeds a hash of `(data, optimizerConfig)` inside the pickled blob at pack time. On a subsequent request the server recomputes the fingerprint and compares it to the one inside the blob. If they don't match — because the client sent a stale cache or changed the config — the cache is silently ignored and a full run is performed. + +`result.extras.pickledUsed` tells you which path the server took: `true` for the fast path, `false` for a full run. The UI can use this to surface latency expectations without measuring. + +Three things can cause a fall-through to a full run (each logs a single `WARNING`): + +| Reason | When | Logger / tag | +| ----------------------- | ----------------------------------------------------------------------------------- | ------------------------------------- | +| `decrypt_failed` | Blob cannot be decrypted (rotated `PICKLE_KEY`, corrupted bytes) | `optimizerapi.pickled_state` | +| `bad_structure` | Blob decrypts but isn't the expected `{fingerprint, result, next, optimizer}` shape | `optimizerapi.pickled_state` | +| `fingerprint_mismatch` | Blob decrypts and is well-formed, but the embedded fingerprint differs from the request | `optimizerapi.pickled_state` | + +Two other warnings, not cache fall-throughs, surface useful misuse signals: + +- `optimizerapi.optimizer`: `"selectedPoint ignored on png path"` — `selectedPoint` was sent with `graphFormat: "png"`. The field is JSON-only; on the PNG path it does nothing. +- `optimizerapi.optimizer`: `"includeModel=false with extras.pickled — next call will pay the full cost"` — the client used the cache to take the fast path but suppressed the new cache from the response, so the next call cannot reuse it. + +None of these conditions are errors. The HTTP response is always valid in all five cases. + +> **Why fingerprinting?** It guarantees the equivalence property mechanically: a client can never accidentally short-circuit the optimizer with a cache that was produced from different data, because such a cache is detected and ignored. + +## 9. Error handling + +The endpoint returns three status codes: + +- `200 OK` — Success. Body is the result schema described in §5. +- `400 Bad Request` — Validation, type, or I/O error in the request. +- `500 Internal Server Error` — Unexpected server error. + +Both error responses use the RFC 7807 problem+json shape produced by Connexion: + +```jsonc +{ + "title": "Bad request", // or "Internal server error" + "detail": "", // exception message or validator output + "status": 400, // or 500 + "type": "about:blank" +} +``` + +For health checks: `GET /v1.0/health` returns `200` if the service is reachable. + +## 10. End-to-end example: pareto-point exploration + +This is the same flow as `scripts/sample-multi-with-selection.curl`, narrated step by step. + +### Step 1 — initial multi-objective run + +The client sends 3 measurements and asks for JSON pareto and single plots. The server returns plots, a next-experiment suggestion, and a `result.pickled` cache blob. + +```bash +curl 'http://localhost:9090/v1.0/optimizer?apikey=none' \ + -X POST \ + -H 'Content-Type: application/json' \ + --data-raw '{ + "extras": { + "experimentSuggestionCount": 1, + "graphs": ["pareto", "single"], + "graphFormat": "json" + }, + "data": [ + {"xi": [16.7, 500, 250, 20, "None"], "yi": [-2, -17]}, + {"xi": [50, 833, 150, 60, "Whipped cream"], "yi": [-3, -6]}, + {"xi": [58.3, 22, 85, 6, "Frosting"], "yi": [-6, -25]} + ], + "optimizerConfig": { + "baseEstimator": "GP", + "acqFunc": "EI", + "initialPoints": 3, + "kappa": 1.96, + "xi": 2, + "space": [ + {"type": "continuous", "name": "Sugar", "from": 0, "to": 100}, + {"type": "continuous", "name": "Flour", "from": 0, "to": 1000}, + {"type": "discrete", "name": "Temperature", "from": 0, "to": 300}, + {"type": "discrete", "name": "Time", "from": 0, "to": 120}, + {"type": "category", "name": "Finish", "categories": ["None", "Frosting", "Whipped cream"]} + ], + "constraints": [] + } + }' +``` + +The response contains, among other things: + +- A `pareto_data` plot entry the UI uses to render the pareto front. +- `result.pickled` — store this for the next call. +- `result.extras.pickledUsed: false` — this was a full run. + +### Step 2 — the user clicks a pareto point + +The UI takes the coordinates of the clicked point (read from `pareto_data.front_x_data[i]`) and re-requests with `selectedPoint` set, including the previous `pickled` for speed: + +```bash +curl 'http://localhost:9090/v1.0/optimizer?apikey=none' \ + -X POST \ + -H 'Content-Type: application/json' \ + --data-raw '{ + "extras": { + "experimentSuggestionCount": 1, + "graphs": ["pareto", "single"], + "graphFormat": "json", + "selectedPoint": [50, 833, 150, 60, "Whipped cream"], + "pickled": "" + }, + "data": [ ... same as step 1 ... ], + "optimizerConfig": { ... same as step 1 ... } + }' +``` + +If `data` and `optimizerConfig` match what produced the cache, `result.extras.pickledUsed` comes back `true` and the single plots are re-rendered with the new highlight point — without retraining the GP. If anything has drifted, the server silently falls through to a full run and the response is still correct (just slower). + +That's the loop: keep accumulating measurements, keep round-tripping `pickled`, and reach for `selectedPoint` whenever a UI interaction needs a fresh single-plot view. + +## 11. Quick reference + +- **Endpoint:** `POST /v1.0/optimizer` +- **Auth:** `?apikey=` or `Authorization: Bearer ` +- **Request:** `{data, optimizerConfig, extras}` +- **Response:** `{plots[], result: {next, models, expected_minimum, pickled, extras: {pickledUsed, parameters, version}}}` +- **Health:** `GET /v1.0/health` +- **Swagger UI:** `http://localhost:9090/v1.0/ui/` +- **Sample requests:** `scripts/sample.curl`, `scripts/sample-multi.curl`, `scripts/sample-multi-with-selection.curl` +- **Design notes:** `docs/superpowers/specs/2026-05-18-pareto-extras-redesign-design.md` diff --git a/mise.toml b/mise.toml new file mode 100644 index 0000000..f5142b3 --- /dev/null +++ b/mise.toml @@ -0,0 +1,8 @@ +[tools] +python = "3.13" +uv = "latest" + +[env] +_.python.venv = { + path = ".venv", create = true, +} diff --git a/optimizerapi/auth.py b/optimizerapi/auth.py index fa41e02..61af6cc 100644 --- a/optimizerapi/auth.py +++ b/optimizerapi/auth.py @@ -1,12 +1,20 @@ -"""Authenitcation module +"""Authentication module. -This module will verify tokens provided bt a Keycloak OpenID server +Verifies bearer tokens issued by a Keycloak OpenID server, and the +static API key provided via ``AUTH_API_KEY``. """ +import logging import os +import secrets +from typing import Any, cast + from keycloak import KeycloakOpenID AUTH_API_KEY = os.getenv("AUTH_API_KEY", "none") AUTH_SERVER = os.getenv("AUTH_SERVER", None) +FLASK_ENV = os.getenv("FLASK_ENV", "development") + +_DEFAULT_APIKEY = "none" # sentinel: matches AUTH_API_KEY default; never accept in production AUTH_CLIENT_ID = os.getenv("AUTH_CLIENT_ID", None) AUTH_CLIENT_SECRET = os.getenv("AUTH_CLIENT_SECRET", None) AUTH_REALM_NAME = os.getenv("AUTH_REALM_NAME", None) @@ -18,38 +26,51 @@ client_secret_key=AUTH_CLIENT_SECRET, ) +_LOG = logging.getLogger(__name__) + -def token_info(access_token) -> dict: - """Verify token with authentication server +def token_info(access_token: str) -> dict | None: + """Verify a bearer token against the configured Keycloak server. Returns ------- dict - a dictionary containing sub and scope - None in case of invalid token + Token data from Keycloak introspection when the token is active. + If no OIDC server is configured, returns ``{"scope": []}``. + None + If the server reports the token as inactive. """ - print(access_token) if not AUTH_SERVER: return {"scope": []} - token = access_token - token_data = keycloak_openid.introspect(token) - if "active" in token_data and token_data["active"]: - print("OK") + # keycloak_openid.introspect returns Any; narrow to dict so callers stay typed + token_data: dict[Any, Any] = cast(dict[Any, Any], keycloak_openid.introspect(access_token)) + if token_data.get("active"): + _LOG.debug("token accepted") return token_data - print("NOT OK") - print(token_data) + _LOG.warning("token rejected by Keycloak") return None -def apikey_handler(access_token) -> dict: - """Verify API key based on environment variable +def apikey_handler(access_token: str) -> dict | None: + """Verify the API key passed by the client. Returns ------- dict - a dictionary containing sub and scope - None in case of invalid token + ``{"scope": []}`` if the supplied key matches the configured + ``AUTH_API_KEY``. + None + If the key is wrong, OIDC is configured (OIDC handles this path), + or the operator has not configured a real key in production. """ - if not AUTH_SERVER and AUTH_API_KEY == access_token: + if AUTH_SERVER: + return None + # In production we refuse the unconfigured default value, even if it + # technically matches the request — operators should set a real key. + if FLASK_ENV == "production" and AUTH_API_KEY == _DEFAULT_APIKEY: + return None + expected = AUTH_API_KEY.encode("utf-8") + provided = (access_token or "").encode("utf-8") + if secrets.compare_digest(expected, provided): return {"scope": []} return None diff --git a/optimizerapi/health.py b/optimizerapi/health.py index f658ce6..e9b8b71 100644 --- a/optimizerapi/health.py +++ b/optimizerapi/health.py @@ -1,2 +1,2 @@ -def check() -> str: +def health_check() -> str: return "OK" diff --git a/optimizerapi/openapi/specification.yml b/optimizerapi/openapi/specification.yml index 6648378..db0a1dc 100644 --- a/optimizerapi/openapi/specification.yml +++ b/optimizerapi/openapi/specification.yml @@ -13,7 +13,8 @@ paths: - oauth2: [] - apikey: [] description: Run optimizer with the specified parameters - operationId: optimizerapi.optimizer_handler.run + operationId: run_optimizer + x-openapi-router-controller: optimizerapi.optimizer_handler responses: "200": description: Result of running the optimizer with the specified parameters @@ -125,7 +126,8 @@ paths: /health: get: description: Health check endpoint - operationId: optimizerapi.health.check + operationId: health_check + x-openapi-router-controller: optimizerapi.health responses: "200": description: Service is reachable @@ -169,6 +171,22 @@ components: default: ["objective", "convergence", "pareto"] items: type: string + selectedPoint: + description: > + Override the highlight point in single plots with explicit + X-space coordinates. Honored only when graphFormat is "json"; + ignored on the PNG path. + type: array + items: + anyOf: + - type: string + - type: number + pickled: + description: > + Opaque cache hint produced by a previous response. The server + validates it matches the current data / optimizerConfig; on + mismatch it is ignored and a full run is performed. + type: string additionalProperties: type: string data: @@ -203,6 +221,13 @@ components: $ref: "#/components/schemas/space" constraints: $ref: "#/components/schemas/constraints" + required: + - baseEstimator + - acqFunc + - initialPoints + - kappa + - xi + - space required: - data - optimizerConfig @@ -303,6 +328,10 @@ components: type: object extras: type: object + properties: + pickledUsed: + description: True iff the server reused the pickled cache hint for this response. + type: boolean error: title: Errors returned by the API type: object diff --git a/optimizerapi/optimizer.py b/optimizerapi/optimizer.py index 69ac65c..0c0cf17 100644 --- a/optimizerapi/optimizer.py +++ b/optimizerapi/optimizer.py @@ -4,38 +4,151 @@ It should only depend on ProcessOptimizer specifics and json related features. """ +import importlib.metadata +import json +import logging import os import platform -from time import strftime -import base64 -import io -import json import subprocess +from dataclasses import dataclass +from time import strftime + import json_tricks +import matplotlib.pyplot as plt +import numpy from ProcessOptimizer import Optimizer, expected_minimum -from ProcessOptimizer.plots import ( - plot_objective, - plot_convergence, - plot_Pareto, - plot_brownie_bee_frontend, -) from ProcessOptimizer.space import Real from ProcessOptimizer.space.constraints import SumEquals -import matplotlib.pyplot as plt -import numpy -from .securepickle import pickleToString, get_crypto +from typing import TYPE_CHECKING, Any, cast + +from .securepickle import get_crypto +from .pickled_state import compute_fingerprint, pack, unpack_if_valid +from .plot_emitters import emit_json_single_plots, emit_pareto_data, emit_png_plots + +if TYPE_CHECKING: + from .types import Extras, OptimizerConfig, Plot, RequestBody numpy.random.seed(42) plt.switch_backend("Agg") -def run(body) -> dict: - """ "Handle the run request""" +@dataclass(frozen=True) +class _ParsedExtras: + graph_format: str + max_quality: int + graphs_to_return: list[str] + objective_pars: str + include_model: bool + selected_point: "list[str | float] | None" + experiment_suggestion_count: int + + +def _parse_bool(value: object, default: bool = True) -> bool: + """Coerce extras' stringly-typed boolean fields. + + Accepts the literals "true" / "false" (case-insensitive), real bools, + and JSON-style ``true``/``false``. Anything else falls back to *default*. + """ + if isinstance(value, bool): + return value + if value is None: + return default + text = str(value).strip().lower() + if text in ("true", "1", "yes"): + return True + if text in ("false", "0", "no"): + return False + return default + + +def _parse_extras(extras: "Extras", logger: "logging.Logger") -> _ParsedExtras: + """Read the request ``extras`` block into a typed view. + + Emits the PNG + selectedPoint warning here so callers don't need to + duplicate the check. + """ + graph_format = extras.get("graphFormat", "png") + selected_point = extras.get("selectedPoint") + if selected_point is not None and graph_format != "json": + logger.warning( + "selectedPoint ignored on png path (graphFormat=%s)", graph_format + ) + return _ParsedExtras( + graph_format=graph_format, + max_quality=int(extras.get("maxQuality", 5)), + graphs_to_return=list(extras.get( + "graphs", ["objective", "convergence", "pareto", "single"] + )), + objective_pars=extras.get("objectivePars", "result"), + include_model=_parse_bool(extras.get("includeModel", "true")), + selected_point=selected_point, + experiment_suggestion_count=int(extras.get("experimentSuggestionCount", 1)), + ) + + +def _compute_next_experiments( + optimizer: Any, + cfg: "OptimizerConfig", + n_points: int, +) -> list[list[str | float]]: + """Ask the optimizer for the next N experiments, normalising the shape. + + ``optimizer.ask`` can return either a single experiment (flat list) or + a list of experiments. We always return a list of lists. + """ + constraints = cfg.get("constraints", []) + if constraints: + next_exp = optimizer.ask(n_points=n_points, strategy="cl_min") + else: + next_exp = optimizer.ask(n_points=n_points) + if next_exp and not any(isinstance(x, list) for x in next_exp): + next_exp = [next_exp] + return cast(list[list[str | float]], round_to_length_scales(next_exp, optimizer.space)) + + +def _flatten_expected_minima(models: list[dict]) -> None: + """In-place flatten of nested ``expected_minimum`` entries on each model. + + The pre-flatten shape from ``process_model`` can be a list of mixed + scalars and lists; the response contract is a single flat list inside + a one-element outer list. This function normalises that. + """ + for model in models: + flat = [] + for x in model["expected_minimum"]: + if isinstance(x, list): + flat.extend(x) + else: + flat.append(x) + model["expected_minimum"] = [flat] + + +def _set_expected_minimum( + result_details: dict, + single_result: object, + space: object, +) -> None: + """Compute and store the expected minimum (single-objective only).""" + minimum = expected_minimum(single_result, return_std=True) + result_details["expected_minimum"] = [ + round_to_length_scales(minimum[0], space), + minimum[1], + ] + + +def run(body: "RequestBody") -> dict: + """Handle the run request. + + Returns the response envelope as a plain ``dict`` — the json_tricks + round-trip at the bottom of the function drops NumPy types, which is + why we cannot return ``ResponseEnvelope`` directly without an + explicit cast. + """ data = [(run["xi"], run["yi"]) for run in body["data"]] cfg = body["optimizerConfig"] - constraints = cfg["constraints"] if "constraints" in cfg else [] - extras = body["extras"] if "extras" in body else {} + constraints = cfg.get("constraints", []) + extras = body.get("extras", {}) use_actual_measurement_histogram = json.loads( extras.get("useActualMeasurementHistogram", "true").lower() ) @@ -45,7 +158,7 @@ def run(body) -> dict: convert_number_type(x["from"], x["type"]), convert_number_type(x["to"], x["type"]), ) - if (x["type"] == "discrete" or x["type"] == "continuous") + if x["type"] in ("discrete", "continuous") else tuple(x["categories"]) ) for x in cfg["space"] @@ -67,6 +180,37 @@ def run(body) -> dict: if len(Yi) > 0: n_objectives = len(Yi[0]) + request_fingerprint = compute_fingerprint(body["data"], cfg) + pickled_input = extras.get("pickled", "") + if pickled_input: + include_model_str = str(extras.get("includeModel", "true")).lower() + if include_model_str == "false": + logging.getLogger(__name__).warning( + "includeModel=false with extras.pickled present — if the cache hint is used, " + "the next call will not have one to reuse" + ) + cached = unpack_if_valid( + pickled_input, expected_fingerprint=request_fingerprint, crypto=get_crypto() + ) if pickled_input else None + + if cached is not None: + result = cached["result"] + optimizer = cached["optimizer"] + response = process_result( + result, optimizer, dimensions, cfg, extras, data, space, + request_fingerprint=request_fingerprint, pickled_used=True, + ) + response["result"]["extras"]["parameters"] = { + "dimensions": dimensions, + "space": space, + "hyperparams": hyperparams, + "Xi": Xi, + "Yi": Yi, + "extras": extras, + } + # json_tricks roundtrip drops NumPy types and produces a plain dict + return cast(dict[Any, Any], json.loads(json_tricks.dumps(response))) + if constraints is not None and len(constraints) > 0: optimizer = Optimizer( space, **hyperparams, lhs=False, n_objectives=n_objectives @@ -90,7 +234,10 @@ def run(body) -> dict: else: result = [] - response = process_result(result, optimizer, dimensions, cfg, extras, data, space) + response = process_result( + result, optimizer, dimensions, cfg, extras, data, space, + request_fingerprint=request_fingerprint, pickled_used=False, + ) response["result"]["extras"]["parameters"] = { "dimensions": dimensions, @@ -103,7 +250,8 @@ def run(body) -> dict: # It is necesarry to convert response to a json string and then back to # dictionary because NumPy types are not serializable by default - return json.loads(json_tricks.dumps(response)) + # json_tricks roundtrip drops NumPy types and produces a plain dict + return cast(dict[Any, Any], json.loads(json_tricks.dumps(response))) def convert_number_type(value, num_type): @@ -113,7 +261,18 @@ def convert_number_type(value, num_type): return float(value) -def process_result(result, optimizer, dimensions, cfg, extras, data, space): +def process_result( + result: Any, + optimizer: Any, + dimensions: list[str], + cfg: "OptimizerConfig", + extras: "Extras", + data: list[tuple[list[str | float], list[float]]], + space: list, + *, + request_fingerprint: str, + pickled_used: bool, +) -> dict: """Extracts results from the OptimizerResult. Parameters @@ -146,95 +305,77 @@ def process_result(result, optimizer, dimensions, cfg, extras, data, space): model representation etc.} } """ - result_details = {"next": [], "models": [], "pickled": "", "extras": {}} - plots = [] - response = {"plots": plots, "result": result_details} + result_details: dict[str, Any] = {"next": [], "models": [], "pickled": "", "extras": {}} + plots: "list[Plot]" = [] + response: dict[str, Any] = {"plots": plots, "result": result_details} # GraphFormat should, at the moment, be either "png" or "none". Default (legacy) # behavior is "png", so the API returns png images. Any other input is interpreted # as "None" at the moment. - graph_format = extras.get("graphFormat", "png") - max_quality = int(extras.get("maxQuality", "5")) - graphs_to_return = extras.get("graphs", ["objective", "convergence", "pareto"]) - - objective_pars = extras.get("objectivePars", "result") - - pickle_model = json.loads(extras.get("includeModel", "true").lower()) + parsed = _parse_extras(extras, logging.getLogger(__name__)) - # In the following section details that should be reported to - # clients should go into the "resultDetails" dictionary and plots - # go into the "plots" list (this is handled by calling the "addPlot" function) - experiment_suggestion_count = 1 - if "experimentSuggestionCount" in extras: - experiment_suggestion_count = extras["experimentSuggestionCount"] - - if "constraints" in cfg and len(cfg["constraints"]) > 0: - next_exp = optimizer.ask( - n_points=experiment_suggestion_count, strategy="cl_min" - ) - else: - next_exp = optimizer.ask(n_points=experiment_suggestion_count) - if len(next_exp) > 0 and not any(isinstance(x, list) for x in next_exp): - next_exp = [next_exp] - result_details["next"] = round_to_length_scales(next_exp, optimizer.space) + result_details["next"] = _compute_next_experiments( + optimizer, cfg, parsed.experiment_suggestion_count + ) if len(data) >= cfg["initialPoints"]: # Some calculations are only possible if the model has # processed more than "initialPoints" data points result_details["models"] = [process_model(model, optimizer) for model in result] - if graph_format == "png": + if parsed.graph_format == "png": + emit_png_plots( + plots, + result=result, + dimensions=dimensions, + graphs=parsed.graphs_to_return, + max_quality=parsed.max_quality, + objective_pars=parsed.objective_pars, + ) + if optimizer.n_objectives == 1: + _set_expected_minimum(result_details, result[0], optimizer.space) + elif parsed.graph_format == "json": for idx, model in enumerate(result): - if "single" in graphs_to_return: - bb_plots = plot_brownie_bee_frontend(model, max_quality=max_quality) - for i, plot in enumerate(bb_plots): - pic_io_bytes = io.BytesIO() - plot.savefig(pic_io_bytes, format="png") - pic_io_bytes.seek(0) - pic_hash = base64.b64encode(pic_io_bytes.read()) - plots.append( - {"id": f"single_{idx}_{i}", "plot": str(pic_hash, "utf-8")} - ) - if "convergence" in graphs_to_return: - plot_convergence(model) - add_plot(plots, f"convergence_{idx}") - - if "objective" in graphs_to_return: - plot_objective( - model, - dimensions=dimensions, - usepartialdependence=False, - show_confidence=True, - pars=objective_pars, + if "single" in parsed.graphs_to_return and optimizer.n_objectives != 2: + emit_json_single_plots( + plots, + result=result[idx], + prefix=f"single_{idx}", + selected_point=parsed.selected_point, ) - add_plot(plots, f"objective_{idx}") + # convergence and objective plots are PNG-only; nothing to emit here. if optimizer.n_objectives == 1: - minimum = expected_minimum(result[0], return_std=True) - result_details["expected_minimum"] = [ - round_to_length_scales(minimum[0], optimizer.space), - minimum[1], - ] - elif "pareto" in graphs_to_return: - plot_Pareto(optimizer) - add_plot(plots, "pareto") - - if pickle_model: - result_details["pickled"] = pickleToString(result, get_crypto()) + _set_expected_minimum(result_details, result[0], optimizer.space) + + if optimizer.n_objectives == 2 and "pareto" in parsed.graphs_to_return: + emit_pareto_data(plots, optimizer) + + if optimizer.n_objectives == 2 and "single" in parsed.graphs_to_return: + emit_json_single_plots( + plots, + result=result[0], + prefix="objective_1", + selected_point=parsed.selected_point, + ) + emit_json_single_plots( + plots, + result=result[1], + prefix="objective_2", + selected_point=parsed.selected_point, + ) + + if parsed.include_model: + result_details["pickled"] = pack( + result=result, + next_points=result_details["next"], + optimizer=optimizer, + fingerprint=request_fingerprint, + crypto=get_crypto(), + ) add_version_info(result_details["extras"]) + result_details["extras"]["pickledUsed"] = pickled_used - # print(str(response)) - org_models = response["result"]["models"] - for model in org_models: - # Flatten expected minimum entries - model["expected_minimum"] = [ - [ - item - for sublist in [ - x if isinstance(x, list) else [x] for x in model["expected_minimum"] - ] - for item in sublist - ] - ] + _flatten_expected_minima(response["result"]["models"]) return response @@ -251,7 +392,7 @@ def process_model(model, optimizer): dict a dictionary containing the model specific results. """ - result_details = {"expected_minimum": [], "extras": {}} + result_details: dict[str, Any] = {"expected_minimum": [], "extras": {}} minimum = expected_minimum(model) result_details["expected_minimum"] = [ round_to_length_scales(minimum[0], optimizer.space), @@ -260,43 +401,7 @@ def process_model(model, optimizer): return result_details -def add_plot(result, id="generic", close=True, debug=False): - """Add the current figure to result as a base64 encoded string. - - This function should be called after every plot that is generated. - It takes the current state of the figure canvas and writes it to - a base64 encoded string which is then appended to the list supplied. - - Parameters - ---------- - result : list - The list of plots to which new plots should be addeed. - id : str - Identifier for the plot (default is "generic") - close : bool - If set to True the current matplot figure is cleared after the plot - has been saved. (default is True) - debug : bool - Indicate if plots should be written to local files. - If set to True plots are stored in tmp/process_optimizer_[id].png - relative to current working directory. (default is False) - """ - pic_io_bytes = io.BytesIO() - plt.savefig(pic_io_bytes, format="png", bbox_inches="tight") - pic_io_bytes.seek(0) - pic_hash = base64.b64encode(pic_io_bytes.read()) - result.append({"id": id, "plot": str(pic_hash, "utf-8")}) - - if debug: - with open("tmp/process_optimizer_" + id + ".png", "wb") as imgfile: - plt.savefig(imgfile, bbox_inches="tight", pad_inches=0) - - # print("IMAGE: " + str(pic_hash, "utf-8")) - if close: - plt.clf() - - -def round_to_length_scales(x, space): +def round_to_length_scales(x: Any, space: Any) -> Any: """Rounds a suggested experiment to to the length scales of each dimension For each dimension the length of the dimension is calculated and the @@ -317,7 +422,7 @@ def round_to_length_scales(x, space): The space of the optimizer. Contains information about each dimension of the space """ - for dim, i in zip(space.dimensions, range(len(space.dimensions))): + for i, dim in enumerate(space.dimensions): # Checking if dimension is real. Else do nothing if isinstance(dim, Real): length = dim.high - dim.low @@ -346,9 +451,12 @@ def add_version_info(extras): The dictionary to hold the version information """ - with open("requirements-freeze.txt", "r", encoding="utf-8") as requirements_file: - requirements = requirements_file.readlines() - extras["libraries"] = [x.rstrip() for x in requirements] + extras["libraries"] = sorted( + [ + f"{dist.metadata['Name']}=={dist.version}" + for dist in importlib.metadata.distributions() + ] + ) extras["pythonVersion"] = platform.python_version() diff --git a/optimizerapi/optimizer_handler.py b/optimizerapi/optimizer_handler.py index ee496e8..867f046 100644 --- a/optimizerapi/optimizer_handler.py +++ b/optimizerapi/optimizer_handler.py @@ -5,11 +5,13 @@ in the specification.yml file found in the folder "openapi" in the root of this project. """ +import hashlib +import json +import logging import os import time -import json -import traceback -import hashlib +from typing import TYPE_CHECKING + from rq import Queue from rq.job import Job from rq.exceptions import NoSuchJobError @@ -18,11 +20,40 @@ import connexion from .optimizer import run as handle_run -if "REDIS_URL" in os.environ: - REDIS_URL = os.environ["REDIS_URL"] -else: - REDIS_URL = "redis://localhost:6379" -print("Connecting to" + REDIS_URL) +if TYPE_CHECKING: + from .types import RequestBody, ResponseEnvelope + + +def _parse_env_bool(name: str, default: bool = False) -> bool: + """Parse a boolean-ish env var. + + Accepts "true", "1", "yes" (case-insensitive) as True. + Anything else — including unset or "false" — is False. + """ + raw = os.environ.get(name, "").strip().lower() + if not raw: + return default + return raw in ("true", "1", "yes") + + +def _resolve_disconnect_check(): + """Return a callable that reports whether the client has disconnected. + + Outside of a request context (e.g. unit tests) or when running under + a server that doesn't expose ``waitress.client_disconnected``, this + returns a no-op that always says "still connected". + """ + try: + env = connexion.request.environ + except RuntimeError: + return lambda: False + return env.get("waitress.client_disconnected", lambda: False) + + +_LOG = logging.getLogger(__name__) + +REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379") +_LOG.info("Connecting to %s", REDIS_URL) redis = Redis.from_url(REDIS_URL) if "REDIS_TTL" in os.environ: TTL = int(os.environ["REDIS_TTL"]) @@ -36,37 +67,19 @@ queue = Queue(connection=redis) -def run(body) -> dict: - """Executes the ProcessOptimizer - - Returns - ------- - dict - a JSON encodable dictionary representation of the result. - """ - try: - if "waitress.client_disconnected" in connexion.request.environ: - disconnect_check = connexion.request.environ["waitress.client_disconnected"] - else: - - def disconnect_check(): - return False - - except RuntimeError: - - def disconnect_check(): - return False +def run_optimizer(body: "RequestBody") -> "ResponseEnvelope | tuple[dict[str, str], int]": + """Handle the optimizer run request (POST /optimizer).""" + disconnect_check = _resolve_disconnect_check() - if "USE_WORKER" in os.environ and os.environ["USE_WORKER"]: + if _parse_env_bool("USE_WORKER"): body_hash = hashlib.new("sha256") body_hash.update(json.dumps(body).encode()) job_id = body_hash.hexdigest() try: job = Job.fetch(job_id, connection=redis) - - print("Found existing job") + _LOG.info("Found existing job %s", job_id) except NoSuchJobError: - print(f"Creating new job (WORKER_TIMEOUT={WORKER_TIMEOUT})") + _LOG.info("Creating new job (WORKER_TIMEOUT=%s)", WORKER_TIMEOUT) job = queue.enqueue( do_run_work, body, @@ -77,29 +90,30 @@ def disconnect_check(): while job.return_value() is None: if disconnect_check(): try: - print(f"Client disconnected, cancelling job {job.id}") + _LOG.warning("Client disconnected, cancelling job %s", job.id) job.cancel() send_stop_job_command(redis, job.id) job.delete() except Exception: pass - return {} + return {} # type: ignore[return-value] # empty sentinel on client disconnect time.sleep(0.2) - return job.return_value() + return job.return_value() # type: ignore[return-value] # RQ returns Any; narrowed in Phase 4 return do_run_work(body) -def do_run_work(body) -> dict: - """ "Handle the run request""" +def do_run_work(body: "RequestBody") -> "ResponseEnvelope": + """Handle the run request. + + On error we return a Connexion ``problem`` response, which serialises + to the OpenAPI-declared 400 / 500 response shape. + """ try: - return handle_run(body) - except IOError as err: - return ({"message": "I/O error", "error": str(err)}, 400) - except TypeError as err: - return ({"message": "Type error", "error": str(err)}, 400) - except ValueError as err: - return ({"message": "Validation error", "error": str(err)}, 400) + # Phase 4 narrows optimizer.run return type to ResponseEnvelope + return handle_run(body) # type: ignore[return-value] + except (IOError, TypeError, ValueError) as err: + _LOG.warning("client error: %s", err) + return connexion.problem(400, "Bad request", str(err)) # type: ignore[no-any-return] except Exception as err: - # Log unknown exceptions to support debugging - traceback.print_exc() - return ({"message": "Unknown error", "error": str(err)}, 500) + _LOG.exception("unexpected error during optimizer run") + return connexion.problem(500, "Internal server error", str(err)) # type: ignore[no-any-return] diff --git a/optimizerapi/pickled_state.py b/optimizerapi/pickled_state.py new file mode 100644 index 0000000..6b681f8 --- /dev/null +++ b/optimizerapi/pickled_state.py @@ -0,0 +1,83 @@ +"""Pickled-state cache: fingerprint a request, pack/unpack pickled payloads. + +The fingerprint binds a pickled blob to the request that produced it, so a +client that sends a stale pickled along with newer data gets a clean fall- +through to a full run instead of silently buggy reuse. +""" + +import hashlib +import json +import logging +from typing import TYPE_CHECKING, cast + +from .securepickle import pickleToString, unpickleFromString + +if TYPE_CHECKING: + from cryptography.fernet import Fernet + + from .types import CachePayload, DataPoint, OptimizerConfig + +_LOG = logging.getLogger(__name__) + +_REQUIRED_KEYS = ("fingerprint", "result", "next", "optimizer") + + +def compute_fingerprint( + data: list["DataPoint"], + optimizer_config: "OptimizerConfig", +) -> str: + """Return sha256 hex of the canonical-JSON of (data, optimizerConfig). + + Canonical JSON: sorted keys, no whitespace. Numbers are emitted as Python's + default JSON representation, which is stable for the integer / float values + that flow through the API. + """ + payload = {"data": data, "optimizerConfig": optimizer_config} + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def pack( + *, + result: object, + next_points: list[list[str | float]], + optimizer: object, + fingerprint: str, + crypto: "Fernet", +) -> str: + """Encrypt a pickled cache payload for the given fingerprint.""" + payload = { + "fingerprint": fingerprint, + "result": result, + "next": next_points, + "optimizer": optimizer, + } + return cast(str, pickleToString(payload, crypto)) + + +def unpack_if_valid( + blob: str, + *, + expected_fingerprint: str, + crypto: "Fernet", +) -> "CachePayload | None": + """Decrypt and validate a pickled cache payload. + + Returns the payload dict on success, or None if anything is off. Any + failure is logged once at WARNING with a reason tag in the message: + decrypt_failed | bad_structure | fingerprint_mismatch. + """ + if not blob: + return None + try: + payload = unpickleFromString(blob, crypto) + except Exception: + _LOG.warning("pickled cache ignored: decrypt_failed") + return None + if not isinstance(payload, dict) or not all(k in payload for k in _REQUIRED_KEYS): + _LOG.warning("pickled cache ignored: bad_structure") + return None + if payload["fingerprint"] != expected_fingerprint: + _LOG.warning("pickled cache ignored: fingerprint_mismatch") + return None + return payload # type: ignore[return-value] diff --git a/optimizerapi/plot_emitters.py b/optimizerapi/plot_emitters.py new file mode 100644 index 0000000..fb15408 --- /dev/null +++ b/optimizerapi/plot_emitters.py @@ -0,0 +1,163 @@ +"""Plot emission for the optimizer API response. + +Two output formats are supported: +- ``emit_png_plots`` — base64 PNG plots, used by the legacy UI. +- ``emit_json_single_plots`` / ``emit_pareto_data`` — structured JSON + plot data, used by the React-based UI. +""" + +import base64 +import io +from typing import TYPE_CHECKING, Any + +import json_tricks +import matplotlib.pyplot as plt +import numpy +from ProcessOptimizer.plots import ( + get_Brownie_Bee_1d_plot, + get_Brownie_Bee_Pareto, + plot_brownie_bee_frontend, + plot_convergence, + plot_objective, +) +from ProcessOptimizer.utils.utils import get_Pareto_front_compromise + +if TYPE_CHECKING: + from .types import Plot + + +def _get_brownie_bee_1d_plot_safe(result, x_eval=None, **kwargs): + """Workaround for ProcessOptimizer bug: model.predict must receive space.transform output, + not a raw list containing categorical strings.""" + if x_eval is None: + return get_Brownie_Bee_1d_plot(result, x_eval=x_eval, **kwargs) + + space = result.space + has_categoricals = any( + not isinstance(v, (int, float)) for v in x_eval + ) + if not has_categoricals: + return get_Brownie_Bee_1d_plot(result, x_eval=x_eval, **kwargs) + + model = result.models[-1] + x_transformed = space.transform([x_eval]) + original_predict = model.predict + + def _predict_with_transform(X, **predict_kwargs): + arr = numpy.array(X) + if arr.dtype.kind in ("U", "S", "O"): + return original_predict(x_transformed, **predict_kwargs) + return original_predict(X, **predict_kwargs) + + model.predict = _predict_with_transform + try: + return get_Brownie_Bee_1d_plot(result, x_eval=x_eval, **kwargs) + finally: + model.predict = original_predict + + +def emit_json_single_plots( + plots: "list[Plot]", + *, + result: Any, + prefix: str, + selected_point: "list[str | float] | None", +) -> None: + """Append one plot entry per dimension plus a final histogram entry. + + Parameters + ---------- + plots: + The mutable list of plot entries to append to. + result: + A ProcessOptimizer ``OptimizerResult``-like object. + prefix: + Plot-id prefix. The emitted ids are ``{prefix}_0``, ``{prefix}_1``, + ..., with the final id being ``{prefix}_`` for the histogram. + selected_point: + X-space coordinates to highlight, or ``None`` to use the default. + """ + one_d_data = _get_brownie_bee_1d_plot_safe(result, x_eval=selected_point) + histogram_entry = one_d_data[-1] + for i, dim_data in enumerate(one_d_data[:-1]): + plots.append( + {"id": f"{prefix}_{i}", "plot": json_tricks.dumps({"data": dim_data})} + ) + plots.append( + { + "id": f"{prefix}_{len(one_d_data) - 1}", + "plot": json_tricks.dumps( + { + "histogram": { + "mean": float(numpy.ravel(histogram_entry[0])[0]), + "std": float(numpy.ravel(histogram_entry[1])[0]), + } + } + ), + } + ) + + +def emit_png_plots( + plots: "list[Plot]", + *, + result: list, + dimensions: list[str], + graphs: list[str], + max_quality: int, + objective_pars: str, +) -> None: + """Append base64-encoded PNG plots for each model in ``result``.""" + for idx, model in enumerate(result): + if "single" in graphs: + bb_plots = plot_brownie_bee_frontend(model, max_quality=max_quality) + for i, plot in enumerate(bb_plots): + plots.append( + {"id": f"single_{idx}_{i}", "plot": _figure_to_b64(plot)} + ) + plt.close(plot) + if "convergence" in graphs: + plot_convergence(model) + _emit_current_figure(plots, f"convergence_{idx}") + + if "objective" in graphs: + plot_objective( + model, + dimensions=dimensions, + usepartialdependence=False, + show_confidence=True, + pars=objective_pars, + ) + _emit_current_figure(plots, f"single_{idx}") + + +def emit_pareto_data(plots: "list[Plot]", optimizer: object) -> None: + """Append the pareto-front payload (multi-objective only).""" + front_x_data, front_y_data, obj1_error, obj2_error = get_Brownie_Bee_Pareto( + optimizer, n_points=200 + ) + best_idx = get_Pareto_front_compromise(front_y_data) + pareto_data = { + "front_x_data": front_x_data.tolist(), + "front_y_data": front_y_data.tolist(), + "obj1_error": obj1_error.tolist(), + "obj2_error": obj2_error.tolist(), + "best_idx": best_idx, + } + plots.append({"id": "pareto_data", "plot": json_tricks.dumps(pareto_data)}) + + +def _figure_to_b64(figure) -> str: + buf = io.BytesIO() + figure.savefig(buf, format="png") + buf.seek(0) + return base64.b64encode(buf.read()).decode("utf-8") + + +def _emit_current_figure(plots: "list[Plot]", plot_id: str) -> None: + """Snapshot the current matplotlib figure into ``plots`` and clear it.""" + buf = io.BytesIO() + plt.savefig(buf, format="png", bbox_inches="tight") + buf.seek(0) + plots.append({"id": plot_id, "plot": base64.b64encode(buf.read()).decode("utf-8")}) + plt.clf() diff --git a/optimizerapi/securepickle/__init__.py b/optimizerapi/securepickle/__init__.py index b21e39f..09815ff 100644 --- a/optimizerapi/securepickle/__init__.py +++ b/optimizerapi/securepickle/__init__.py @@ -1,5 +1,6 @@ -""" -Module that provides support for encrypted pickle functionality -""" +"""Secure pickle module — Fernet-encrypted pickle round-trip.""" + from .pickler import pickleToString, unpickleFromString from .secure import get_crypto + +__all__ = ["get_crypto", "pickleToString", "unpickleFromString"] diff --git a/optimizerapi/securepickle/secure.py b/optimizerapi/securepickle/secure.py index 7fb1685..78f38d8 100644 --- a/optimizerapi/securepickle/secure.py +++ b/optimizerapi/securepickle/secure.py @@ -1,14 +1,37 @@ -from cryptography.fernet import Fernet +"""Fernet crypto handle factory. + +Resolution order for the key: + 1. ``key`` argument (highest priority). + 2. ``PICKLE_KEY`` environment variable. + 3. Generate a new ephemeral key and log a WARNING. + +The ephemeral path is intended for local development only. In any +environment where pickled state must survive a restart, ``PICKLE_KEY`` +must be set explicitly. This function no longer mutates ``os.environ``, +so caller environments are unchanged. +""" + +import logging import os +from cryptography.fernet import Fernet + +_LOG = logging.getLogger(__name__) + def get_crypto(key=None): - if key == None: - key = os.getenv("PICKLE_KEY", None) - if key == None: - print("No key found, generating new key") + """Return a Fernet crypto handle. + + See the module docstring for the key-resolution order. + """ + if key is None: + key = os.getenv("PICKLE_KEY") + if key is None: key = Fernet.generate_key() - os.environ["PICKLE_KEY"] = key.decode("utf-8") - print("To reuse key for future server runs, set environment variable PICKLE_KEY=" + - os.environ["PICKLE_KEY"]) + _LOG.warning( + "No PICKLE_KEY set; generated an ephemeral key. Set " + "PICKLE_KEY=%s in the environment to persist pickled state " + "across restarts.", + key.decode("utf-8"), + ) return Fernet(key) diff --git a/optimizerapi/server.py b/optimizerapi/server.py index e425aa9..1f2026d 100644 --- a/optimizerapi/server.py +++ b/optimizerapi/server.py @@ -1,14 +1,35 @@ """ Main server """ +import logging import os import re + import connexion -from waitress import serve from flask_cors import CORS +from waitress import serve + from .securepickle import get_crypto +_LOG = logging.getLogger(__name__) + + +def _configure_logging() -> None: + """Configure root logging once, before any handler is dispatched. + + Level defaults to INFO. Override with the LOG_LEVEL environment + variable (e.g. LOG_LEVEL=DEBUG for local debugging). + """ + level_name = os.getenv("LOG_LEVEL", "INFO").upper() + level = getattr(logging, level_name, logging.INFO) + logging.basicConfig( + level=level, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + + if __name__ == "__main__": + _configure_logging() # Initialize crypto get_crypto() app = connexion.FlaskApp( @@ -46,11 +67,11 @@ # what we want to support. origins=re.compile(cors_origin), ) - print("CORS: " + cors_origin) + _LOG.info("CORS: %s", cors_origin) except re.error: - print("CORS: failed - the regex might be malformed.") + _LOG.warning("CORS: failed - the regex might be malformed.") else: - print("CORS: disabled") + _LOG.info("CORS: disabled") if development: app.run(port=9090) diff --git a/optimizerapi/types.py b/optimizerapi/types.py new file mode 100644 index 0000000..f74ca51 --- /dev/null +++ b/optimizerapi/types.py @@ -0,0 +1,120 @@ +"""Boundary types for the optimizer API. + +These ``TypedDict``s describe the shape of the request body, response +envelope, and internal cache payload. They are *static* contracts — +they exist to be checked by ``mypy`` and to document the API to +readers. Runtime validation of incoming requests lives in Connexion + +OpenAPI; we do not duplicate that here. + +Note on the ``Dimension`` field name ``from``: + ``from`` is a Python keyword, so the class-based ``TypedDict`` + syntax cannot declare it. We use the alternative call-based + syntax to define ``Dimension``. +""" + +from typing import Any, Literal, NotRequired, TypedDict + + +# --- Request types --------------------------------------------------------- + +Dimension = TypedDict( + "Dimension", + { + "type": Literal["continuous", "discrete", "category"], + "name": str, + "from": NotRequired[float], + "to": NotRequired[float], + "categories": NotRequired[list[str]], + }, +) + + +class Constraint(TypedDict): + type: Literal["sum"] + dimensions: list[int] + value: float + + +class OptimizerConfig(TypedDict): + baseEstimator: str + acqFunc: str + initialPoints: int + kappa: float + xi: float + space: list[Dimension] + constraints: NotRequired[list[Constraint]] + + +class DataPoint(TypedDict): + xi: list[str | float] + yi: list[float] + + +GraphFormat = Literal["png", "json", "none"] +GraphName = Literal["objective", "convergence", "pareto", "single"] +ObjectivePars = Literal["result", "expected_minimum"] + + +class Extras(TypedDict, total=False): + objectivePars: ObjectivePars + graphFormat: GraphFormat + experimentSuggestionCount: int + maxQuality: int + graphs: list[GraphName] + selectedPoint: list[str | float] + pickled: str + # NB: stringly-typed; see audit §3 for the rationale for not migrating yet. + includeModel: Literal["true", "false"] + useActualMeasurementHistogram: Literal["true", "false"] + + +class RequestBody(TypedDict): + data: list[DataPoint] + optimizerConfig: OptimizerConfig + extras: NotRequired[Extras] + + +# --- Response types -------------------------------------------------------- + +class Plot(TypedDict): + id: str + plot: str # base64 PNG OR JSON-serialised plot data + + +class ResponseResultExtras(TypedDict, total=False): + pickledUsed: bool + parameters: dict[str, Any] + libraries: list[str] + pythonVersion: str + apiVersion: str + timeOfExecution: str + + +class ResponseModel(TypedDict, total=False): + expected_minimum: list[list[str | float]] + extras: dict[str, Any] + + +class ResponseResult(TypedDict, total=False): + next: list[list[str | float]] + models: list[ResponseModel] + pickled: str + extras: ResponseResultExtras + expected_minimum: list[list[str | float] | float] + + +class ResponseEnvelope(TypedDict): + plots: list[Plot] + result: ResponseResult + + +# --- Cache payload (pickled_state) ----------------------------------------- + +class CachePayload(TypedDict): + fingerprint: str + # ProcessOptimizer's OptimizerResult or list thereof — kept opaque + # so types.py does not depend on the optimizer library. + result: Any + next: list[list[str | float]] + # ProcessOptimizer.Optimizer — also opaque. + optimizer: Any diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..bc2ecdb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,79 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "process-optimizer-api" +version = "1.0.0" +description = "REST based API for ProcessOptimizer" +readme = "README.md" +requires-python = ">=3.13" +license = { text = "MIT" } +authors = [{ name = "BoostV" }] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.13", +] + +dependencies = [ + "connexion[swagger-ui]==2.14.2", + "python-keycloak==2.13.2", + "Flask==2.2.3", + "Flask-Cors==3.0.10", + # develop as of 2026-03-11 + "ProcessOptimizer[brownie-bee] @ git+https://github.com/novonordisk-research/ProcessOptimizer.git@f1c8637d5eab", + "json-tricks==3.15.5", + "jsonschema>=4.21,<5", + "cryptography>=44,<46", + "waitress==2.1.2", + "rq==2.0.0", + # torch arrives transitively via ProcessOptimizer, but uv source pins only + # apply to *direct* dependencies — so we declare it here to force the CPU-only + # build (see [tool.uv.sources] below). The API runs CPU inference; the default + # PyPI torch wheel drags in ~5 GB of CUDA libraries we never use. + "torch", +] + +[project.optional-dependencies] +dev = ["pytest==8.3.3", "pytest-watch==4.2.0", "flake8>=7", "mypy>=1.13"] + +[project.urls] +Homepage = "https://github.com/BoostV/process-optimizer-api" +Repository = "https://github.com/BoostV/process-optimizer-api" +Issues = "https://github.com/BoostV/process-optimizer-api/issues" + +# Pull torch from PyTorch's CPU-only index so the install/image stays slim. +# `explicit = true` means only packages mapped to it in [tool.uv.sources] use +# this index; everything else still resolves from PyPI. +[[tool.uv.index]] +name = "pytorch-cpu" +url = "https://download.pytorch.org/whl/cpu" +explicit = true + +[tool.uv.sources] +torch = { index = "pytorch-cpu" } + +[tool.setuptools.packages.find] +include = ["optimizerapi*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = "test_*.py" +python_classes = "Test*" +python_functions = "test_*" + +[tool.mypy] +files = ["optimizerapi"] +python_version = "3.13" +strict = false +# Start permissive; we tighten as Phase 4 lands. +warn_unused_ignores = true +warn_redundant_casts = true +warn_return_any = true +disallow_any_unimported = false +check_untyped_defs = true +# Third-party libraries that don't ship type stubs. +ignore_missing_imports = true diff --git a/pytest.ini b/pytest.ini index ba8ba7e..4ed8dd9 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,4 +2,7 @@ filterwarnings = error ignore::UserWarning - ignore:.*np\.int.*is a deprecated alias.*:DeprecationWarning \ No newline at end of file + ignore:.*np\.int.*is a deprecated alias.*:DeprecationWarning + ignore:Accessing jsonschema\.draft4_format_checker.*:DeprecationWarning + ignore:jsonschema\.RefResolver is deprecated.*:DeprecationWarning + ignore:jsonschema\.exceptions\.RefResolutionError is deprecated.*:DeprecationWarning \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 37019c1..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,3 +0,0 @@ --r requirements.txt -pytest==8.3.3 -pytest-watch==4.2.0 diff --git a/requirements-freeze.txt b/requirements-freeze.txt deleted file mode 100644 index 1f37a8c..0000000 --- a/requirements-freeze.txt +++ /dev/null @@ -1,65 +0,0 @@ -async-timeout==4.0.2 -attrs==23.1.0 -bokeh==2.4.3 -certifi==2022.12.7 -cffi==1.15.1 -charset-normalizer==3.1.0 -click==8.1.3 -clickclick==20.10.2 -colorama==0.4.6 -connexion==2.14.2 -cryptography==3.4.7 -cycler==0.11.0 -deap==1.3.3 -deprecation==2.1.0 -docopt==0.6.2 -ecdsa==0.18.0 -exceptiongroup==1.2.2 -Flask==2.2.3 -Flask-Cors==3.0.10 -fonttools==4.39.3 -idna==3.4 -inflection==0.5.1 -iniconfig==2.0.0 -itsdangerous==2.1.2 -Jinja2==3.1.2 -joblib==1.2.0 -json-tricks==3.15.5 -jsonschema==4.15.0 -kiwisolver==1.4.4 -MarkupSafe==2.1.2 -matplotlib==3.5.3 -numpy==1.23.3 -packaging==23.1 -Pillow==9.5.0 -pluggy==1.5.0 -ProcessOptimizer==1.0.2 -py==1.11.0 -pyasn1==0.4.8 -pycparser==2.21 -pyparsing==3.0.9 -pyrsistent==0.19.3 -pytest==8.3.3 -pytest-watch==4.2.0 -python-dateutil==2.8.2 -python-jose==3.3.0 -python-keycloak==2.13.2 -PyYAML==6.0 -redis==4.5.4 -requests==2.28.2 -requests-toolbelt==0.10.1 -rq==2.0.0 -rsa==4.9 -scikit-learn==1.1.2 -scipy==1.9.1 -six==1.16.0 -swagger-ui-bundle==0.0.9 -threadpoolctl==3.1.0 -toml==0.10.2 -tomli==2.1.0 -tornado==6.2 -typing_extensions==4.5.0 -urllib3==1.26.15 -waitress==2.1.2 -watchdog==6.0.0 -Werkzeug==2.2.3 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 8afc15b..0000000 --- a/requirements.txt +++ /dev/null @@ -1,20 +0,0 @@ -connexion==2.14.2 -connexion[swagger-ui] -python-keycloak==2.13.2 -Flask==2.2.3 -Flask-Cors==3.0.10 -ProcessOptimizer==1.0.2 -json-tricks==3.15.5 -jsonschema==4.15.0 -cryptography==3.4.7 -waitress==2.1.2 -rq==2.0.0 -# ProcessOptimizer requirements -numpy==1.23.3 -matplotlib==3.5.3 -scipy==1.9.1 -bokeh==2.4.3 -scikit-learn==1.1.2 -six==1.16.0 -deap==1.3.3 -pyYAML==6.0 diff --git a/scripts/multi-blank-first-run.curl b/scripts/multi-blank-first-run.curl new file mode 100644 index 0000000..f7b15d7 --- /dev/null +++ b/scripts/multi-blank-first-run.curl @@ -0,0 +1,11 @@ +curl -S 'http://localhost:9090/v1.0/optimizer?apikey=none' \ +-X 'POST' \ +-H 'Content-Type: application/json' \ +-H 'Origin: http://localhost:5173' \ +-H 'Referer: http://localhost:5173/' \ +-H 'Accept: */*' \ +-H 'Sec-Fetch-Dest: empty' \ +-H 'Sec-Fetch-Mode: cors' \ +-H 'Sec-Fetch-Site: same-site' \ +-H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.4 Safari/605.1.15' \ +--data-raw '{"extras":{"experimentSuggestionCount":5},"data":[],"optimizerConfig":{"baseEstimator":"GP","acqFunc":"EI","initialPoints":5,"kappa":1.96,"xi":0.01,"space":[{"type":"discrete","name":"water","from":0,"to":100},{"type":"discrete","name":"temperature","from":-50,"to":120},{"type":"continuous","name":"angle","from":0,"to":360},{"type":"category","name":"color","categories":["red","green","blue"]}],"constraints":[]}}' diff --git a/scripts/multi-blank-second-run.curl b/scripts/multi-blank-second-run.curl new file mode 100644 index 0000000..6cce921 --- /dev/null +++ b/scripts/multi-blank-second-run.curl @@ -0,0 +1,11 @@ +curl -S 'http://localhost:9090/v1.0/optimizer?apikey=none' \ +-X 'POST' \ +-H 'Content-Type: application/json' \ +-H 'Origin: http://localhost:5173' \ +-H 'Referer: http://localhost:5173/' \ +-H 'Accept: */*' \ +-H 'Sec-Fetch-Dest: empty' \ +-H 'Sec-Fetch-Mode: cors' \ +-H 'Sec-Fetch-Site: same-site' \ +-H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.4 Safari/605.1.15' \ +--data-raw '{"extras":{"objectivePars":"expected_minimum","experimentSuggestionCount":5},"data":[{"xi":[10,69,36,"red"],"yi":[-2.5,-12]},{"xi":[70,103,324,"blue"],"yi":[-1.6,-50]},{"xi":[90,1,108,"green"],"yi":[-2.8,-5]},{"xi":[50,35,180,"red"],"yi":[-3.8,-10]},{"xi":[30,-33,252,"blue"],"yi":[-3.6,-40]}],"optimizerConfig":{"baseEstimator":"GP","acqFunc":"EI","initialPoints":5,"kappa":1.96,"xi":1.2,"space":[{"type":"discrete","name":"water","from":0,"to":100},{"type":"discrete","name":"temperature","from":-50,"to":120},{"type":"continuous","name":"angle","from":0,"to":360},{"type":"category","name":"color","categories":["red","green","blue"]}],"constraints":[]}}' diff --git a/scripts/sample-multi-with-selection.curl b/scripts/sample-multi-with-selection.curl new file mode 100644 index 0000000..139f3a3 --- /dev/null +++ b/scripts/sample-multi-with-selection.curl @@ -0,0 +1,35 @@ +# Sample curl script demonstrating selectedPoint + pickled flow +# This shows how to: +# 1. Run an initial optimization and capture the pickled state +# 2. Use selectedPoint with pickled to regenerate plots without retraining + +# ============================================================================ +# STEP 1: Run initial optimization to get pickled state +# ============================================================================ +# This request runs the optimizer and returns result.pickled in the response. +# Copy the value of result.pickled from the JSON response for use in Step 2. + +curl 'http://localhost:9090/v1.0/optimizer?apikey=none' \ +-X 'POST' \ +-H 'Content-Type: application/json' \ +-H 'Accept: */*' \ +--data-raw '{"extras":{"experimentSuggestionCount":1,"graphs":["pareto","single"],"graphFormat":"json"},"data":[{"xi":[16.7,500,250,20,"None"],"yi":[-2,-17]},{"xi":[50,833,150,60,"Whipped cream"],"yi":[-3,-6]},{"xi":[58.3,22,85,6,"Frosting"],"yi":[-6,-25]}],"optimizerConfig":{"baseEstimator":"GP","acqFunc":"EI","initialPoints":3,"kappa":1.96,"xi":2,"space":[{"type":"continuous","name":"Sugar","from":0,"to":100},{"type":"continuous","name":"Flour","from":0,"to":1000},{"type":"discrete","name":"Temperature","from":0,"to":300},{"type":"discrete","name":"Time","from":0,"to":120},{"type":"category","name":"Finish","categories":["None","Frosting","Whipped cream"]}],"constraints":[]}}' + +# ============================================================================ +# STEP 2: Use selectedPoint with pickled to regenerate plots +# ============================================================================ +# This request reuses the optimizer state (pickled) from Step 1 and selects +# a specific point [50, 833, 150, 60, "Whipped cream"] from the X-space +# to regenerate the plots without retraining the model. +# +# INSTRUCTIONS: +# 1. Run the command above and capture the JSON response +# 2. Find "pickled" in the response (it's a base64-encoded string) +# 3. Replace with that value +# 4. Run this command: + +curl 'http://localhost:9090/v1.0/optimizer?apikey=none' \ +-X 'POST' \ +-H 'Content-Type: application/json' \ +-H 'Accept: */*' \ +--data-raw '{"extras":{"experimentSuggestionCount":1,"graphs":["pareto","single"],"graphFormat":"json","selectedPoint":[50,833,150,60,"Whipped cream"],"pickled":""},"data":[{"xi":[16.7,500,250,20,"None"],"yi":[-2,-17]},{"xi":[50,833,150,60,"Whipped cream"],"yi":[-3,-6]},{"xi":[58.3,22,85,6,"Frosting"],"yi":[-6,-25]}],"optimizerConfig":{"baseEstimator":"GP","acqFunc":"EI","initialPoints":3,"kappa":1.96,"xi":2,"space":[{"type":"continuous","name":"Sugar","from":0,"to":100},{"type":"continuous","name":"Flour","from":0,"to":1000},{"type":"discrete","name":"Temperature","from":0,"to":300},{"type":"discrete","name":"Time","from":0,"to":120},{"type":"category","name":"Finish","categories":["None","Frosting","Whipped cream"]}],"constraints":[]}}' diff --git a/scripts/sample-multi.curl b/scripts/sample-multi.curl new file mode 100644 index 0000000..3c4bb4e --- /dev/null +++ b/scripts/sample-multi.curl @@ -0,0 +1,5 @@ +curl 'http://localhost:9090/v1.0/optimizer?apikey=none' \ +-X 'POST' \ +-H 'Content-Type: application/json' \ +-H 'Accept: */*' \ +--data-raw '{"extras":{"experimentSuggestionCount":1,"graphs":["pareto","single"],"graphFormat":"json"},"data":[{"xi":[16.7,500,250,20,"None"],"yi":[-2,-17]},{"xi":[50,833,150,60,"Whipped cream"],"yi":[-3,-6]},{"xi":[58.3,22,85,6,"Frosting"],"yi":[-6,-25]}],"optimizerConfig":{"baseEstimator":"GP","acqFunc":"EI","initialPoints":3,"kappa":1.96,"xi":2,"space":[{"type":"continuous","name":"Sugar","from":0,"to":100},{"type":"continuous","name":"Flour","from":0,"to":1000},{"type":"discrete","name":"Temperature","from":0,"to":300},{"type":"discrete","name":"Time","from":0,"to":120},{"type":"category","name":"Finish","categories":["None","Frosting","Whipped cream"]}],"constraints":[]}}' diff --git a/scripts/sample.curl b/scripts/sample.curl new file mode 100644 index 0000000..3519c4d --- /dev/null +++ b/scripts/sample.curl @@ -0,0 +1,6 @@ +curl 'http://127.0.0.1:9090/v1.0/optimizer?apikey=none' \ +-X 'POST' \ +-H 'Content-Type: application/json' \ +-H 'Accept: */*' \ +--data-raw '{"extras":{"experimentSuggestionCount":1,"graphs":["single", "pareto"],"graphFormat":"json","includeModel":"false","objectivePars":"expected_minimum"},"data":[{"xi":[130,170,210,105],"yi":[-2.1]},{"xi":[190,150,250,135],"yi":[-3.1]},{"xi":[150,130,230,95],"yi":[-1.8]},{"xi":[110,111,270,115],"yi":[-2.1]},{"xi":[176,200,300,122],"yi":[-5]},{"xi":[177,200,300,133],"yi":[-2.5]},{"xi":[146,158,260,112],"yi":[-4.5]}],"optimizerConfig":{"baseEstimator":"GP","acqFunc":"EI","initialPoints":5,"kappa":1.96,"xi":0.1,"space":[{"type":"discrete","name":"Pin elevation","from":100,"to":200},{"type":"discrete","name":"Bungee position","from":100,"to":200},{"type":"discrete","name":"Cup elevation","from":200,"to":300},{"type":"discrete","name":"Firing angle","from":90,"to":140}],"constraints":[]}}' + diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..7109e06 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,67 @@ +"""Shared pytest fixtures and session-level configuration.""" + +import os + +import pytest + +# A fixed Fernet key used by all tests so that encrypt/decrypt round-trips +# remain stable across multiple get_crypto() calls within a test session. +# Never use this value in production. +_TEST_PICKLE_KEY = "y3kUY6D3DAVy1NR8oK8Q7mmYkuvLtE8buQEO4IMSvOk=" + + +@pytest.fixture(autouse=True, scope="session") +def stable_pickle_key(): + """Set a fixed PICKLE_KEY for all tests. + + Without this, get_crypto() generates a new ephemeral Fernet key on every + call, so encrypt/decrypt round-trips across two optimizer.run_optimizer() calls (or + between run_optimizer() and a direct unpickleFromString() call in a test) will fail + with InvalidToken. A stable key makes the test environment mirror a real + deployment where PICKLE_KEY is set in the environment. + """ + original = os.environ.get("PICKLE_KEY") + os.environ["PICKLE_KEY"] = _TEST_PICKLE_KEY + yield _TEST_PICKLE_KEY + if original is None: + os.environ.pop("PICKLE_KEY", None) + else: + os.environ["PICKLE_KEY"] = original + + +@pytest.fixture(scope="session") +def app(): + """Create the Flask application for testing. + + This fixture creates a Connexion app with the same configuration + as the production server, but in test mode. + """ + from pathlib import Path + import connexion + from optimizerapi.securepickle import get_crypto + + # Initialize crypto with the stable key + get_crypto() + + # Use absolute path to the openapi spec + api_spec_dir = Path(__file__).parent.parent / "optimizerapi" / "openapi" + + app = connexion.FlaskApp( + __name__, + specification_dir=str(api_spec_dir), + ) + app.add_api("specification.yml", strict_validation=True, validate_responses=True) + app.app.config["TESTING"] = True + + return app + + +@pytest.fixture(scope="session") +def app_client(app): + """Create a test client for the Flask application. + + This client can be used to make HTTP requests to the API endpoints + as if they were being served by a real HTTP server. + """ + with app.app.test_client() as client: + yield client diff --git a/tests/context.py b/tests/context.py deleted file mode 100644 index 300a47e..0000000 --- a/tests/context.py +++ /dev/null @@ -1,9 +0,0 @@ -import os -import sys - -if True: - sys.path.insert( - 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../optimizerapi")) - ) - import optimizer - import securepickle diff --git a/tests/test_auth.py b/tests/test_auth.py new file mode 100644 index 0000000..d8fa672 --- /dev/null +++ b/tests/test_auth.py @@ -0,0 +1,55 @@ +"""Tests for the API-key handler.""" + +from unittest.mock import patch + + +def test_correct_apikey_returns_scope_dict(): + """A request with the configured API key returns a scope dict.""" + with patch("optimizerapi.auth.AUTH_API_KEY", "secret-key"), \ + patch("optimizerapi.auth.AUTH_SERVER", None): + from optimizerapi import auth + result = auth.apikey_handler("secret-key") + assert result == {"scope": []} + + +def test_wrong_apikey_returns_none(): + """An incorrect API key is rejected.""" + with patch("optimizerapi.auth.AUTH_API_KEY", "secret-key"), \ + patch("optimizerapi.auth.AUTH_SERVER", None): + from optimizerapi import auth + result = auth.apikey_handler("not-the-key") + assert result is None + + +def test_apikey_handler_uses_constant_time_comparison(): + """The handler must compare keys via secrets.compare_digest.""" + from optimizerapi import auth + import secrets + + with patch("optimizerapi.auth.AUTH_API_KEY", "secret-key"), \ + patch("optimizerapi.auth.AUTH_SERVER", None), \ + patch.object(secrets, "compare_digest", wraps=secrets.compare_digest) as mock_cmp: + auth.apikey_handler("secret-key") + assert mock_cmp.called, "apikey_handler must use secrets.compare_digest" + + +def test_default_apikey_value_is_rejected_in_production_mode(): + """When AUTH_API_KEY is the default 'none' and FLASK_ENV is 'production', + even matching the default value must be rejected.""" + with patch("optimizerapi.auth.AUTH_API_KEY", "none"), \ + patch("optimizerapi.auth.AUTH_SERVER", None), \ + patch("optimizerapi.auth.FLASK_ENV", "production"): + from optimizerapi import auth + result = auth.apikey_handler("none") + assert result is None + + +def test_default_apikey_value_is_accepted_in_development_mode(): + """In development mode (FLASK_ENV != 'production'), the legacy + behaviour of accepting AUTH_API_KEY='none' is preserved for ergonomics.""" + with patch("optimizerapi.auth.AUTH_API_KEY", "none"), \ + patch("optimizerapi.auth.AUTH_SERVER", None), \ + patch("optimizerapi.auth.FLASK_ENV", "development"): + from optimizerapi import auth + result = auth.apikey_handler("none") + assert result == {"scope": []} diff --git a/tests/test_e2e_pareto.py b/tests/test_e2e_pareto.py new file mode 100644 index 0000000..6ed673f --- /dev/null +++ b/tests/test_e2e_pareto.py @@ -0,0 +1,515 @@ +""" +End-to-end tests for the pareto front exploration feature. + +These tests hit the actual HTTP layer using Flask's test client, +verifying the full request/response cycle including authentication, +validation, and the pareto front exploration workflow. +""" + +import copy +import json +import pytest + +# Multi-objective test data from docs/api-usage.md +MULTI_OBJECTIVE_DATA = [ + {"xi": [16.7, 500, 250, 20, "None"], "yi": [-2, -17]}, + {"xi": [50, 833, 150, 60, "Whipped cream"], "yi": [-3, -6]}, + {"xi": [58.3, 22, 85, 6, "Frosting"], "yi": [-6, -25]}, +] + +MULTI_OBJECTIVE_CONFIG = { + "baseEstimator": "GP", + "acqFunc": "EI", + "initialPoints": 3, + "kappa": 1.96, + "xi": 2, + "space": [ + {"type": "continuous", "name": "Sugar", "from": 0, "to": 100}, + {"type": "continuous", "name": "Flour", "from": 0, "to": 1000}, + {"type": "discrete", "name": "Temperature", "from": 0, "to": 300}, + {"type": "discrete", "name": "Time", "from": 0, "to": 120}, + {"type": "category", "name": "Finish", "categories": ["None", "Frosting", "Whipped cream"]}, + ], + "constraints": [], +} + + +@pytest.mark.filterwarnings("ignore::DeprecationWarning") +class TestE2EParetoFront: + """End-to-end tests for pareto front exploration via HTTP.""" + + BASE_URL = "/v1.0/optimizer" + + @pytest.fixture + def api_key(self): + """Return the test API key.""" + return "none" + + def build_request(self, data, optimizer_config, extras=None): + """Build a request payload.""" + payload = { + "data": data, + "optimizerConfig": optimizer_config, + } + if extras: + payload["extras"] = extras + return payload + + def test_health_endpoint(self, app_client): + """Test that the health endpoint is reachable.""" + response = app_client.get("/v1.0/health") + assert response.status_code == 200 + + def test_initial_multi_objective_run_returns_pareto_data(self, app_client, api_key): + """Test that an initial multi-objective run includes pareto_data in plots.""" + payload = self.build_request( + data=MULTI_OBJECTIVE_DATA, + optimizer_config=MULTI_OBJECTIVE_CONFIG, + extras={ + "experimentSuggestionCount": 1, + "graphs": ["pareto", "single"], + "graphFormat": "json", + }, + ) + + response = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload, + content_type="application/json", + ) + + assert response.status_code == 200 + result = response.get_json() + + # Verify response structure + assert "plots" in result + assert "result" in result + assert "next" in result["result"] + assert "pickled" in result["result"] + + # Verify pareto_data plot is present + plot_ids = [p["id"] for p in result["plots"]] + assert "pareto_data" in plot_ids, "pareto_data plot should be present in multi-objective response" + + # Verify pareto_data has expected structure + pareto_plot = next(p for p in result["plots"] if p["id"] == "pareto_data") + pareto_data = json.loads(pareto_plot["plot"]) + assert "front_y_data" in pareto_data + assert "front_x_data" in pareto_data + assert "best_idx" in pareto_data + + def test_pareto_front_x_data_matches_input_dimensions(self, app_client, api_key): + """Test that front_x_data points have the same dimensionality as the space.""" + payload = self.build_request( + data=MULTI_OBJECTIVE_DATA, + optimizer_config=MULTI_OBJECTIVE_CONFIG, + extras={ + "graphs": ["pareto"], + "graphFormat": "json", + }, + ) + + response = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload, + content_type="application/json", + ) + + assert response.status_code == 200 + result = response.get_json() + + pareto_plot = next(p for p in result["plots"] if p["id"] == "pareto_data") + pareto_data = json.loads(pareto_plot["plot"]) + + space_dim = len(MULTI_OBJECTIVE_CONFIG["space"]) + for point in pareto_data["front_x_data"]: + assert len(point) == space_dim, ( + f"front_x_data point has {len(point)} dimensions, expected {space_dim}" + ) + + def test_selectedPoint_highlights_correct_point(self, app_client, api_key): + """Test that selectedPoint correctly highlights a specific point in plots.""" + # First, get the pareto front + payload = self.build_request( + data=MULTI_OBJECTIVE_DATA, + optimizer_config=MULTI_OBJECTIVE_CONFIG, + extras={ + "graphs": ["pareto", "single"], + "graphFormat": "json", + "includeModel": "true", + }, + ) + + response = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload, + content_type="application/json", + ) + assert response.status_code == 200 + run1 = response.get_json() + + # Extract a point from the pareto front + pareto_plot = next(p for p in run1["plots"] if p["id"] == "pareto_data") + pareto_data = json.loads(pareto_plot["plot"]) + selected_point = pareto_data["front_x_data"][0] + + # Now request with selectedPoint + payload_with_selection = self.build_request( + data=MULTI_OBJECTIVE_DATA, + optimizer_config=MULTI_OBJECTIVE_CONFIG, + extras={ + "graphs": ["single"], + "graphFormat": "json", + "selectedPoint": selected_point, + "includeModel": "false", + }, + ) + + response = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload_with_selection, + content_type="application/json", + ) + assert response.status_code == 200 + run2 = response.get_json() + + # Verify selectedPoint is reflected in objective_1_0 plot (dimension 0). + # In get_Brownie_Bee_1d_plot's output shape, index 3 is the x-coord of + # the highlight point for the dimension. + obj1_dim0 = next(p for p in run2["plots"] if p["id"] == "objective_1_0") + plot_data = json.loads(obj1_dim0["plot"]) + assert plot_data["data"][3] == selected_point[0], ( + "selectedPoint should be reflected in the single plot highlight" + ) + + def test_full_pareto_exploration_workflow(self, app_client, api_key): + """ + Test the complete pareto exploration workflow: + 1. Initial run to get pareto front and pickled state + 2. User selects a point from pareto front + 3. Re-request with selectedPoint + pickled for fast response + 4. Verify pickledUsed is True (fast path engaged) + """ + # Step 1: Initial run + payload = self.build_request( + data=MULTI_OBJECTIVE_DATA, + optimizer_config=MULTI_OBJECTIVE_CONFIG, + extras={ + "experimentSuggestionCount": 1, + "graphs": ["pareto", "single"], + "graphFormat": "json", + "includeModel": "true", + }, + ) + + response = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload, + content_type="application/json", + ) + assert response.status_code == 200 + run1 = response.get_json() + + # Verify initial run used full training (not cache) + assert run1["result"]["extras"]["pickledUsed"] is False, ( + "Initial run should not use pickled cache" + ) + + # Extract pickled and a point from pareto front + pickled_value = run1["result"]["pickled"] + assert len(pickled_value) > 0, "pickled should be present in response" + + pareto_plot = next(p for p in run1["plots"] if p["id"] == "pareto_data") + pareto_data = json.loads(pareto_plot["plot"]) + selected_point = pareto_data["front_x_data"][0] + + # Step 2: Re-request with selectedPoint + pickled + payload_with_cache = self.build_request( + data=MULTI_OBJECTIVE_DATA, + optimizer_config=MULTI_OBJECTIVE_CONFIG, + extras={ + "experimentSuggestionCount": 1, + "graphs": ["pareto", "single"], + "graphFormat": "json", + "selectedPoint": selected_point, + "pickled": pickled_value, + "includeModel": "true", + }, + ) + + response = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload_with_cache, + content_type="application/json", + ) + assert response.status_code == 200 + run2 = response.get_json() + + # Verify fast path was used + assert run2["result"]["extras"]["pickledUsed"] is True, ( + "Second run with pickled should use fast path" + ) + + # Verify response is valid and selectedPoint is applied + assert "next" in run2["result"] + assert len(run2["result"]["next"]) > 0 + + # Index 3 is the highlight x-coord in get_Brownie_Bee_1d_plot's output. + obj1_dim0 = next(p for p in run2["plots"] if p["id"] == "objective_1_0") + plot_data = json.loads(obj1_dim0["plot"]) + assert plot_data["data"][3] == selected_point[0] + + def test_equivalence_with_and_without_pickled(self, app_client, api_key): + """ + Test that pickled vs no-pickled produce identical plots when + selectedPoint, data, and config are the same. + """ + base_payload = self.build_request( + data=MULTI_OBJECTIVE_DATA, + optimizer_config=MULTI_OBJECTIVE_CONFIG, + extras={ + "graphs": ["single", "pareto"], + "graphFormat": "json", + "selectedPoint": [50, 833, 150, 60, "Whipped cream"], + "includeModel": "true", + }, + ) + + # Run without pickled + response_no_pickle = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=base_payload, + content_type="application/json", + ) + assert response_no_pickle.status_code == 200 + result_no_pickle = response_no_pickle.get_json() + + # Get pickled from the response for the next request + pickled_value = result_no_pickle["result"]["pickled"] + + # Run with pickled + payload_with_pickle = copy.deepcopy(base_payload) + payload_with_pickle["extras"]["pickled"] = pickled_value + + response_with_pickle = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload_with_pickle, + content_type="application/json", + ) + assert response_with_pickle.status_code == 200 + result_with_pickle = response_with_pickle.get_json() + + # Verify fast path was engaged + assert result_with_pickle["result"]["extras"]["pickledUsed"] is True + assert result_no_pickle["result"]["extras"]["pickledUsed"] is False + + # Equivalence contract (docs §8): same plots and same next. + assert result_no_pickle["result"]["next"] == result_with_pickle["result"]["next"], ( + "result.next should be identical with and without pickled" + ) + + plots_no_pickle = {p["id"]: p["plot"] for p in result_no_pickle["plots"]} + plots_with_pickle = {p["id"]: p["plot"] for p in result_with_pickle["plots"]} + + assert set(plots_no_pickle.keys()) == set(plots_with_pickle.keys()), ( + "Plot IDs should match between pickled and non-pickled runs" + ) + + for plot_id in plots_no_pickle: + assert plots_no_pickle[plot_id] == plots_with_pickle[plot_id], ( + f"Plot {plot_id} content should be identical with and without pickled" + ) + + def test_pareto_data_structure(self, app_client, api_key): + """Test the detailed structure of pareto_data response.""" + payload = self.build_request( + data=MULTI_OBJECTIVE_DATA, + optimizer_config=MULTI_OBJECTIVE_CONFIG, + extras={ + "graphs": ["pareto"], + "graphFormat": "json", + }, + ) + + response = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload, + content_type="application/json", + ) + assert response.status_code == 200 + result = response.get_json() + + pareto_plot = next(p for p in result["plots"] if p["id"] == "pareto_data") + pareto_data = json.loads(pareto_plot["plot"]) + + # All fields emit_pareto_data writes (docs §7 "per-objective uncertainty" + # corresponds to obj1_error / obj2_error). + assert set(pareto_data.keys()) >= { + "front_x_data", + "front_y_data", + "obj1_error", + "obj2_error", + "best_idx", + } + + # front_y_data is a list of [y_obj1, y_obj2] for each pareto point + assert isinstance(pareto_data["front_y_data"], list) + assert len(pareto_data["front_y_data"]) > 0 # At least one point on pareto front + num_front_points = len(pareto_data["front_y_data"]) + for point in pareto_data["front_y_data"]: + assert isinstance(point, list) + assert len(point) == 2, f"Each point should have 2 objectives, got {len(point)}" + + # front_x_data should have shape (num_points, num_dimensions) + assert isinstance(pareto_data["front_x_data"], list) + assert len(pareto_data["front_x_data"]) == num_front_points, ( + "front_x_data and front_y_data should have same number of points" + ) + for point in pareto_data["front_x_data"]: + assert len(point) == len(MULTI_OBJECTIVE_CONFIG["space"]) + + # Per-objective uncertainty arrays line up with the front + assert isinstance(pareto_data["obj1_error"], list) + assert isinstance(pareto_data["obj2_error"], list) + assert len(pareto_data["obj1_error"]) == num_front_points + assert len(pareto_data["obj2_error"]) == num_front_points + + # best_idx should be a valid index + assert 0 <= pareto_data["best_idx"] < num_front_points + + def test_pareto_with_png_format(self, app_client, api_key): + """PNG mode never emits pareto_data (it is JSON-only).""" + payload = self.build_request( + data=MULTI_OBJECTIVE_DATA, + optimizer_config=MULTI_OBJECTIVE_CONFIG, + extras={ + "graphs": ["pareto", "single"], + "graphFormat": "png", + }, + ) + + response = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload, + content_type="application/json", + ) + assert response.status_code == 200 + result = response.get_json() + + plot_ids = [p["id"] for p in result["plots"]] + # PNG path produces single plots per model, never a pareto_data entry. + assert "pareto_data" not in plot_ids + assert len(result["plots"]) > 0 + assert "next" in result["result"] + + def test_empty_data_with_pareto_request(self, app_client, api_key): + """Empty data + pareto in graphs returns initial-point suggestions without crashing. + + Note: with data=[] there are no yi vectors, so n_objectives defaults to 1 + and the pareto code path is never entered. This test only guards the + empty-data path; the pareto-specific behavior is covered elsewhere. + """ + payload = self.build_request( + data=[], # No data yet + optimizer_config=MULTI_OBJECTIVE_CONFIG, + extras={ + "graphs": ["pareto", "single"], + "graphFormat": "json", + }, + ) + + response = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload, + content_type="application/json", + ) + + # Should return 200 with initial point suggestions + assert response.status_code == 200 + result = response.get_json() + assert "result" in result + assert "next" in result["result"] + + def test_missing_api_key_returns_401(self, app_client): + """Test that missing API key is rejected with 401 Unauthorized.""" + payload = self.build_request( + data=MULTI_OBJECTIVE_DATA, + optimizer_config=MULTI_OBJECTIVE_CONFIG, + extras={"graphs": ["pareto"]}, + ) + + # No apikey query parameter + response = app_client.post( + self.BASE_URL, + json=payload, + content_type="application/json", + ) + + assert response.status_code == 401 + + def test_invalid_optimizer_config_returns_400(self, app_client, api_key): + """Missing required optimizerConfig fields are rejected at validation time as 400.""" + payload = { + "data": MULTI_OBJECTIVE_DATA, + "optimizerConfig": { + "invalid_field": "value", + # Missing required fields (space, baseEstimator, ...) + }, + } + + response = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload, + content_type="application/json", + ) + + assert response.status_code == 400 + result = response.get_json() + # Connexion returns RFC 7807 problem+json + assert "title" in result and "detail" in result + assert result.get("status") == 400 + + def test_single_objective_does_not_include_pareto(self, app_client, api_key): + """Test that single-objective optimization doesn't include pareto_data.""" + single_objective_data = [ + {"xi": [651, 56, 722, "Ræv"], "yi": [1]}, + {"xi": [651, 42, 722, "Ræv"], "yi": [0.2]}, + ] + + single_objective_config = { + "baseEstimator": "GP", + "acqFunc": "gp_hedge", + "initialPoints": 2, + "kappa": 1.96, + "xi": 0.012, + "space": [ + {"type": "discrete", "name": "Sukker", "from": 0, "to": 1000}, + {"type": "continuous", "name": "Peber", "from": 0, "to": 1000}, + {"type": "continuous", "name": "Hvedemel", "from": 0, "to": 1000}, + {"type": "category", "name": "Kunde", "categories": ["Mus", "Ræv"]}, + ], + } + + payload = self.build_request( + data=single_objective_data, + optimizer_config=single_objective_config, + extras={ + "graphs": ["single", "pareto"], # Requesting pareto but it's single-objective + "graphFormat": "json", + }, + ) + + response = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload, + content_type="application/json", + ) + + assert response.status_code == 200 + result = response.get_json() + + # pareto_data should NOT be present for single-objective + plot_ids = [p["id"] for p in result["plots"]] + assert "pareto_data" not in plot_ids, ( + "pareto_data should not be present for single-objective optimization" + ) diff --git a/tests/test_no_prints.py b/tests/test_no_prints.py new file mode 100644 index 0000000..eac01e1 --- /dev/null +++ b/tests/test_no_prints.py @@ -0,0 +1,35 @@ +"""Lint test: no stray print() in production code. + +Tests in this repo configure their own logging; production code MUST +route everything through ``logging`` so operators can filter, level- +gate, and route messages in production. ``print`` calls bypass that. +""" + +import ast +import pathlib + +import pytest + +OPTIMIZERAPI_ROOT = pathlib.Path(__file__).parent.parent / "optimizerapi" + + +def _all_python_files() -> list[pathlib.Path]: + return [p for p in OPTIMIZERAPI_ROOT.rglob("*.py") if p.is_file()] + + +@pytest.mark.parametrize("path", _all_python_files(), ids=lambda p: str(p.relative_to(OPTIMIZERAPI_ROOT.parent))) +def test_no_print_calls(path): + """No `print()` call is allowed under optimizerapi/.""" + source = path.read_text(encoding="utf-8") + tree = ast.parse(source, filename=str(path)) + offenders = [ + node.lineno + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "print" + ] + assert not offenders, ( + f"{path}: found print() calls at line(s) {offenders}. " + "Use logging.getLogger(__name__) instead." + ) diff --git a/tests/test_optimizer.py b/tests/test_optimizer.py index 94a6c81..75a2425 100644 --- a/tests/test_optimizer.py +++ b/tests/test_optimizer.py @@ -5,12 +5,9 @@ from unittest.mock import patch import copy import collections.abc +import json from optimizerapi import optimizer_handler as optimizer - -# {'data': [{'xi': [651, 56, 722, 'Ræv'], 'yi': 1}, {'xi': [651, 42, 722, 'Ræv'], 'yi': 0.2}], 'optimizerConfig': {'baseEstimator': 'GP', 'acqFunc': 'gp_hedge', 'initialPoints': 5, 'kappa': 1.96, 'xi': 0.012, 'space': [{'type': 'numeric', 'name': 'Sukker', 'from': 0, 'to': 1000}, {'type': 'numeric', 'name': 'Peber', 'from': 0, 'to': 1000}, {'type': 'numeric', 'name': 'Hvedemel', 'from': 0, 'to': 1000}, {'type': 'category', 'name': 'Kunde', 'categories': ['Mus', 'Ræv']}]}} -# 'data': [{'xi': [0, 5, 'Rød'], 'yi': 10}, {'xi': [5, 8.33, 'Hvid'], 'yi': 3}, {'xi': [10, 1.66, 'Rød'], 'yi': 5}], -# 'optimizerConfig': {'baseEstimator': 'GP', 'acqFunc': 'gp_hedge', 'initialPoints': 3, 'kappa': 1.96, 'xi': 0.01, -# 'space': [{'type': 'discrete', 'name': 'Alkohol', 'from': 0, 'to': 10}, {'type': 'continuous', 'name': 'Vand', 'from': 0, 'to': 10}, {'type': 'category', 'name': 'Farve', 'categories': ['Rød', 'Hvid']}]}} Received extras {'experimentSuggestionCount': 2} +from optimizerapi.securepickle import get_crypto, pickleToString, unpickleFromString sampleData = [ {"xi": [651, 56, 722, "Ræv"], "yi": [1]}, @@ -111,32 +108,31 @@ def validateResult(result): def test_can_be_run_without_data(): - result = optimizer.run(body={"data": [], "optimizerConfig": sampleConfig}) + result = optimizer.run_optimizer(body={"data": [], "optimizerConfig": sampleConfig}) validateResult(result) assert len(result["plots"]) == 0 def test_generates_plots_when_run_with_more_than_initialPoints_samples(): - result = optimizer.run(body={"data": sampleData, "optimizerConfig": sampleConfig}) + result = optimizer.run_optimizer(body={"data": sampleData, "optimizerConfig": sampleConfig}) validateResult(result) assert len(result["result"]["models"]) > 0 - assert len(result["plots"]) == 2 + assert len(result["plots"]) == 7 def test_generates_convergence_plots(): convergence_config = copy.deepcopy(sampleConfig) - convergence_config["extras"] = {"graphs": ["convergence"]} - result = optimizer.run( - body={"data": sampleData, "optimizerConfig": convergence_config} + result = optimizer.run_optimizer( + body={"data": sampleData, "optimizerConfig": convergence_config, "extras": {"graphs": ["convergence"]}} ) validateResult(result) assert len(result["result"]["models"]) > 0 - assert len(result["plots"]) == 2 + assert len(result["plots"]) == 1 assert result["plots"][0]["id"] == "convergence_0" def test_specifying_png_plots(): - result = optimizer.run( + result = optimizer.run_optimizer( body={ "data": sampleData, "optimizerConfig": sampleConfig, @@ -145,21 +141,62 @@ def test_specifying_png_plots(): ) validateResult(result) assert len(result["result"]["models"]) > 0 - assert len(result["plots"]) == 2 + assert len(result["plots"]) == 7 + + +def test_specifying_json_single_plots(): + result = optimizer.run_optimizer( + body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"graphFormat": "json", "graphs": ["single"], "includeModel": "false"}, + } + ) + # Don't call validateResult() because includeModel is false, so pickled model won't be included + assert len(result["result"]["models"]) > 0 + assert len(result["plots"]) == 5 + + plot_ids = [p["id"] for p in result["plots"]] + + assert "single_0_0" in plot_ids + assert "single_0_1" in plot_ids + assert "single_0_2" in plot_ids + assert "single_0_3" in plot_ids + assert "single_0_4" in plot_ids + assert "single_0" not in plot_ids + + for dim_idx in range(4): + plot_id = f"single_0_{dim_idx}" + plot_entry = next(p for p in result["plots"] if p["id"] == plot_id) + plot_data = json.loads(plot_entry["plot"]) + + assert "data" in plot_data + assert isinstance(plot_data["data"], list) + assert len(plot_data["data"]) == 4 + + histogram_entry = next(p for p in result["plots"] if p["id"] == "single_0_4") + histogram_data = json.loads(histogram_entry["plot"]) + + assert "histogram" in histogram_data + assert isinstance(histogram_data["histogram"], dict) + assert "mean" in histogram_data["histogram"] + assert "std" in histogram_data["histogram"] + assert isinstance(histogram_data["histogram"]["mean"], (int, float)) + assert isinstance(histogram_data["histogram"]["std"], (int, float)) def test_specifying_empty_extras_preserve_legacy_plotting(): - result = optimizer.run( + result = optimizer.run_optimizer( body={"data": sampleData, "optimizerConfig": sampleConfig, "extras": {}} ) validateResult(result) assert len(result["result"]["models"]) > 0 - assert len(result["plots"]) == 2 + assert len(result["plots"]) == 7 def test_deselecting_plots(): # If graphFormat is none, no plots should be returned. This should be faster. - result = optimizer.run( + result = optimizer.run_optimizer( body={ "data": sampleData, "optimizerConfig": sampleConfig, @@ -172,17 +209,79 @@ def test_deselecting_plots(): def test_can_accept_multi_objective_data(): - result = optimizer.run( - body={"data": sampleMultiObjectiveData, "optimizerConfig": sampleConfig} + result = optimizer.run_optimizer( + body={ + "data": sampleMultiObjectiveData, + "optimizerConfig": sampleConfig, + "extras": { + "experimentSuggestionCount": 1, + "graphFormat": "json", + "graphs": ["pareto", "single"], + "objectivePars": "expected_minimum", + }, + } ) validateResult(result) assert len(result["result"]["models"]) > 1 - assert len(result["plots"]) == 5 + assert "pareto_data" in [x["id"] for x in result["plots"]] + assert len(result["plots"]) == 11 + + +def test_multi_objective_json_single_plots(): + result = optimizer.run_optimizer( + body={ + "data": sampleMultiObjectiveData, + "optimizerConfig": sampleConfig, + "extras": { + "graphFormat": "json", + "graphs": ["single", "pareto"], + "includeModel": "false", + "experimentSuggestionCount": 1, + "objectivePars": "expected_minimum", + }, + } + ) + assert len(result["result"]["models"]) > 1 + plot_ids = [x["id"] for x in result["plots"]] + + assert len(result["plots"]) == 11 + + assert "pareto_data" in plot_ids + + assert "objective_1_data" not in plot_ids + assert "objective_2_data" not in plot_ids + + for objective_prefix in ["objective_1", "objective_2"]: + for idx in range(5): + plot_id = f"{objective_prefix}_{idx}" + assert plot_id in plot_ids, f"Expected {plot_id} in plot IDs" + + for objective_prefix in ["objective_1", "objective_2"]: + for dim_idx in range(4): + plot_id = f"{objective_prefix}_{dim_idx}" + plot_entry = next(x for x in result["plots"] if x["id"] == plot_id) + plot_data = json.loads(plot_entry["plot"]) + + assert "data" in plot_data + assert isinstance(plot_data["data"], list) + assert len(plot_data["data"]) == 4 + + for objective_prefix in ["objective_1", "objective_2"]: + histogram_id = f"{objective_prefix}_4" + histogram_entry = next(x for x in result["plots"] if x["id"] == histogram_id) + histogram_data = json.loads(histogram_entry["plot"]) + + assert "histogram" in histogram_data + assert isinstance(histogram_data["histogram"], dict) + assert "mean" in histogram_data["histogram"] + assert "std" in histogram_data["histogram"] + assert isinstance(histogram_data["histogram"]["mean"], (int, float)) + assert isinstance(histogram_data["histogram"]["std"], (int, float)) def test_deselecting_pickled_model(): # If includeModel is false, pickled data should not be included in result - result = optimizer.run( + result = optimizer.run_optimizer( body={ "data": sampleData, "optimizerConfig": sampleConfig, @@ -195,7 +294,7 @@ def test_deselecting_pickled_model(): def test_selecting_pickled_model(): # If includeModel is true, pickled data should be included in result - result = optimizer.run( + result = optimizer.run_optimizer( body={ "data": sampleData, "optimizerConfig": sampleConfig, @@ -207,7 +306,7 @@ def test_selecting_pickled_model(): def test_expected_minimum_contains_std_deviation(): - result = optimizer.run(body={"data": sampleData, "optimizerConfig": sampleConfig}) + result = optimizer.run_optimizer(body={"data": sampleData, "optimizerConfig": sampleConfig}) assert "expected_minimum" in result["result"] expected_minimum = result["result"]["expected_minimum"] assert isinstance(expected_minimum[1], collections.abc.Sequence) @@ -217,7 +316,7 @@ def test_expected_minimum_contains_std_deviation(): def test_when_using_constraints_set_constraints_should_be_called(mock): instance = mock.return_value request = brownie_with_constraints - optimizer.run(body=request) + optimizer.run_optimizer(body=request) instance.set_constraints.assert_called_once() @@ -225,7 +324,7 @@ def test_when_using_constraints_set_constraints_should_be_called(mock): def test_when_not_using_constraints_set_constraints_should_not_be_called(mock): instance = mock.return_value request = brownie_without_constraints - optimizer.run(body=request) + optimizer.run_optimizer(body=request) instance.set_constraints.assert_not_called() @@ -233,7 +332,7 @@ def test_when_not_using_constraints_set_constraints_should_not_be_called(mock): def test_when_using_constraints_strategy_cl_min_should_be_used(mock): instance = mock.return_value request = brownie_with_constraints - optimizer.run(body=request) + optimizer.run_optimizer(body=request) instance.ask.assert_called_once_with(n_points=3, strategy="cl_min") @@ -241,5 +340,467 @@ def test_when_using_constraints_strategy_cl_min_should_be_used(mock): def test_when_not_using_constraints_standard_strategy_should_be_used(mock): instance = mock.return_value request = brownie_without_constraints - optimizer.run(body=request) + optimizer.run_optimizer(body=request) instance.ask.assert_called_once_with(n_points=3) + + +def test_selectedPoint_single_objective_json(): + default_result = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"graphFormat": "json", "graphs": ["single"], "includeModel": "false"}, + }) + default_dim0_plot = next(p for p in default_result["plots"] if p["id"] == "single_0_0") + default_x_highlight = json.loads(default_dim0_plot["plot"])["data"][3] + selected_point = [100, 200, 300, "Mus"] + assert selected_point[0] != default_x_highlight, ( + "selected_point[0] must differ from default highlight for this test to be meaningful" + ) + result = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": { + "graphFormat": "json", + "graphs": ["single"], + "includeModel": "false", + "selectedPoint": selected_point, + }, + }) + assert len(result["plots"]) == len(default_result["plots"]) + dim0_plot = next(p for p in result["plots"] if p["id"] == "single_0_0") + plot_data = json.loads(dim0_plot["plot"]) + assert plot_data["data"][3] == selected_point[0] + + +def test_selectedPoint_multi_objective_json(): + selected_point = [651, 56, 722, "Ræv"] + result = optimizer.run_optimizer(body={ + "data": sampleMultiObjectiveData, + "optimizerConfig": sampleConfig, + "extras": { + "graphFormat": "json", + "graphs": ["single"], + "includeModel": "false", + "experimentSuggestionCount": 1, + "selectedPoint": selected_point, + }, + }) + obj1_dim0 = next(p for p in result["plots"] if p["id"] == "objective_1_0") + plot_data = json.loads(obj1_dim0["plot"]) + assert plot_data["data"][3] == selected_point[0] + obj2_dim0 = next(p for p in result["plots"] if p["id"] == "objective_2_0") + plot_data2 = json.loads(obj2_dim0["plot"]) + assert plot_data2["data"][3] == selected_point[0] + + +def test_selectedPoint_with_png_logs_warning_and_is_ignored(caplog): + import logging as _logging + + with caplog.at_level(_logging.WARNING, logger="optimizerapi.optimizer"): + result = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": { + "graphFormat": "png", + "selectedPoint": [651, 56, 722, "Ræv"], + "includeModel": "false", + }, + }) + + assert any("selectedPoint ignored on png path" in r.message for r in caplog.records) + # No crash, response is valid. + assert "plots" in result + assert "next" in result["result"] + + +def test_no_selectedPoint_preserves_default(): + result = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"graphFormat": "json", "graphs": ["single"], "includeModel": "false"}, + }) + assert len(result["plots"]) > 0 + single_plots = [p for p in result["plots"] if p["id"].startswith("single_")] + assert len(single_plots) > 0 + for p in single_plots: + plot_data = json.loads(p["plot"]) + if "histogram" not in plot_data: + assert "data" in plot_data + assert len(plot_data["data"]) == 4 + + +def test_selectedPoint_does_not_change_expected_minimum(): + default_result = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"graphFormat": "json", "graphs": ["single"], "includeModel": "false"}, + }) + selected_point = [651, 56, 722, "Ræv"] + result_with_selection = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": { + "graphFormat": "json", + "graphs": ["single"], + "includeModel": "false", + "selectedPoint": selected_point, + }, + }) + assert ( + default_result["result"]["models"][0]["expected_minimum"] + == result_with_selection["result"]["models"][0]["expected_minimum"] + ) + + +def test_pickled_consumption_skips_training(): + """Test that providing extras.pickled skips model retraining (tell() not called)""" + # First run to get pickled + first_result = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"includeModel": "true"}, + }) + pickled_value = first_result["result"]["pickled"] + assert len(pickled_value) > 0 + + # Second run with pickled - tell() should NOT be called + with patch("optimizerapi.optimizer.Optimizer.tell") as mock_tell: + second_result = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"pickled": pickled_value, "includeModel": "false"}, + }) + mock_tell.assert_not_called() + + assert "result" in second_result + assert "next" in second_result["result"] + assert len(second_result["result"]["next"]) > 0 + + +def test_old_format_pickled_falls_back(caplog): + """A pickled payload that decrypts but isn't the new dict shape falls through.""" + import logging as _logging + + old_format_data = ["some", "old", "data"] + old_pickled = pickleToString(old_format_data, get_crypto()) + + with caplog.at_level(_logging.WARNING, logger="optimizerapi.pickled_state"): + result = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"pickled": old_pickled, "includeModel": "false", "graphFormat": "json"}, + }) + + assert "result" in result + assert len(result["result"]["next"]) > 0 + assert result["result"]["extras"]["pickledUsed"] is False + assert any("bad_structure" in r.message for r in caplog.records) + + +def test_invalid_pickled_falls_back(caplog): + """An undecodable pickled string falls through to a full run.""" + import logging as _logging + + with caplog.at_level(_logging.WARNING, logger="optimizerapi.pickled_state"): + result = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"pickled": "this_is_not_valid_pickled_data_at_all", "includeModel": "false", "graphFormat": "json"}, + }) + + assert "result" in result + assert len(result["result"]["next"]) > 0 + assert result["result"]["extras"]["pickledUsed"] is False + assert any("decrypt_failed" in r.message for r in caplog.records) + + +def test_pickled_response_is_dict_format(): + """Test that pickled response is a dict with keys fingerprint, result, next, optimizer""" + result = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"includeModel": "true", "graphFormat": "json"}, + }) + pickled_value = result["result"]["pickled"] + assert len(pickled_value) > 0 + unpickled = unpickleFromString(pickled_value, get_crypto()) + assert isinstance(unpickled, dict), f"Expected dict, got {type(unpickled)}" + assert set(unpickled.keys()) >= {"fingerprint", "result", "next", "optimizer"} + assert isinstance(unpickled["fingerprint"], str) and len(unpickled["fingerprint"]) == 64 + + +def test_pickled_round_trip(): + """Test that pickled from run 1 can be sent as extras.pickled in run 2""" + # Run 1 + first_result = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"includeModel": "true"}, + }) + pickled_value = first_result["result"]["pickled"] + assert len(pickled_value) > 0 + + # Run 2 with pickled - should produce a valid response + second_result = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"pickled": pickled_value, "includeModel": "false"}, + }) + assert "result" in second_result + assert "next" in second_result["result"] + assert len(second_result["result"]["next"]) > 0 + # Verify plot structure is valid + assert "plots" in second_result + + +# Multi-objective 5-dim data from sample-multi.curl +sampleMultiObjective5DimData = [ + {"xi": [16.7, 500, 250, 20, "None"], "yi": [-2, -17]}, + {"xi": [50, 833, 150, 60, "Whipped cream"], "yi": [-3, -6]}, + {"xi": [58.3, 22, 85, 6, "Frosting"], "yi": [-6, -25]}, +] + +sampleMultiObjective5DimConfig = { + "baseEstimator": "GP", + "acqFunc": "EI", + "initialPoints": 3, + "kappa": 1.96, + "xi": 2, + "space": [ + {"type": "continuous", "name": "Sugar", "from": 0, "to": 100}, + {"type": "continuous", "name": "Flour", "from": 0, "to": 1000}, + {"type": "discrete", "name": "Temperature", "from": 0, "to": 300}, + {"type": "discrete", "name": "Time", "from": 0, "to": 120}, + {"type": "category", "name": "Finish", "categories": ["None", "Frosting", "Whipped cream"]}, + ], + "constraints": [], +} + + +def test_pickled_with_selectedPoint(): + """Integration test: pickled from run 1 consumed in run 2 with selectedPoint from pareto front""" + # Run 1: multi-objective JSON request — get pickled + run1_result = optimizer.run_optimizer(body={ + "data": sampleMultiObjective5DimData, + "optimizerConfig": sampleMultiObjective5DimConfig, + "extras": { + "experimentSuggestionCount": 1, + "graphs": ["pareto", "single"], + "graphFormat": "json", + "includeModel": "true", + }, + }) + assert "result" in run1_result + pickled_value = run1_result["result"]["pickled"] + assert len(pickled_value) > 0 + + # Extract a point from pareto front_x_data + pareto_plot_entry = next(p for p in run1_result["plots"] if p["id"] == "pareto_data") + pareto_data = json.loads(pareto_plot_entry["plot"]) + front_x_data = pareto_data["front_x_data"] + assert len(front_x_data) > 0 + selected_point = front_x_data[0] + + # Run 2: same request + pickled + selectedPoint + run2_result = optimizer.run_optimizer(body={ + "data": sampleMultiObjective5DimData, + "optimizerConfig": sampleMultiObjective5DimConfig, + "extras": { + "experimentSuggestionCount": 1, + "graphs": ["pareto", "single"], + "graphFormat": "json", + "pickled": pickled_value, + "selectedPoint": selected_point, + }, + }) + + # Assert valid response + assert "result" in run2_result + assert "plots" in run2_result + assert len(run2_result["plots"]) > 0 + assert "next" in run2_result["result"] + assert len(run2_result["result"]["next"]) > 0 + + # Assert new pickled is present + new_pickled = run2_result["result"]["pickled"] + assert len(new_pickled) > 0 + + # Assert selectedPoint is reflected in objective_1_0 plot + obj1_dim0 = next(p for p in run2_result["plots"] if p["id"] == "objective_1_0") + plot_data = json.loads(obj1_dim0["plot"]) + assert plot_data["data"][3] == selected_point[0] + + +def test_pickled_consumption_with_include_model_false(): + """Integration test: pickled consumed + includeModel=false → pickled suppressed in response""" + # Run 1: get pickled + run1_result = optimizer.run_optimizer(body={ + "data": sampleMultiObjective5DimData, + "optimizerConfig": sampleMultiObjective5DimConfig, + "extras": { + "experimentSuggestionCount": 1, + "graphs": ["pareto", "single"], + "graphFormat": "json", + "includeModel": "true", + }, + }) + pickled_value = run1_result["result"]["pickled"] + assert len(pickled_value) > 0 + + # Run 2: consume pickled + includeModel=false + run2_result = optimizer.run_optimizer(body={ + "data": sampleMultiObjective5DimData, + "optimizerConfig": sampleMultiObjective5DimConfig, + "extras": { + "experimentSuggestionCount": 1, + "graphs": ["pareto", "single"], + "graphFormat": "json", + "pickled": pickled_value, + "includeModel": "false", + }, + }) + + # includeModel=false suppresses pickled + assert "result" in run2_result + assert "pickled" in run2_result["result"] + assert run2_result["result"]["pickled"] == "" + + # Response is otherwise valid + assert "next" in run2_result["result"] + assert len(run2_result["result"]["next"]) > 0 + assert "plots" in run2_result + assert len(run2_result["plots"]) > 0 + + +def test_pickled_used_flag_round_trip(): + """First run has pickledUsed=False; second run with returned pickled has pickledUsed=True.""" + first = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"includeModel": "true", "graphFormat": "json"}, + }) + assert first["result"]["extras"]["pickledUsed"] is False + pickled_value = first["result"]["pickled"] + assert len(pickled_value) > 0 + + second = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"includeModel": "true", "pickled": pickled_value, "graphFormat": "json"}, + }) + assert second["result"]["extras"]["pickledUsed"] is True + + +def test_selectedPoint_with_no_data(): + """Integration test: selectedPoint with empty data — server handles gracefully""" + selected_point = [50, 833, 150, 60, "Whipped cream"] + result = optimizer.run_optimizer(body={ + "data": [], + "optimizerConfig": sampleMultiObjective5DimConfig, + "extras": { + "experimentSuggestionCount": 1, + "graphs": ["pareto", "single"], + "graphFormat": "json", + "selectedPoint": selected_point, + }, + }) + + # Server handles gracefully — no exception, valid response + assert "result" in result + # No data means no models trained, so no plots + assert "plots" in result + + +def test_equivalence_with_and_without_pickled_multi_objective(): + """Same selectedPoint, same data: pickled vs no-pickled produce identical plots (all ids).""" + base_body = { + "data": sampleMultiObjective5DimData, + "optimizerConfig": sampleMultiObjective5DimConfig, + "extras": { + "graphs": ["single", "pareto"], + "graphFormat": "json", + "selectedPoint": [50, 833, 150, 60, "Whipped cream"], + "includeModel": "true", + }, + } + + # Seed: one full run to capture pickled. + seed = optimizer.run_optimizer(body=copy.deepcopy({ + **base_body, + "extras": {**base_body["extras"], "selectedPoint": None}, + })) + pickled_value = seed["result"]["pickled"] + assert len(pickled_value) > 0 + + # Run A: no pickled (full retrain), with selectedPoint. + body_no_pickle = copy.deepcopy(base_body) + result_no_pickle = optimizer.run_optimizer(body=body_no_pickle) + + # Run B: same request + pickled. + body_with_pickle = copy.deepcopy(base_body) + body_with_pickle["extras"]["pickled"] = pickled_value + result_with_pickle = optimizer.run_optimizer(body=body_with_pickle) + + # Sanity: fast path actually engaged. + assert result_with_pickle["result"]["extras"]["pickledUsed"] is True + assert result_no_pickle["result"]["extras"]["pickledUsed"] is False + + # Compare every plot entry except the pickled string itself (which is + # not in plots) — identical plot ids and identical plot bodies. + plots_no_pickle = {p["id"]: p["plot"] for p in result_no_pickle["plots"]} + plots_with_pickle = {p["id"]: p["plot"] for p in result_with_pickle["plots"]} + assert set(plots_no_pickle) == set(plots_with_pickle) + for plot_id in plots_no_pickle: + assert plots_no_pickle[plot_id] == plots_with_pickle[plot_id], ( + f"divergence on plot {plot_id}" + ) + + +def test_pickled_fingerprint_mismatch_falls_through(caplog): + """A pickled produced from one data set is ignored when data changes.""" + import logging as _logging + + seed = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"includeModel": "true", "graphFormat": "json"}, + }) + pickled_value = seed["result"]["pickled"] + assert len(pickled_value) > 0 + + altered_data = sampleData + [{"xi": [100, 100, 100, "Mus"], "yi": [0.5]}] + + with caplog.at_level(_logging.WARNING, logger="optimizerapi.pickled_state"): + result = optimizer.run_optimizer(body={ + "data": altered_data, + "optimizerConfig": sampleConfig, + "extras": {"includeModel": "true", "pickled": pickled_value, "graphFormat": "json"}, + }) + + assert result["result"]["extras"]["pickledUsed"] is False + assert any("fingerprint_mismatch" in r.message for r in caplog.records) + # Response still valid — server fell through to a full run. + assert "next" in result["result"] + assert len(result["result"]["next"]) > 0 + + +def test_includeModel_false_with_pickled_logs_chain_break_warning(caplog): + import logging as _logging + + seed = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"includeModel": "true", "graphFormat": "json"}, + }) + pickled_value = seed["result"]["pickled"] + + with caplog.at_level(_logging.WARNING, logger="optimizerapi.optimizer"): + second = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"pickled": pickled_value, "includeModel": "false", "graphFormat": "json"}, + }) + + assert any("includeModel=false with extras.pickled" in r.message for r in caplog.records) + # Contract preserved: empty pickled returned. + assert second["result"]["pickled"] == "" diff --git a/tests/test_pickled_state.py b/tests/test_pickled_state.py new file mode 100644 index 0000000..918dad2 --- /dev/null +++ b/tests/test_pickled_state.py @@ -0,0 +1,114 @@ +"""Tests for pickled_state: fingerprint + pack/unpack helpers.""" + +import logging + +from optimizerapi.pickled_state import compute_fingerprint, pack, unpack_if_valid +from optimizerapi.securepickle import get_crypto, pickleToString + + +SAMPLE_DATA = [ + {"xi": [651, 56, 722, "Ræv"], "yi": [1]}, + {"xi": [651, 42, 722, "Ræv"], "yi": [0.2]}, +] + +SAMPLE_CONFIG = { + "baseEstimator": "GP", + "acqFunc": "gp_hedge", + "initialPoints": 2, + "kappa": 1.96, + "xi": 0.012, + "space": [ + {"type": "discrete", "name": "Sukker", "from": 0, "to": 1000}, + {"type": "continuous", "name": "Peber", "from": 0, "to": 1000}, + {"type": "continuous", "name": "Hvedemel", "from": 0, "to": 1000}, + {"type": "category", "name": "Kunde", "categories": ["Mus", "Ræv"]}, + ], +} + + +def test_fingerprint_is_deterministic(): + fp1 = compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) + fp2 = compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) + assert fp1 == fp2 + assert isinstance(fp1, str) + assert len(fp1) == 64 # sha256 hex + + +def test_fingerprint_is_independent_of_dict_key_order(): + reordered_config = { + "space": SAMPLE_CONFIG["space"], + "xi": SAMPLE_CONFIG["xi"], + "kappa": SAMPLE_CONFIG["kappa"], + "initialPoints": SAMPLE_CONFIG["initialPoints"], + "acqFunc": SAMPLE_CONFIG["acqFunc"], + "baseEstimator": SAMPLE_CONFIG["baseEstimator"], + } + assert compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) == compute_fingerprint( + SAMPLE_DATA, reordered_config + ) + + +def test_fingerprint_changes_when_data_changes(): + other_data = SAMPLE_DATA + [{"xi": [100, 100, 100, "Mus"], "yi": [0.5]}] + assert compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) != compute_fingerprint( + other_data, SAMPLE_CONFIG + ) + + +def test_fingerprint_changes_when_config_changes(): + other_config = dict(SAMPLE_CONFIG, kappa=2.0) + assert compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) != compute_fingerprint( + SAMPLE_DATA, other_config + ) + + +def _crypto(): + return get_crypto() + + +def test_pack_unpack_round_trip(): + crypto = _crypto() + fingerprint = compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) + blob = pack(result="result-sentinel", next_points=["next-sentinel"], + optimizer="opt-sentinel", fingerprint=fingerprint, crypto=crypto) + assert isinstance(blob, str) and len(blob) > 0 + + payload = unpack_if_valid(blob, expected_fingerprint=fingerprint, crypto=crypto) + assert payload is not None + assert payload["result"] == "result-sentinel" + assert payload["next"] == ["next-sentinel"] + assert payload["optimizer"] == "opt-sentinel" + assert payload["fingerprint"] == fingerprint + + +def test_unpack_returns_none_on_fingerprint_mismatch(caplog): + crypto = _crypto() + fingerprint = compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) + blob = pack(result="r", next_points=[], optimizer="o", + fingerprint=fingerprint, crypto=crypto) + + with caplog.at_level(logging.WARNING, logger="optimizerapi.pickled_state"): + result = unpack_if_valid(blob, expected_fingerprint="0" * 64, crypto=crypto) + assert result is None + assert any("fingerprint_mismatch" in record.message for record in caplog.records) + + +def test_unpack_returns_none_on_decrypt_failure(caplog): + crypto = _crypto() + with caplog.at_level(logging.WARNING, logger="optimizerapi.pickled_state"): + result = unpack_if_valid("not-a-valid-blob", expected_fingerprint="x" * 64, + crypto=crypto) + assert result is None + assert any("decrypt_failed" in record.message for record in caplog.records) + + +def test_unpack_returns_none_on_bad_structure(caplog): + crypto = _crypto() + # Encrypt a non-dict payload using the same machinery. + bogus_blob = pickleToString(["not", "a", "dict"], crypto) + + with caplog.at_level(logging.WARNING, logger="optimizerapi.pickled_state"): + result = unpack_if_valid(bogus_blob, expected_fingerprint="x" * 64, + crypto=crypto) + assert result is None + assert any("bad_structure" in record.message for record in caplog.records) diff --git a/tests/test_plot_emitters.py b/tests/test_plot_emitters.py new file mode 100644 index 0000000..f242509 --- /dev/null +++ b/tests/test_plot_emitters.py @@ -0,0 +1,67 @@ +"""Tests for the plot-emitters module.""" + +import json + +import numpy +import pytest + +from optimizerapi.plot_emitters import emit_json_single_plots + + +class _FakePlotData: + """Mimics ProcessOptimizer's get_Brownie_Bee_1d_plot return value.""" + + @staticmethod + def make(n_dims: int): + # Each dimension yields a 4-element series; the last entry is a + # histogram (mean, std) pair. + dims = [[[1.0, 2.0, 3.0, 4.0] for _ in range(4)] for _ in range(n_dims)] + histogram = (numpy.array([42.0]), numpy.array([1.5])) + return dims + [histogram] + + +@pytest.fixture +def fake_brownie_1d(monkeypatch): + """Patch _get_brownie_bee_1d_plot_safe so this test doesn't need ProcessOptimizer.""" + def _stub(result, x_eval=None): + return _FakePlotData.make(n_dims=3) + monkeypatch.setattr( + "optimizerapi.plot_emitters._get_brownie_bee_1d_plot_safe", _stub + ) + + +def test_emit_json_single_plots_writes_one_entry_per_dim_plus_histogram(fake_brownie_1d): + plots: list = [] + emit_json_single_plots(plots, result=object(), prefix="single_0", selected_point=None) + ids = [p["id"] for p in plots] + assert ids == ["single_0_0", "single_0_1", "single_0_2", "single_0_3"] + # Last entry is the histogram payload. + histogram_payload = json.loads(plots[-1]["plot"]) + assert "histogram" in histogram_payload + assert histogram_payload["histogram"]["mean"] == 42.0 + assert histogram_payload["histogram"]["std"] == 1.5 + + +def test_emit_json_single_plots_supports_different_prefixes(fake_brownie_1d): + plots: list = [] + emit_json_single_plots(plots, result=object(), prefix="objective_2", selected_point=None) + ids = [p["id"] for p in plots] + assert ids[0] == "objective_2_0" + assert ids[-1] == "objective_2_3" + + +def test_emit_json_single_plots_passes_selected_point_through(monkeypatch): + captured = {} + + def _stub(result, x_eval=None): + captured["x_eval"] = x_eval + return _FakePlotData.make(n_dims=2) + + monkeypatch.setattr( + "optimizerapi.plot_emitters._get_brownie_bee_1d_plot_safe", _stub + ) + + plots: list = [] + sp = [50, 833, "Whipped cream"] + emit_json_single_plots(plots, result=object(), prefix="single_0", selected_point=sp) + assert captured["x_eval"] == sp diff --git a/tests/test_types.py b/tests/test_types.py new file mode 100644 index 0000000..46026e3 --- /dev/null +++ b/tests/test_types.py @@ -0,0 +1,65 @@ +"""Sanity checks for the boundary type definitions.""" + +from optimizerapi.types import ( # noqa: F401 + CachePayload, + Constraint, + DataPoint, + Dimension, + Extras, + OptimizerConfig, + Plot, + RequestBody, + ResponseEnvelope, + ResponseResult, + ResponseResultExtras, +) + + +def test_request_body_accepts_realistic_request(): + """A typical request body matches the RequestBody shape at runtime. + + TypedDicts don't enforce at runtime; we use isinstance(d, dict) as a + proxy and let mypy catch shape mismatches statically. + """ + body: RequestBody = { + "data": [{"xi": [50, 833, 150, 60, "Whipped cream"], "yi": [-3, -6]}], + "optimizerConfig": { + "baseEstimator": "GP", + "acqFunc": "EI", + "initialPoints": 3, + "kappa": 1.96, + "xi": 2, + "space": [ + {"type": "continuous", "name": "Sugar", "from": 0, "to": 100}, + {"type": "category", "name": "Finish", + "categories": ["None", "Whipped cream"]}, + ], + "constraints": [], + }, + "extras": {"graphFormat": "json", "includeModel": "true"}, + } + assert isinstance(body, dict) + assert body["data"][0]["xi"][-1] == "Whipped cream" + + +def test_response_envelope_shape(): + response: ResponseEnvelope = { + "plots": [{"id": "single_0_0", "plot": "{}"}], + "result": { + "next": [[1, 2, 3]], + "models": [], + "pickled": "", + "extras": {"pickledUsed": False}, + }, + } + assert response["result"]["extras"]["pickledUsed"] is False + + +def test_cache_payload_shape(): + payload: CachePayload = { + "fingerprint": "a" * 64, + "result": "opaque", + "next": [[1, 2]], + "optimizer": "opaque", + } + assert payload["fingerprint"] == "a" * 64