From 48434361d5aef9866c005b881415cdb275f392f5 Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Sat, 30 May 2026 07:57:09 +0000 Subject: [PATCH 01/14] build: modernize packaging and toolchain Replace requirements*.txt with a standard pyproject.toml, target Python 3.13 (matching Pyodide 0.29.x), and add mise.toml plus uv-based dev loop. Updates Dockerfile, pytest.ini, and .gitignore accordingly. --- .gitignore | 9 +++++- Dockerfile | 9 +++--- mise.toml | 8 +++++ pyproject.toml | 63 +++++++++++++++++++++++++++++++++++++++ pytest.ini | 5 +++- requirements-dev.txt | 3 -- requirements-freeze.txt | 65 ----------------------------------------- requirements.txt | 20 ------------- 8 files changed, 87 insertions(+), 95 deletions(-) create mode 100644 mise.toml create mode 100644 pyproject.toml delete mode 100644 requirements-dev.txt delete mode 100644 requirements-freeze.txt delete mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index 6036b2a..814de84 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,11 @@ .vscode /tmp version.txt -.env \ No newline at end of file +.env + +*.egg-info/ +dist/ +build/ +*.egg +.sisyphus +/.claude diff --git a/Dockerfile b/Dockerfile index 79d40a4..0ac7d5c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,11 +6,10 @@ FROM python:3.9-bullseye AS builder RUN python3 -m venv /opt/venv ENV PATH="/opt/venv/bin:${PATH}" -RUN pip install --upgrade pip && pip install pip-tools && pip install --upgrade pip +RUN pip install --upgrade pip -COPY requirements-freeze.txt . -RUN cat requirements-freeze.txt | grep --invert-match pkg_resources > requirements-fixed.txt -RUN pip install -r requirements-fixed.txt +COPY pyproject.toml . +RUN pip install . # Second stage @@ -29,7 +28,7 @@ RUN mkdir -p /code/mapplotlib USER user -COPY --from=builder /requirements-fixed.txt /code/requirements-freeze.txt +COPY --from=builder /pyproject.toml /code/pyproject.toml #COPY version.txt /code RUN echo "${VERSION}-${SHA}" > /code/version.txt COPY optimizerapi/ /code/optimizerapi diff --git a/mise.toml b/mise.toml new file mode 100644 index 0000000..f5142b3 --- /dev/null +++ b/mise.toml @@ -0,0 +1,8 @@ +[tools] +python = "3.13" +uv = "latest" + +[env] +_.python.venv = { + path = ".venv", create = true, +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9e2d2c7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,63 @@ +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "process-optimizer-api" +version = "1.0.0" +description = "REST based API for ProcessOptimizer" +readme = "README.md" +requires-python = ">=3.13" +license = { text = "MIT" } +authors = [{ name = "BoostV" }] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.13", +] + +dependencies = [ + "connexion[swagger-ui]==2.14.2", + "python-keycloak==2.13.2", + "Flask==2.2.3", + "Flask-Cors==3.0.10", + # develop as of 2026-03-11 + "ProcessOptimizer[brownie-bee] @ git+https://github.com/novonordisk-research/ProcessOptimizer.git@f1c8637d5eab", + "json-tricks==3.15.5", + "jsonschema>=4.21,<5", + "cryptography>=44,<46", + "waitress==2.1.2", + "rq==2.0.0", +] + +[project.optional-dependencies] +dev = ["pytest==8.3.3", "pytest-watch==4.2.0", "flake8>=7", "mypy>=1.13"] + +[project.urls] +Homepage = "https://github.com/BoostV/process-optimizer-api" +Repository = "https://github.com/BoostV/process-optimizer-api" +Issues = "https://github.com/BoostV/process-optimizer-api/issues" + +[tool.setuptools.packages.find] +include = ["optimizerapi*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = "test_*.py" +python_classes = "Test*" +python_functions = "test_*" + +[tool.mypy] +files = ["optimizerapi"] +python_version = "3.13" +strict = false +# Start permissive; we tighten as Phase 4 lands. +warn_unused_ignores = true +warn_redundant_casts = true +warn_return_any = true +disallow_any_unimported = false +check_untyped_defs = true +# Third-party libraries that don't ship type stubs. +ignore_missing_imports = true diff --git a/pytest.ini b/pytest.ini index ba8ba7e..4ed8dd9 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,4 +2,7 @@ filterwarnings = error ignore::UserWarning - ignore:.*np\.int.*is a deprecated alias.*:DeprecationWarning \ No newline at end of file + ignore:.*np\.int.*is a deprecated alias.*:DeprecationWarning + ignore:Accessing jsonschema\.draft4_format_checker.*:DeprecationWarning + ignore:jsonschema\.RefResolver is deprecated.*:DeprecationWarning + ignore:jsonschema\.exceptions\.RefResolutionError is deprecated.*:DeprecationWarning \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 37019c1..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,3 +0,0 @@ --r requirements.txt -pytest==8.3.3 -pytest-watch==4.2.0 diff --git a/requirements-freeze.txt b/requirements-freeze.txt deleted file mode 100644 index 1f37a8c..0000000 --- a/requirements-freeze.txt +++ /dev/null @@ -1,65 +0,0 @@ -async-timeout==4.0.2 -attrs==23.1.0 -bokeh==2.4.3 -certifi==2022.12.7 -cffi==1.15.1 -charset-normalizer==3.1.0 -click==8.1.3 -clickclick==20.10.2 -colorama==0.4.6 -connexion==2.14.2 -cryptography==3.4.7 -cycler==0.11.0 -deap==1.3.3 -deprecation==2.1.0 -docopt==0.6.2 -ecdsa==0.18.0 -exceptiongroup==1.2.2 -Flask==2.2.3 -Flask-Cors==3.0.10 -fonttools==4.39.3 -idna==3.4 -inflection==0.5.1 -iniconfig==2.0.0 -itsdangerous==2.1.2 -Jinja2==3.1.2 -joblib==1.2.0 -json-tricks==3.15.5 -jsonschema==4.15.0 -kiwisolver==1.4.4 -MarkupSafe==2.1.2 -matplotlib==3.5.3 -numpy==1.23.3 -packaging==23.1 -Pillow==9.5.0 -pluggy==1.5.0 -ProcessOptimizer==1.0.2 -py==1.11.0 -pyasn1==0.4.8 -pycparser==2.21 -pyparsing==3.0.9 -pyrsistent==0.19.3 -pytest==8.3.3 -pytest-watch==4.2.0 -python-dateutil==2.8.2 -python-jose==3.3.0 -python-keycloak==2.13.2 -PyYAML==6.0 -redis==4.5.4 -requests==2.28.2 -requests-toolbelt==0.10.1 -rq==2.0.0 -rsa==4.9 -scikit-learn==1.1.2 -scipy==1.9.1 -six==1.16.0 -swagger-ui-bundle==0.0.9 -threadpoolctl==3.1.0 -toml==0.10.2 -tomli==2.1.0 -tornado==6.2 -typing_extensions==4.5.0 -urllib3==1.26.15 -waitress==2.1.2 -watchdog==6.0.0 -Werkzeug==2.2.3 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 8afc15b..0000000 --- a/requirements.txt +++ /dev/null @@ -1,20 +0,0 @@ -connexion==2.14.2 -connexion[swagger-ui] -python-keycloak==2.13.2 -Flask==2.2.3 -Flask-Cors==3.0.10 -ProcessOptimizer==1.0.2 -json-tricks==3.15.5 -jsonschema==4.15.0 -cryptography==3.4.7 -waitress==2.1.2 -rq==2.0.0 -# ProcessOptimizer requirements -numpy==1.23.3 -matplotlib==3.5.3 -scipy==1.9.1 -bokeh==2.4.3 -scikit-learn==1.1.2 -six==1.16.0 -deap==1.3.3 -pyYAML==6.0 From d3eaffb851832a45f228e725b068326b0c6154b7 Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Sat, 30 May 2026 07:57:14 +0000 Subject: [PATCH 02/14] feat(optimizer): multi-objective support, pareto plots, and JSON single-plot emission Add ProcessOptimizer-backed multi-objective optimization with pareto plots and per-dimension JSON single plots, and extract plot emission into a dedicated plot_emitters module. Includes the optimizer refactors that split process_result into focused helpers. --- optimizerapi/health.py | 2 +- optimizerapi/optimizer.py | 380 ++++++++++++++++++++++------------ optimizerapi/plot_emitters.py | 163 +++++++++++++++ 3 files changed, 408 insertions(+), 137 deletions(-) create mode 100644 optimizerapi/plot_emitters.py diff --git a/optimizerapi/health.py b/optimizerapi/health.py index f658ce6..e9b8b71 100644 --- a/optimizerapi/health.py +++ b/optimizerapi/health.py @@ -1,2 +1,2 @@ -def check() -> str: +def health_check() -> str: return "OK" diff --git a/optimizerapi/optimizer.py b/optimizerapi/optimizer.py index 69ac65c..0c0cf17 100644 --- a/optimizerapi/optimizer.py +++ b/optimizerapi/optimizer.py @@ -4,38 +4,151 @@ It should only depend on ProcessOptimizer specifics and json related features. """ +import importlib.metadata +import json +import logging import os import platform -from time import strftime -import base64 -import io -import json import subprocess +from dataclasses import dataclass +from time import strftime + import json_tricks +import matplotlib.pyplot as plt +import numpy from ProcessOptimizer import Optimizer, expected_minimum -from ProcessOptimizer.plots import ( - plot_objective, - plot_convergence, - plot_Pareto, - plot_brownie_bee_frontend, -) from ProcessOptimizer.space import Real from ProcessOptimizer.space.constraints import SumEquals -import matplotlib.pyplot as plt -import numpy -from .securepickle import pickleToString, get_crypto +from typing import TYPE_CHECKING, Any, cast + +from .securepickle import get_crypto +from .pickled_state import compute_fingerprint, pack, unpack_if_valid +from .plot_emitters import emit_json_single_plots, emit_pareto_data, emit_png_plots + +if TYPE_CHECKING: + from .types import Extras, OptimizerConfig, Plot, RequestBody numpy.random.seed(42) plt.switch_backend("Agg") -def run(body) -> dict: - """ "Handle the run request""" +@dataclass(frozen=True) +class _ParsedExtras: + graph_format: str + max_quality: int + graphs_to_return: list[str] + objective_pars: str + include_model: bool + selected_point: "list[str | float] | None" + experiment_suggestion_count: int + + +def _parse_bool(value: object, default: bool = True) -> bool: + """Coerce extras' stringly-typed boolean fields. + + Accepts the literals "true" / "false" (case-insensitive), real bools, + and JSON-style ``true``/``false``. Anything else falls back to *default*. + """ + if isinstance(value, bool): + return value + if value is None: + return default + text = str(value).strip().lower() + if text in ("true", "1", "yes"): + return True + if text in ("false", "0", "no"): + return False + return default + + +def _parse_extras(extras: "Extras", logger: "logging.Logger") -> _ParsedExtras: + """Read the request ``extras`` block into a typed view. + + Emits the PNG + selectedPoint warning here so callers don't need to + duplicate the check. + """ + graph_format = extras.get("graphFormat", "png") + selected_point = extras.get("selectedPoint") + if selected_point is not None and graph_format != "json": + logger.warning( + "selectedPoint ignored on png path (graphFormat=%s)", graph_format + ) + return _ParsedExtras( + graph_format=graph_format, + max_quality=int(extras.get("maxQuality", 5)), + graphs_to_return=list(extras.get( + "graphs", ["objective", "convergence", "pareto", "single"] + )), + objective_pars=extras.get("objectivePars", "result"), + include_model=_parse_bool(extras.get("includeModel", "true")), + selected_point=selected_point, + experiment_suggestion_count=int(extras.get("experimentSuggestionCount", 1)), + ) + + +def _compute_next_experiments( + optimizer: Any, + cfg: "OptimizerConfig", + n_points: int, +) -> list[list[str | float]]: + """Ask the optimizer for the next N experiments, normalising the shape. + + ``optimizer.ask`` can return either a single experiment (flat list) or + a list of experiments. We always return a list of lists. + """ + constraints = cfg.get("constraints", []) + if constraints: + next_exp = optimizer.ask(n_points=n_points, strategy="cl_min") + else: + next_exp = optimizer.ask(n_points=n_points) + if next_exp and not any(isinstance(x, list) for x in next_exp): + next_exp = [next_exp] + return cast(list[list[str | float]], round_to_length_scales(next_exp, optimizer.space)) + + +def _flatten_expected_minima(models: list[dict]) -> None: + """In-place flatten of nested ``expected_minimum`` entries on each model. + + The pre-flatten shape from ``process_model`` can be a list of mixed + scalars and lists; the response contract is a single flat list inside + a one-element outer list. This function normalises that. + """ + for model in models: + flat = [] + for x in model["expected_minimum"]: + if isinstance(x, list): + flat.extend(x) + else: + flat.append(x) + model["expected_minimum"] = [flat] + + +def _set_expected_minimum( + result_details: dict, + single_result: object, + space: object, +) -> None: + """Compute and store the expected minimum (single-objective only).""" + minimum = expected_minimum(single_result, return_std=True) + result_details["expected_minimum"] = [ + round_to_length_scales(minimum[0], space), + minimum[1], + ] + + +def run(body: "RequestBody") -> dict: + """Handle the run request. + + Returns the response envelope as a plain ``dict`` — the json_tricks + round-trip at the bottom of the function drops NumPy types, which is + why we cannot return ``ResponseEnvelope`` directly without an + explicit cast. + """ data = [(run["xi"], run["yi"]) for run in body["data"]] cfg = body["optimizerConfig"] - constraints = cfg["constraints"] if "constraints" in cfg else [] - extras = body["extras"] if "extras" in body else {} + constraints = cfg.get("constraints", []) + extras = body.get("extras", {}) use_actual_measurement_histogram = json.loads( extras.get("useActualMeasurementHistogram", "true").lower() ) @@ -45,7 +158,7 @@ def run(body) -> dict: convert_number_type(x["from"], x["type"]), convert_number_type(x["to"], x["type"]), ) - if (x["type"] == "discrete" or x["type"] == "continuous") + if x["type"] in ("discrete", "continuous") else tuple(x["categories"]) ) for x in cfg["space"] @@ -67,6 +180,37 @@ def run(body) -> dict: if len(Yi) > 0: n_objectives = len(Yi[0]) + request_fingerprint = compute_fingerprint(body["data"], cfg) + pickled_input = extras.get("pickled", "") + if pickled_input: + include_model_str = str(extras.get("includeModel", "true")).lower() + if include_model_str == "false": + logging.getLogger(__name__).warning( + "includeModel=false with extras.pickled present — if the cache hint is used, " + "the next call will not have one to reuse" + ) + cached = unpack_if_valid( + pickled_input, expected_fingerprint=request_fingerprint, crypto=get_crypto() + ) if pickled_input else None + + if cached is not None: + result = cached["result"] + optimizer = cached["optimizer"] + response = process_result( + result, optimizer, dimensions, cfg, extras, data, space, + request_fingerprint=request_fingerprint, pickled_used=True, + ) + response["result"]["extras"]["parameters"] = { + "dimensions": dimensions, + "space": space, + "hyperparams": hyperparams, + "Xi": Xi, + "Yi": Yi, + "extras": extras, + } + # json_tricks roundtrip drops NumPy types and produces a plain dict + return cast(dict[Any, Any], json.loads(json_tricks.dumps(response))) + if constraints is not None and len(constraints) > 0: optimizer = Optimizer( space, **hyperparams, lhs=False, n_objectives=n_objectives @@ -90,7 +234,10 @@ def run(body) -> dict: else: result = [] - response = process_result(result, optimizer, dimensions, cfg, extras, data, space) + response = process_result( + result, optimizer, dimensions, cfg, extras, data, space, + request_fingerprint=request_fingerprint, pickled_used=False, + ) response["result"]["extras"]["parameters"] = { "dimensions": dimensions, @@ -103,7 +250,8 @@ def run(body) -> dict: # It is necesarry to convert response to a json string and then back to # dictionary because NumPy types are not serializable by default - return json.loads(json_tricks.dumps(response)) + # json_tricks roundtrip drops NumPy types and produces a plain dict + return cast(dict[Any, Any], json.loads(json_tricks.dumps(response))) def convert_number_type(value, num_type): @@ -113,7 +261,18 @@ def convert_number_type(value, num_type): return float(value) -def process_result(result, optimizer, dimensions, cfg, extras, data, space): +def process_result( + result: Any, + optimizer: Any, + dimensions: list[str], + cfg: "OptimizerConfig", + extras: "Extras", + data: list[tuple[list[str | float], list[float]]], + space: list, + *, + request_fingerprint: str, + pickled_used: bool, +) -> dict: """Extracts results from the OptimizerResult. Parameters @@ -146,95 +305,77 @@ def process_result(result, optimizer, dimensions, cfg, extras, data, space): model representation etc.} } """ - result_details = {"next": [], "models": [], "pickled": "", "extras": {}} - plots = [] - response = {"plots": plots, "result": result_details} + result_details: dict[str, Any] = {"next": [], "models": [], "pickled": "", "extras": {}} + plots: "list[Plot]" = [] + response: dict[str, Any] = {"plots": plots, "result": result_details} # GraphFormat should, at the moment, be either "png" or "none". Default (legacy) # behavior is "png", so the API returns png images. Any other input is interpreted # as "None" at the moment. - graph_format = extras.get("graphFormat", "png") - max_quality = int(extras.get("maxQuality", "5")) - graphs_to_return = extras.get("graphs", ["objective", "convergence", "pareto"]) - - objective_pars = extras.get("objectivePars", "result") - - pickle_model = json.loads(extras.get("includeModel", "true").lower()) + parsed = _parse_extras(extras, logging.getLogger(__name__)) - # In the following section details that should be reported to - # clients should go into the "resultDetails" dictionary and plots - # go into the "plots" list (this is handled by calling the "addPlot" function) - experiment_suggestion_count = 1 - if "experimentSuggestionCount" in extras: - experiment_suggestion_count = extras["experimentSuggestionCount"] - - if "constraints" in cfg and len(cfg["constraints"]) > 0: - next_exp = optimizer.ask( - n_points=experiment_suggestion_count, strategy="cl_min" - ) - else: - next_exp = optimizer.ask(n_points=experiment_suggestion_count) - if len(next_exp) > 0 and not any(isinstance(x, list) for x in next_exp): - next_exp = [next_exp] - result_details["next"] = round_to_length_scales(next_exp, optimizer.space) + result_details["next"] = _compute_next_experiments( + optimizer, cfg, parsed.experiment_suggestion_count + ) if len(data) >= cfg["initialPoints"]: # Some calculations are only possible if the model has # processed more than "initialPoints" data points result_details["models"] = [process_model(model, optimizer) for model in result] - if graph_format == "png": + if parsed.graph_format == "png": + emit_png_plots( + plots, + result=result, + dimensions=dimensions, + graphs=parsed.graphs_to_return, + max_quality=parsed.max_quality, + objective_pars=parsed.objective_pars, + ) + if optimizer.n_objectives == 1: + _set_expected_minimum(result_details, result[0], optimizer.space) + elif parsed.graph_format == "json": for idx, model in enumerate(result): - if "single" in graphs_to_return: - bb_plots = plot_brownie_bee_frontend(model, max_quality=max_quality) - for i, plot in enumerate(bb_plots): - pic_io_bytes = io.BytesIO() - plot.savefig(pic_io_bytes, format="png") - pic_io_bytes.seek(0) - pic_hash = base64.b64encode(pic_io_bytes.read()) - plots.append( - {"id": f"single_{idx}_{i}", "plot": str(pic_hash, "utf-8")} - ) - if "convergence" in graphs_to_return: - plot_convergence(model) - add_plot(plots, f"convergence_{idx}") - - if "objective" in graphs_to_return: - plot_objective( - model, - dimensions=dimensions, - usepartialdependence=False, - show_confidence=True, - pars=objective_pars, + if "single" in parsed.graphs_to_return and optimizer.n_objectives != 2: + emit_json_single_plots( + plots, + result=result[idx], + prefix=f"single_{idx}", + selected_point=parsed.selected_point, ) - add_plot(plots, f"objective_{idx}") + # convergence and objective plots are PNG-only; nothing to emit here. if optimizer.n_objectives == 1: - minimum = expected_minimum(result[0], return_std=True) - result_details["expected_minimum"] = [ - round_to_length_scales(minimum[0], optimizer.space), - minimum[1], - ] - elif "pareto" in graphs_to_return: - plot_Pareto(optimizer) - add_plot(plots, "pareto") - - if pickle_model: - result_details["pickled"] = pickleToString(result, get_crypto()) + _set_expected_minimum(result_details, result[0], optimizer.space) + + if optimizer.n_objectives == 2 and "pareto" in parsed.graphs_to_return: + emit_pareto_data(plots, optimizer) + + if optimizer.n_objectives == 2 and "single" in parsed.graphs_to_return: + emit_json_single_plots( + plots, + result=result[0], + prefix="objective_1", + selected_point=parsed.selected_point, + ) + emit_json_single_plots( + plots, + result=result[1], + prefix="objective_2", + selected_point=parsed.selected_point, + ) + + if parsed.include_model: + result_details["pickled"] = pack( + result=result, + next_points=result_details["next"], + optimizer=optimizer, + fingerprint=request_fingerprint, + crypto=get_crypto(), + ) add_version_info(result_details["extras"]) + result_details["extras"]["pickledUsed"] = pickled_used - # print(str(response)) - org_models = response["result"]["models"] - for model in org_models: - # Flatten expected minimum entries - model["expected_minimum"] = [ - [ - item - for sublist in [ - x if isinstance(x, list) else [x] for x in model["expected_minimum"] - ] - for item in sublist - ] - ] + _flatten_expected_minima(response["result"]["models"]) return response @@ -251,7 +392,7 @@ def process_model(model, optimizer): dict a dictionary containing the model specific results. """ - result_details = {"expected_minimum": [], "extras": {}} + result_details: dict[str, Any] = {"expected_minimum": [], "extras": {}} minimum = expected_minimum(model) result_details["expected_minimum"] = [ round_to_length_scales(minimum[0], optimizer.space), @@ -260,43 +401,7 @@ def process_model(model, optimizer): return result_details -def add_plot(result, id="generic", close=True, debug=False): - """Add the current figure to result as a base64 encoded string. - - This function should be called after every plot that is generated. - It takes the current state of the figure canvas and writes it to - a base64 encoded string which is then appended to the list supplied. - - Parameters - ---------- - result : list - The list of plots to which new plots should be addeed. - id : str - Identifier for the plot (default is "generic") - close : bool - If set to True the current matplot figure is cleared after the plot - has been saved. (default is True) - debug : bool - Indicate if plots should be written to local files. - If set to True plots are stored in tmp/process_optimizer_[id].png - relative to current working directory. (default is False) - """ - pic_io_bytes = io.BytesIO() - plt.savefig(pic_io_bytes, format="png", bbox_inches="tight") - pic_io_bytes.seek(0) - pic_hash = base64.b64encode(pic_io_bytes.read()) - result.append({"id": id, "plot": str(pic_hash, "utf-8")}) - - if debug: - with open("tmp/process_optimizer_" + id + ".png", "wb") as imgfile: - plt.savefig(imgfile, bbox_inches="tight", pad_inches=0) - - # print("IMAGE: " + str(pic_hash, "utf-8")) - if close: - plt.clf() - - -def round_to_length_scales(x, space): +def round_to_length_scales(x: Any, space: Any) -> Any: """Rounds a suggested experiment to to the length scales of each dimension For each dimension the length of the dimension is calculated and the @@ -317,7 +422,7 @@ def round_to_length_scales(x, space): The space of the optimizer. Contains information about each dimension of the space """ - for dim, i in zip(space.dimensions, range(len(space.dimensions))): + for i, dim in enumerate(space.dimensions): # Checking if dimension is real. Else do nothing if isinstance(dim, Real): length = dim.high - dim.low @@ -346,9 +451,12 @@ def add_version_info(extras): The dictionary to hold the version information """ - with open("requirements-freeze.txt", "r", encoding="utf-8") as requirements_file: - requirements = requirements_file.readlines() - extras["libraries"] = [x.rstrip() for x in requirements] + extras["libraries"] = sorted( + [ + f"{dist.metadata['Name']}=={dist.version}" + for dist in importlib.metadata.distributions() + ] + ) extras["pythonVersion"] = platform.python_version() diff --git a/optimizerapi/plot_emitters.py b/optimizerapi/plot_emitters.py new file mode 100644 index 0000000..fb15408 --- /dev/null +++ b/optimizerapi/plot_emitters.py @@ -0,0 +1,163 @@ +"""Plot emission for the optimizer API response. + +Two output formats are supported: +- ``emit_png_plots`` — base64 PNG plots, used by the legacy UI. +- ``emit_json_single_plots`` / ``emit_pareto_data`` — structured JSON + plot data, used by the React-based UI. +""" + +import base64 +import io +from typing import TYPE_CHECKING, Any + +import json_tricks +import matplotlib.pyplot as plt +import numpy +from ProcessOptimizer.plots import ( + get_Brownie_Bee_1d_plot, + get_Brownie_Bee_Pareto, + plot_brownie_bee_frontend, + plot_convergence, + plot_objective, +) +from ProcessOptimizer.utils.utils import get_Pareto_front_compromise + +if TYPE_CHECKING: + from .types import Plot + + +def _get_brownie_bee_1d_plot_safe(result, x_eval=None, **kwargs): + """Workaround for ProcessOptimizer bug: model.predict must receive space.transform output, + not a raw list containing categorical strings.""" + if x_eval is None: + return get_Brownie_Bee_1d_plot(result, x_eval=x_eval, **kwargs) + + space = result.space + has_categoricals = any( + not isinstance(v, (int, float)) for v in x_eval + ) + if not has_categoricals: + return get_Brownie_Bee_1d_plot(result, x_eval=x_eval, **kwargs) + + model = result.models[-1] + x_transformed = space.transform([x_eval]) + original_predict = model.predict + + def _predict_with_transform(X, **predict_kwargs): + arr = numpy.array(X) + if arr.dtype.kind in ("U", "S", "O"): + return original_predict(x_transformed, **predict_kwargs) + return original_predict(X, **predict_kwargs) + + model.predict = _predict_with_transform + try: + return get_Brownie_Bee_1d_plot(result, x_eval=x_eval, **kwargs) + finally: + model.predict = original_predict + + +def emit_json_single_plots( + plots: "list[Plot]", + *, + result: Any, + prefix: str, + selected_point: "list[str | float] | None", +) -> None: + """Append one plot entry per dimension plus a final histogram entry. + + Parameters + ---------- + plots: + The mutable list of plot entries to append to. + result: + A ProcessOptimizer ``OptimizerResult``-like object. + prefix: + Plot-id prefix. The emitted ids are ``{prefix}_0``, ``{prefix}_1``, + ..., with the final id being ``{prefix}_`` for the histogram. + selected_point: + X-space coordinates to highlight, or ``None`` to use the default. + """ + one_d_data = _get_brownie_bee_1d_plot_safe(result, x_eval=selected_point) + histogram_entry = one_d_data[-1] + for i, dim_data in enumerate(one_d_data[:-1]): + plots.append( + {"id": f"{prefix}_{i}", "plot": json_tricks.dumps({"data": dim_data})} + ) + plots.append( + { + "id": f"{prefix}_{len(one_d_data) - 1}", + "plot": json_tricks.dumps( + { + "histogram": { + "mean": float(numpy.ravel(histogram_entry[0])[0]), + "std": float(numpy.ravel(histogram_entry[1])[0]), + } + } + ), + } + ) + + +def emit_png_plots( + plots: "list[Plot]", + *, + result: list, + dimensions: list[str], + graphs: list[str], + max_quality: int, + objective_pars: str, +) -> None: + """Append base64-encoded PNG plots for each model in ``result``.""" + for idx, model in enumerate(result): + if "single" in graphs: + bb_plots = plot_brownie_bee_frontend(model, max_quality=max_quality) + for i, plot in enumerate(bb_plots): + plots.append( + {"id": f"single_{idx}_{i}", "plot": _figure_to_b64(plot)} + ) + plt.close(plot) + if "convergence" in graphs: + plot_convergence(model) + _emit_current_figure(plots, f"convergence_{idx}") + + if "objective" in graphs: + plot_objective( + model, + dimensions=dimensions, + usepartialdependence=False, + show_confidence=True, + pars=objective_pars, + ) + _emit_current_figure(plots, f"single_{idx}") + + +def emit_pareto_data(plots: "list[Plot]", optimizer: object) -> None: + """Append the pareto-front payload (multi-objective only).""" + front_x_data, front_y_data, obj1_error, obj2_error = get_Brownie_Bee_Pareto( + optimizer, n_points=200 + ) + best_idx = get_Pareto_front_compromise(front_y_data) + pareto_data = { + "front_x_data": front_x_data.tolist(), + "front_y_data": front_y_data.tolist(), + "obj1_error": obj1_error.tolist(), + "obj2_error": obj2_error.tolist(), + "best_idx": best_idx, + } + plots.append({"id": "pareto_data", "plot": json_tricks.dumps(pareto_data)}) + + +def _figure_to_b64(figure) -> str: + buf = io.BytesIO() + figure.savefig(buf, format="png") + buf.seek(0) + return base64.b64encode(buf.read()).decode("utf-8") + + +def _emit_current_figure(plots: "list[Plot]", plot_id: str) -> None: + """Snapshot the current matplotlib figure into ``plots`` and clear it.""" + buf = io.BytesIO() + plt.savefig(buf, format="png", bbox_inches="tight") + buf.seek(0) + plots.append({"id": plot_id, "plot": base64.b64encode(buf.read()).decode("utf-8")}) + plt.clf() From 6435a22a5d1536f85cfcaaae5318af31bf43c6f3 Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Sat, 30 May 2026 07:57:20 +0000 Subject: [PATCH 03/14] feat(api): selectedPoint override and pickled model cache Add the pickled_state module with request fingerprinting and pack/unpack-if-valid round-trip, wire the handler to consume a pickled model cache (surfacing pickledUsed) and a selectedPoint override with fall-through warnings, and document both in the OpenAPI extras schema. --- optimizerapi/openapi/specification.yml | 33 +++++++- optimizerapi/optimizer_handler.py | 108 ++++++++++++++----------- optimizerapi/pickled_state.py | 83 +++++++++++++++++++ optimizerapi/securepickle/__init__.py | 7 +- optimizerapi/securepickle/secure.py | 39 +++++++-- 5 files changed, 210 insertions(+), 60 deletions(-) create mode 100644 optimizerapi/pickled_state.py diff --git a/optimizerapi/openapi/specification.yml b/optimizerapi/openapi/specification.yml index 6648378..db0a1dc 100644 --- a/optimizerapi/openapi/specification.yml +++ b/optimizerapi/openapi/specification.yml @@ -13,7 +13,8 @@ paths: - oauth2: [] - apikey: [] description: Run optimizer with the specified parameters - operationId: optimizerapi.optimizer_handler.run + operationId: run_optimizer + x-openapi-router-controller: optimizerapi.optimizer_handler responses: "200": description: Result of running the optimizer with the specified parameters @@ -125,7 +126,8 @@ paths: /health: get: description: Health check endpoint - operationId: optimizerapi.health.check + operationId: health_check + x-openapi-router-controller: optimizerapi.health responses: "200": description: Service is reachable @@ -169,6 +171,22 @@ components: default: ["objective", "convergence", "pareto"] items: type: string + selectedPoint: + description: > + Override the highlight point in single plots with explicit + X-space coordinates. Honored only when graphFormat is "json"; + ignored on the PNG path. + type: array + items: + anyOf: + - type: string + - type: number + pickled: + description: > + Opaque cache hint produced by a previous response. The server + validates it matches the current data / optimizerConfig; on + mismatch it is ignored and a full run is performed. + type: string additionalProperties: type: string data: @@ -203,6 +221,13 @@ components: $ref: "#/components/schemas/space" constraints: $ref: "#/components/schemas/constraints" + required: + - baseEstimator + - acqFunc + - initialPoints + - kappa + - xi + - space required: - data - optimizerConfig @@ -303,6 +328,10 @@ components: type: object extras: type: object + properties: + pickledUsed: + description: True iff the server reused the pickled cache hint for this response. + type: boolean error: title: Errors returned by the API type: object diff --git a/optimizerapi/optimizer_handler.py b/optimizerapi/optimizer_handler.py index ee496e8..867f046 100644 --- a/optimizerapi/optimizer_handler.py +++ b/optimizerapi/optimizer_handler.py @@ -5,11 +5,13 @@ in the specification.yml file found in the folder "openapi" in the root of this project. """ +import hashlib +import json +import logging import os import time -import json -import traceback -import hashlib +from typing import TYPE_CHECKING + from rq import Queue from rq.job import Job from rq.exceptions import NoSuchJobError @@ -18,11 +20,40 @@ import connexion from .optimizer import run as handle_run -if "REDIS_URL" in os.environ: - REDIS_URL = os.environ["REDIS_URL"] -else: - REDIS_URL = "redis://localhost:6379" -print("Connecting to" + REDIS_URL) +if TYPE_CHECKING: + from .types import RequestBody, ResponseEnvelope + + +def _parse_env_bool(name: str, default: bool = False) -> bool: + """Parse a boolean-ish env var. + + Accepts "true", "1", "yes" (case-insensitive) as True. + Anything else — including unset or "false" — is False. + """ + raw = os.environ.get(name, "").strip().lower() + if not raw: + return default + return raw in ("true", "1", "yes") + + +def _resolve_disconnect_check(): + """Return a callable that reports whether the client has disconnected. + + Outside of a request context (e.g. unit tests) or when running under + a server that doesn't expose ``waitress.client_disconnected``, this + returns a no-op that always says "still connected". + """ + try: + env = connexion.request.environ + except RuntimeError: + return lambda: False + return env.get("waitress.client_disconnected", lambda: False) + + +_LOG = logging.getLogger(__name__) + +REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379") +_LOG.info("Connecting to %s", REDIS_URL) redis = Redis.from_url(REDIS_URL) if "REDIS_TTL" in os.environ: TTL = int(os.environ["REDIS_TTL"]) @@ -36,37 +67,19 @@ queue = Queue(connection=redis) -def run(body) -> dict: - """Executes the ProcessOptimizer - - Returns - ------- - dict - a JSON encodable dictionary representation of the result. - """ - try: - if "waitress.client_disconnected" in connexion.request.environ: - disconnect_check = connexion.request.environ["waitress.client_disconnected"] - else: - - def disconnect_check(): - return False - - except RuntimeError: - - def disconnect_check(): - return False +def run_optimizer(body: "RequestBody") -> "ResponseEnvelope | tuple[dict[str, str], int]": + """Handle the optimizer run request (POST /optimizer).""" + disconnect_check = _resolve_disconnect_check() - if "USE_WORKER" in os.environ and os.environ["USE_WORKER"]: + if _parse_env_bool("USE_WORKER"): body_hash = hashlib.new("sha256") body_hash.update(json.dumps(body).encode()) job_id = body_hash.hexdigest() try: job = Job.fetch(job_id, connection=redis) - - print("Found existing job") + _LOG.info("Found existing job %s", job_id) except NoSuchJobError: - print(f"Creating new job (WORKER_TIMEOUT={WORKER_TIMEOUT})") + _LOG.info("Creating new job (WORKER_TIMEOUT=%s)", WORKER_TIMEOUT) job = queue.enqueue( do_run_work, body, @@ -77,29 +90,30 @@ def disconnect_check(): while job.return_value() is None: if disconnect_check(): try: - print(f"Client disconnected, cancelling job {job.id}") + _LOG.warning("Client disconnected, cancelling job %s", job.id) job.cancel() send_stop_job_command(redis, job.id) job.delete() except Exception: pass - return {} + return {} # type: ignore[return-value] # empty sentinel on client disconnect time.sleep(0.2) - return job.return_value() + return job.return_value() # type: ignore[return-value] # RQ returns Any; narrowed in Phase 4 return do_run_work(body) -def do_run_work(body) -> dict: - """ "Handle the run request""" +def do_run_work(body: "RequestBody") -> "ResponseEnvelope": + """Handle the run request. + + On error we return a Connexion ``problem`` response, which serialises + to the OpenAPI-declared 400 / 500 response shape. + """ try: - return handle_run(body) - except IOError as err: - return ({"message": "I/O error", "error": str(err)}, 400) - except TypeError as err: - return ({"message": "Type error", "error": str(err)}, 400) - except ValueError as err: - return ({"message": "Validation error", "error": str(err)}, 400) + # Phase 4 narrows optimizer.run return type to ResponseEnvelope + return handle_run(body) # type: ignore[return-value] + except (IOError, TypeError, ValueError) as err: + _LOG.warning("client error: %s", err) + return connexion.problem(400, "Bad request", str(err)) # type: ignore[no-any-return] except Exception as err: - # Log unknown exceptions to support debugging - traceback.print_exc() - return ({"message": "Unknown error", "error": str(err)}, 500) + _LOG.exception("unexpected error during optimizer run") + return connexion.problem(500, "Internal server error", str(err)) # type: ignore[no-any-return] diff --git a/optimizerapi/pickled_state.py b/optimizerapi/pickled_state.py new file mode 100644 index 0000000..6b681f8 --- /dev/null +++ b/optimizerapi/pickled_state.py @@ -0,0 +1,83 @@ +"""Pickled-state cache: fingerprint a request, pack/unpack pickled payloads. + +The fingerprint binds a pickled blob to the request that produced it, so a +client that sends a stale pickled along with newer data gets a clean fall- +through to a full run instead of silently buggy reuse. +""" + +import hashlib +import json +import logging +from typing import TYPE_CHECKING, cast + +from .securepickle import pickleToString, unpickleFromString + +if TYPE_CHECKING: + from cryptography.fernet import Fernet + + from .types import CachePayload, DataPoint, OptimizerConfig + +_LOG = logging.getLogger(__name__) + +_REQUIRED_KEYS = ("fingerprint", "result", "next", "optimizer") + + +def compute_fingerprint( + data: list["DataPoint"], + optimizer_config: "OptimizerConfig", +) -> str: + """Return sha256 hex of the canonical-JSON of (data, optimizerConfig). + + Canonical JSON: sorted keys, no whitespace. Numbers are emitted as Python's + default JSON representation, which is stable for the integer / float values + that flow through the API. + """ + payload = {"data": data, "optimizerConfig": optimizer_config} + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def pack( + *, + result: object, + next_points: list[list[str | float]], + optimizer: object, + fingerprint: str, + crypto: "Fernet", +) -> str: + """Encrypt a pickled cache payload for the given fingerprint.""" + payload = { + "fingerprint": fingerprint, + "result": result, + "next": next_points, + "optimizer": optimizer, + } + return cast(str, pickleToString(payload, crypto)) + + +def unpack_if_valid( + blob: str, + *, + expected_fingerprint: str, + crypto: "Fernet", +) -> "CachePayload | None": + """Decrypt and validate a pickled cache payload. + + Returns the payload dict on success, or None if anything is off. Any + failure is logged once at WARNING with a reason tag in the message: + decrypt_failed | bad_structure | fingerprint_mismatch. + """ + if not blob: + return None + try: + payload = unpickleFromString(blob, crypto) + except Exception: + _LOG.warning("pickled cache ignored: decrypt_failed") + return None + if not isinstance(payload, dict) or not all(k in payload for k in _REQUIRED_KEYS): + _LOG.warning("pickled cache ignored: bad_structure") + return None + if payload["fingerprint"] != expected_fingerprint: + _LOG.warning("pickled cache ignored: fingerprint_mismatch") + return None + return payload # type: ignore[return-value] diff --git a/optimizerapi/securepickle/__init__.py b/optimizerapi/securepickle/__init__.py index b21e39f..09815ff 100644 --- a/optimizerapi/securepickle/__init__.py +++ b/optimizerapi/securepickle/__init__.py @@ -1,5 +1,6 @@ -""" -Module that provides support for encrypted pickle functionality -""" +"""Secure pickle module — Fernet-encrypted pickle round-trip.""" + from .pickler import pickleToString, unpickleFromString from .secure import get_crypto + +__all__ = ["get_crypto", "pickleToString", "unpickleFromString"] diff --git a/optimizerapi/securepickle/secure.py b/optimizerapi/securepickle/secure.py index 7fb1685..78f38d8 100644 --- a/optimizerapi/securepickle/secure.py +++ b/optimizerapi/securepickle/secure.py @@ -1,14 +1,37 @@ -from cryptography.fernet import Fernet +"""Fernet crypto handle factory. + +Resolution order for the key: + 1. ``key`` argument (highest priority). + 2. ``PICKLE_KEY`` environment variable. + 3. Generate a new ephemeral key and log a WARNING. + +The ephemeral path is intended for local development only. In any +environment where pickled state must survive a restart, ``PICKLE_KEY`` +must be set explicitly. This function no longer mutates ``os.environ``, +so caller environments are unchanged. +""" + +import logging import os +from cryptography.fernet import Fernet + +_LOG = logging.getLogger(__name__) + def get_crypto(key=None): - if key == None: - key = os.getenv("PICKLE_KEY", None) - if key == None: - print("No key found, generating new key") + """Return a Fernet crypto handle. + + See the module docstring for the key-resolution order. + """ + if key is None: + key = os.getenv("PICKLE_KEY") + if key is None: key = Fernet.generate_key() - os.environ["PICKLE_KEY"] = key.decode("utf-8") - print("To reuse key for future server runs, set environment variable PICKLE_KEY=" + - os.environ["PICKLE_KEY"]) + _LOG.warning( + "No PICKLE_KEY set; generated an ephemeral key. Set " + "PICKLE_KEY=%s in the environment to persist pickled state " + "across restarts.", + key.decode("utf-8"), + ) return Fernet(key) From e99b084a7dad84365dc2fc0d43056066e15cf19b Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Sat, 30 May 2026 07:57:26 +0000 Subject: [PATCH 04/14] fix(auth): harden API-key authentication Use secrets.compare_digest for API-key comparison, stop logging raw access tokens to stdout, and refuse the default AUTH_API_KEY='none' in production. --- optimizerapi/auth.py | 57 ++++++++++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/optimizerapi/auth.py b/optimizerapi/auth.py index fa41e02..61af6cc 100644 --- a/optimizerapi/auth.py +++ b/optimizerapi/auth.py @@ -1,12 +1,20 @@ -"""Authenitcation module +"""Authentication module. -This module will verify tokens provided bt a Keycloak OpenID server +Verifies bearer tokens issued by a Keycloak OpenID server, and the +static API key provided via ``AUTH_API_KEY``. """ +import logging import os +import secrets +from typing import Any, cast + from keycloak import KeycloakOpenID AUTH_API_KEY = os.getenv("AUTH_API_KEY", "none") AUTH_SERVER = os.getenv("AUTH_SERVER", None) +FLASK_ENV = os.getenv("FLASK_ENV", "development") + +_DEFAULT_APIKEY = "none" # sentinel: matches AUTH_API_KEY default; never accept in production AUTH_CLIENT_ID = os.getenv("AUTH_CLIENT_ID", None) AUTH_CLIENT_SECRET = os.getenv("AUTH_CLIENT_SECRET", None) AUTH_REALM_NAME = os.getenv("AUTH_REALM_NAME", None) @@ -18,38 +26,51 @@ client_secret_key=AUTH_CLIENT_SECRET, ) +_LOG = logging.getLogger(__name__) + -def token_info(access_token) -> dict: - """Verify token with authentication server +def token_info(access_token: str) -> dict | None: + """Verify a bearer token against the configured Keycloak server. Returns ------- dict - a dictionary containing sub and scope - None in case of invalid token + Token data from Keycloak introspection when the token is active. + If no OIDC server is configured, returns ``{"scope": []}``. + None + If the server reports the token as inactive. """ - print(access_token) if not AUTH_SERVER: return {"scope": []} - token = access_token - token_data = keycloak_openid.introspect(token) - if "active" in token_data and token_data["active"]: - print("OK") + # keycloak_openid.introspect returns Any; narrow to dict so callers stay typed + token_data: dict[Any, Any] = cast(dict[Any, Any], keycloak_openid.introspect(access_token)) + if token_data.get("active"): + _LOG.debug("token accepted") return token_data - print("NOT OK") - print(token_data) + _LOG.warning("token rejected by Keycloak") return None -def apikey_handler(access_token) -> dict: - """Verify API key based on environment variable +def apikey_handler(access_token: str) -> dict | None: + """Verify the API key passed by the client. Returns ------- dict - a dictionary containing sub and scope - None in case of invalid token + ``{"scope": []}`` if the supplied key matches the configured + ``AUTH_API_KEY``. + None + If the key is wrong, OIDC is configured (OIDC handles this path), + or the operator has not configured a real key in production. """ - if not AUTH_SERVER and AUTH_API_KEY == access_token: + if AUTH_SERVER: + return None + # In production we refuse the unconfigured default value, even if it + # technically matches the request — operators should set a real key. + if FLASK_ENV == "production" and AUTH_API_KEY == _DEFAULT_APIKEY: + return None + expected = AUTH_API_KEY.encode("utf-8") + provided = (access_token or "").encode("utf-8") + if secrets.compare_digest(expected, provided): return {"scope": []} return None From 0a94409571ca4c3f38c92e3a96f70ae5e77a8f98 Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Sat, 30 May 2026 07:57:26 +0000 Subject: [PATCH 05/14] refactor: structured logging and typed boundaries Replace print() with central logging configuration, and add a types module with TypedDicts for the request, response, and cache payloads behind a permissive mypy baseline. --- optimizerapi/server.py | 29 ++++++++-- optimizerapi/types.py | 120 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+), 4 deletions(-) create mode 100644 optimizerapi/types.py diff --git a/optimizerapi/server.py b/optimizerapi/server.py index e425aa9..1f2026d 100644 --- a/optimizerapi/server.py +++ b/optimizerapi/server.py @@ -1,14 +1,35 @@ """ Main server """ +import logging import os import re + import connexion -from waitress import serve from flask_cors import CORS +from waitress import serve + from .securepickle import get_crypto +_LOG = logging.getLogger(__name__) + + +def _configure_logging() -> None: + """Configure root logging once, before any handler is dispatched. + + Level defaults to INFO. Override with the LOG_LEVEL environment + variable (e.g. LOG_LEVEL=DEBUG for local debugging). + """ + level_name = os.getenv("LOG_LEVEL", "INFO").upper() + level = getattr(logging, level_name, logging.INFO) + logging.basicConfig( + level=level, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + + if __name__ == "__main__": + _configure_logging() # Initialize crypto get_crypto() app = connexion.FlaskApp( @@ -46,11 +67,11 @@ # what we want to support. origins=re.compile(cors_origin), ) - print("CORS: " + cors_origin) + _LOG.info("CORS: %s", cors_origin) except re.error: - print("CORS: failed - the regex might be malformed.") + _LOG.warning("CORS: failed - the regex might be malformed.") else: - print("CORS: disabled") + _LOG.info("CORS: disabled") if development: app.run(port=9090) diff --git a/optimizerapi/types.py b/optimizerapi/types.py new file mode 100644 index 0000000..f74ca51 --- /dev/null +++ b/optimizerapi/types.py @@ -0,0 +1,120 @@ +"""Boundary types for the optimizer API. + +These ``TypedDict``s describe the shape of the request body, response +envelope, and internal cache payload. They are *static* contracts — +they exist to be checked by ``mypy`` and to document the API to +readers. Runtime validation of incoming requests lives in Connexion + +OpenAPI; we do not duplicate that here. + +Note on the ``Dimension`` field name ``from``: + ``from`` is a Python keyword, so the class-based ``TypedDict`` + syntax cannot declare it. We use the alternative call-based + syntax to define ``Dimension``. +""" + +from typing import Any, Literal, NotRequired, TypedDict + + +# --- Request types --------------------------------------------------------- + +Dimension = TypedDict( + "Dimension", + { + "type": Literal["continuous", "discrete", "category"], + "name": str, + "from": NotRequired[float], + "to": NotRequired[float], + "categories": NotRequired[list[str]], + }, +) + + +class Constraint(TypedDict): + type: Literal["sum"] + dimensions: list[int] + value: float + + +class OptimizerConfig(TypedDict): + baseEstimator: str + acqFunc: str + initialPoints: int + kappa: float + xi: float + space: list[Dimension] + constraints: NotRequired[list[Constraint]] + + +class DataPoint(TypedDict): + xi: list[str | float] + yi: list[float] + + +GraphFormat = Literal["png", "json", "none"] +GraphName = Literal["objective", "convergence", "pareto", "single"] +ObjectivePars = Literal["result", "expected_minimum"] + + +class Extras(TypedDict, total=False): + objectivePars: ObjectivePars + graphFormat: GraphFormat + experimentSuggestionCount: int + maxQuality: int + graphs: list[GraphName] + selectedPoint: list[str | float] + pickled: str + # NB: stringly-typed; see audit §3 for the rationale for not migrating yet. + includeModel: Literal["true", "false"] + useActualMeasurementHistogram: Literal["true", "false"] + + +class RequestBody(TypedDict): + data: list[DataPoint] + optimizerConfig: OptimizerConfig + extras: NotRequired[Extras] + + +# --- Response types -------------------------------------------------------- + +class Plot(TypedDict): + id: str + plot: str # base64 PNG OR JSON-serialised plot data + + +class ResponseResultExtras(TypedDict, total=False): + pickledUsed: bool + parameters: dict[str, Any] + libraries: list[str] + pythonVersion: str + apiVersion: str + timeOfExecution: str + + +class ResponseModel(TypedDict, total=False): + expected_minimum: list[list[str | float]] + extras: dict[str, Any] + + +class ResponseResult(TypedDict, total=False): + next: list[list[str | float]] + models: list[ResponseModel] + pickled: str + extras: ResponseResultExtras + expected_minimum: list[list[str | float] | float] + + +class ResponseEnvelope(TypedDict): + plots: list[Plot] + result: ResponseResult + + +# --- Cache payload (pickled_state) ----------------------------------------- + +class CachePayload(TypedDict): + fingerprint: str + # ProcessOptimizer's OptimizerResult or list thereof — kept opaque + # so types.py does not depend on the optimizer library. + result: Any + next: list[list[str | float]] + # ProcessOptimizer.Optimizer — also opaque. + optimizer: Any From 900294848f4b9e182ed1f598071294b9ebc6cc8b Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Sat, 30 May 2026 07:57:31 +0000 Subject: [PATCH 06/14] test: pareto e2e, unit suites, shared fixtures, and sample curls Add an end-to-end pareto HTTP suite plus unit coverage for auth, pickled_state, plot_emitters, and types; introduce a conftest with a stable PICKLE_KEY; enforce no print() under optimizerapi/; expand the optimizer tests; and add sample curl scripts. Drops the dead tests/context.py module. --- scripts/multi-blank-first-run.curl | 11 + scripts/multi-blank-second-run.curl | 11 + scripts/sample-multi-with-selection.curl | 35 ++ scripts/sample-multi.curl | 5 + scripts/sample.curl | 6 + tests/conftest.py | 67 +++ tests/context.py | 9 - tests/test_auth.py | 55 ++ tests/test_e2e_pareto.py | 515 +++++++++++++++++++ tests/test_no_prints.py | 35 ++ tests/test_optimizer.py | 615 ++++++++++++++++++++++- tests/test_pickled_state.py | 114 +++++ tests/test_plot_emitters.py | 67 +++ tests/test_types.py | 65 +++ 14 files changed, 1574 insertions(+), 36 deletions(-) create mode 100644 scripts/multi-blank-first-run.curl create mode 100644 scripts/multi-blank-second-run.curl create mode 100644 scripts/sample-multi-with-selection.curl create mode 100644 scripts/sample-multi.curl create mode 100644 scripts/sample.curl create mode 100644 tests/conftest.py delete mode 100644 tests/context.py create mode 100644 tests/test_auth.py create mode 100644 tests/test_e2e_pareto.py create mode 100644 tests/test_no_prints.py create mode 100644 tests/test_pickled_state.py create mode 100644 tests/test_plot_emitters.py create mode 100644 tests/test_types.py diff --git a/scripts/multi-blank-first-run.curl b/scripts/multi-blank-first-run.curl new file mode 100644 index 0000000..f7b15d7 --- /dev/null +++ b/scripts/multi-blank-first-run.curl @@ -0,0 +1,11 @@ +curl -S 'http://localhost:9090/v1.0/optimizer?apikey=none' \ +-X 'POST' \ +-H 'Content-Type: application/json' \ +-H 'Origin: http://localhost:5173' \ +-H 'Referer: http://localhost:5173/' \ +-H 'Accept: */*' \ +-H 'Sec-Fetch-Dest: empty' \ +-H 'Sec-Fetch-Mode: cors' \ +-H 'Sec-Fetch-Site: same-site' \ +-H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.4 Safari/605.1.15' \ +--data-raw '{"extras":{"experimentSuggestionCount":5},"data":[],"optimizerConfig":{"baseEstimator":"GP","acqFunc":"EI","initialPoints":5,"kappa":1.96,"xi":0.01,"space":[{"type":"discrete","name":"water","from":0,"to":100},{"type":"discrete","name":"temperature","from":-50,"to":120},{"type":"continuous","name":"angle","from":0,"to":360},{"type":"category","name":"color","categories":["red","green","blue"]}],"constraints":[]}}' diff --git a/scripts/multi-blank-second-run.curl b/scripts/multi-blank-second-run.curl new file mode 100644 index 0000000..6cce921 --- /dev/null +++ b/scripts/multi-blank-second-run.curl @@ -0,0 +1,11 @@ +curl -S 'http://localhost:9090/v1.0/optimizer?apikey=none' \ +-X 'POST' \ +-H 'Content-Type: application/json' \ +-H 'Origin: http://localhost:5173' \ +-H 'Referer: http://localhost:5173/' \ +-H 'Accept: */*' \ +-H 'Sec-Fetch-Dest: empty' \ +-H 'Sec-Fetch-Mode: cors' \ +-H 'Sec-Fetch-Site: same-site' \ +-H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/26.4 Safari/605.1.15' \ +--data-raw '{"extras":{"objectivePars":"expected_minimum","experimentSuggestionCount":5},"data":[{"xi":[10,69,36,"red"],"yi":[-2.5,-12]},{"xi":[70,103,324,"blue"],"yi":[-1.6,-50]},{"xi":[90,1,108,"green"],"yi":[-2.8,-5]},{"xi":[50,35,180,"red"],"yi":[-3.8,-10]},{"xi":[30,-33,252,"blue"],"yi":[-3.6,-40]}],"optimizerConfig":{"baseEstimator":"GP","acqFunc":"EI","initialPoints":5,"kappa":1.96,"xi":1.2,"space":[{"type":"discrete","name":"water","from":0,"to":100},{"type":"discrete","name":"temperature","from":-50,"to":120},{"type":"continuous","name":"angle","from":0,"to":360},{"type":"category","name":"color","categories":["red","green","blue"]}],"constraints":[]}}' diff --git a/scripts/sample-multi-with-selection.curl b/scripts/sample-multi-with-selection.curl new file mode 100644 index 0000000..139f3a3 --- /dev/null +++ b/scripts/sample-multi-with-selection.curl @@ -0,0 +1,35 @@ +# Sample curl script demonstrating selectedPoint + pickled flow +# This shows how to: +# 1. Run an initial optimization and capture the pickled state +# 2. Use selectedPoint with pickled to regenerate plots without retraining + +# ============================================================================ +# STEP 1: Run initial optimization to get pickled state +# ============================================================================ +# This request runs the optimizer and returns result.pickled in the response. +# Copy the value of result.pickled from the JSON response for use in Step 2. + +curl 'http://localhost:9090/v1.0/optimizer?apikey=none' \ +-X 'POST' \ +-H 'Content-Type: application/json' \ +-H 'Accept: */*' \ +--data-raw '{"extras":{"experimentSuggestionCount":1,"graphs":["pareto","single"],"graphFormat":"json"},"data":[{"xi":[16.7,500,250,20,"None"],"yi":[-2,-17]},{"xi":[50,833,150,60,"Whipped cream"],"yi":[-3,-6]},{"xi":[58.3,22,85,6,"Frosting"],"yi":[-6,-25]}],"optimizerConfig":{"baseEstimator":"GP","acqFunc":"EI","initialPoints":3,"kappa":1.96,"xi":2,"space":[{"type":"continuous","name":"Sugar","from":0,"to":100},{"type":"continuous","name":"Flour","from":0,"to":1000},{"type":"discrete","name":"Temperature","from":0,"to":300},{"type":"discrete","name":"Time","from":0,"to":120},{"type":"category","name":"Finish","categories":["None","Frosting","Whipped cream"]}],"constraints":[]}}' + +# ============================================================================ +# STEP 2: Use selectedPoint with pickled to regenerate plots +# ============================================================================ +# This request reuses the optimizer state (pickled) from Step 1 and selects +# a specific point [50, 833, 150, 60, "Whipped cream"] from the X-space +# to regenerate the plots without retraining the model. +# +# INSTRUCTIONS: +# 1. Run the command above and capture the JSON response +# 2. Find "pickled" in the response (it's a base64-encoded string) +# 3. Replace with that value +# 4. Run this command: + +curl 'http://localhost:9090/v1.0/optimizer?apikey=none' \ +-X 'POST' \ +-H 'Content-Type: application/json' \ +-H 'Accept: */*' \ +--data-raw '{"extras":{"experimentSuggestionCount":1,"graphs":["pareto","single"],"graphFormat":"json","selectedPoint":[50,833,150,60,"Whipped cream"],"pickled":""},"data":[{"xi":[16.7,500,250,20,"None"],"yi":[-2,-17]},{"xi":[50,833,150,60,"Whipped cream"],"yi":[-3,-6]},{"xi":[58.3,22,85,6,"Frosting"],"yi":[-6,-25]}],"optimizerConfig":{"baseEstimator":"GP","acqFunc":"EI","initialPoints":3,"kappa":1.96,"xi":2,"space":[{"type":"continuous","name":"Sugar","from":0,"to":100},{"type":"continuous","name":"Flour","from":0,"to":1000},{"type":"discrete","name":"Temperature","from":0,"to":300},{"type":"discrete","name":"Time","from":0,"to":120},{"type":"category","name":"Finish","categories":["None","Frosting","Whipped cream"]}],"constraints":[]}}' diff --git a/scripts/sample-multi.curl b/scripts/sample-multi.curl new file mode 100644 index 0000000..3c4bb4e --- /dev/null +++ b/scripts/sample-multi.curl @@ -0,0 +1,5 @@ +curl 'http://localhost:9090/v1.0/optimizer?apikey=none' \ +-X 'POST' \ +-H 'Content-Type: application/json' \ +-H 'Accept: */*' \ +--data-raw '{"extras":{"experimentSuggestionCount":1,"graphs":["pareto","single"],"graphFormat":"json"},"data":[{"xi":[16.7,500,250,20,"None"],"yi":[-2,-17]},{"xi":[50,833,150,60,"Whipped cream"],"yi":[-3,-6]},{"xi":[58.3,22,85,6,"Frosting"],"yi":[-6,-25]}],"optimizerConfig":{"baseEstimator":"GP","acqFunc":"EI","initialPoints":3,"kappa":1.96,"xi":2,"space":[{"type":"continuous","name":"Sugar","from":0,"to":100},{"type":"continuous","name":"Flour","from":0,"to":1000},{"type":"discrete","name":"Temperature","from":0,"to":300},{"type":"discrete","name":"Time","from":0,"to":120},{"type":"category","name":"Finish","categories":["None","Frosting","Whipped cream"]}],"constraints":[]}}' diff --git a/scripts/sample.curl b/scripts/sample.curl new file mode 100644 index 0000000..3519c4d --- /dev/null +++ b/scripts/sample.curl @@ -0,0 +1,6 @@ +curl 'http://127.0.0.1:9090/v1.0/optimizer?apikey=none' \ +-X 'POST' \ +-H 'Content-Type: application/json' \ +-H 'Accept: */*' \ +--data-raw '{"extras":{"experimentSuggestionCount":1,"graphs":["single", "pareto"],"graphFormat":"json","includeModel":"false","objectivePars":"expected_minimum"},"data":[{"xi":[130,170,210,105],"yi":[-2.1]},{"xi":[190,150,250,135],"yi":[-3.1]},{"xi":[150,130,230,95],"yi":[-1.8]},{"xi":[110,111,270,115],"yi":[-2.1]},{"xi":[176,200,300,122],"yi":[-5]},{"xi":[177,200,300,133],"yi":[-2.5]},{"xi":[146,158,260,112],"yi":[-4.5]}],"optimizerConfig":{"baseEstimator":"GP","acqFunc":"EI","initialPoints":5,"kappa":1.96,"xi":0.1,"space":[{"type":"discrete","name":"Pin elevation","from":100,"to":200},{"type":"discrete","name":"Bungee position","from":100,"to":200},{"type":"discrete","name":"Cup elevation","from":200,"to":300},{"type":"discrete","name":"Firing angle","from":90,"to":140}],"constraints":[]}}' + diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..7109e06 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,67 @@ +"""Shared pytest fixtures and session-level configuration.""" + +import os + +import pytest + +# A fixed Fernet key used by all tests so that encrypt/decrypt round-trips +# remain stable across multiple get_crypto() calls within a test session. +# Never use this value in production. +_TEST_PICKLE_KEY = "y3kUY6D3DAVy1NR8oK8Q7mmYkuvLtE8buQEO4IMSvOk=" + + +@pytest.fixture(autouse=True, scope="session") +def stable_pickle_key(): + """Set a fixed PICKLE_KEY for all tests. + + Without this, get_crypto() generates a new ephemeral Fernet key on every + call, so encrypt/decrypt round-trips across two optimizer.run_optimizer() calls (or + between run_optimizer() and a direct unpickleFromString() call in a test) will fail + with InvalidToken. A stable key makes the test environment mirror a real + deployment where PICKLE_KEY is set in the environment. + """ + original = os.environ.get("PICKLE_KEY") + os.environ["PICKLE_KEY"] = _TEST_PICKLE_KEY + yield _TEST_PICKLE_KEY + if original is None: + os.environ.pop("PICKLE_KEY", None) + else: + os.environ["PICKLE_KEY"] = original + + +@pytest.fixture(scope="session") +def app(): + """Create the Flask application for testing. + + This fixture creates a Connexion app with the same configuration + as the production server, but in test mode. + """ + from pathlib import Path + import connexion + from optimizerapi.securepickle import get_crypto + + # Initialize crypto with the stable key + get_crypto() + + # Use absolute path to the openapi spec + api_spec_dir = Path(__file__).parent.parent / "optimizerapi" / "openapi" + + app = connexion.FlaskApp( + __name__, + specification_dir=str(api_spec_dir), + ) + app.add_api("specification.yml", strict_validation=True, validate_responses=True) + app.app.config["TESTING"] = True + + return app + + +@pytest.fixture(scope="session") +def app_client(app): + """Create a test client for the Flask application. + + This client can be used to make HTTP requests to the API endpoints + as if they were being served by a real HTTP server. + """ + with app.app.test_client() as client: + yield client diff --git a/tests/context.py b/tests/context.py deleted file mode 100644 index 300a47e..0000000 --- a/tests/context.py +++ /dev/null @@ -1,9 +0,0 @@ -import os -import sys - -if True: - sys.path.insert( - 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../optimizerapi")) - ) - import optimizer - import securepickle diff --git a/tests/test_auth.py b/tests/test_auth.py new file mode 100644 index 0000000..d8fa672 --- /dev/null +++ b/tests/test_auth.py @@ -0,0 +1,55 @@ +"""Tests for the API-key handler.""" + +from unittest.mock import patch + + +def test_correct_apikey_returns_scope_dict(): + """A request with the configured API key returns a scope dict.""" + with patch("optimizerapi.auth.AUTH_API_KEY", "secret-key"), \ + patch("optimizerapi.auth.AUTH_SERVER", None): + from optimizerapi import auth + result = auth.apikey_handler("secret-key") + assert result == {"scope": []} + + +def test_wrong_apikey_returns_none(): + """An incorrect API key is rejected.""" + with patch("optimizerapi.auth.AUTH_API_KEY", "secret-key"), \ + patch("optimizerapi.auth.AUTH_SERVER", None): + from optimizerapi import auth + result = auth.apikey_handler("not-the-key") + assert result is None + + +def test_apikey_handler_uses_constant_time_comparison(): + """The handler must compare keys via secrets.compare_digest.""" + from optimizerapi import auth + import secrets + + with patch("optimizerapi.auth.AUTH_API_KEY", "secret-key"), \ + patch("optimizerapi.auth.AUTH_SERVER", None), \ + patch.object(secrets, "compare_digest", wraps=secrets.compare_digest) as mock_cmp: + auth.apikey_handler("secret-key") + assert mock_cmp.called, "apikey_handler must use secrets.compare_digest" + + +def test_default_apikey_value_is_rejected_in_production_mode(): + """When AUTH_API_KEY is the default 'none' and FLASK_ENV is 'production', + even matching the default value must be rejected.""" + with patch("optimizerapi.auth.AUTH_API_KEY", "none"), \ + patch("optimizerapi.auth.AUTH_SERVER", None), \ + patch("optimizerapi.auth.FLASK_ENV", "production"): + from optimizerapi import auth + result = auth.apikey_handler("none") + assert result is None + + +def test_default_apikey_value_is_accepted_in_development_mode(): + """In development mode (FLASK_ENV != 'production'), the legacy + behaviour of accepting AUTH_API_KEY='none' is preserved for ergonomics.""" + with patch("optimizerapi.auth.AUTH_API_KEY", "none"), \ + patch("optimizerapi.auth.AUTH_SERVER", None), \ + patch("optimizerapi.auth.FLASK_ENV", "development"): + from optimizerapi import auth + result = auth.apikey_handler("none") + assert result == {"scope": []} diff --git a/tests/test_e2e_pareto.py b/tests/test_e2e_pareto.py new file mode 100644 index 0000000..6ed673f --- /dev/null +++ b/tests/test_e2e_pareto.py @@ -0,0 +1,515 @@ +""" +End-to-end tests for the pareto front exploration feature. + +These tests hit the actual HTTP layer using Flask's test client, +verifying the full request/response cycle including authentication, +validation, and the pareto front exploration workflow. +""" + +import copy +import json +import pytest + +# Multi-objective test data from docs/api-usage.md +MULTI_OBJECTIVE_DATA = [ + {"xi": [16.7, 500, 250, 20, "None"], "yi": [-2, -17]}, + {"xi": [50, 833, 150, 60, "Whipped cream"], "yi": [-3, -6]}, + {"xi": [58.3, 22, 85, 6, "Frosting"], "yi": [-6, -25]}, +] + +MULTI_OBJECTIVE_CONFIG = { + "baseEstimator": "GP", + "acqFunc": "EI", + "initialPoints": 3, + "kappa": 1.96, + "xi": 2, + "space": [ + {"type": "continuous", "name": "Sugar", "from": 0, "to": 100}, + {"type": "continuous", "name": "Flour", "from": 0, "to": 1000}, + {"type": "discrete", "name": "Temperature", "from": 0, "to": 300}, + {"type": "discrete", "name": "Time", "from": 0, "to": 120}, + {"type": "category", "name": "Finish", "categories": ["None", "Frosting", "Whipped cream"]}, + ], + "constraints": [], +} + + +@pytest.mark.filterwarnings("ignore::DeprecationWarning") +class TestE2EParetoFront: + """End-to-end tests for pareto front exploration via HTTP.""" + + BASE_URL = "/v1.0/optimizer" + + @pytest.fixture + def api_key(self): + """Return the test API key.""" + return "none" + + def build_request(self, data, optimizer_config, extras=None): + """Build a request payload.""" + payload = { + "data": data, + "optimizerConfig": optimizer_config, + } + if extras: + payload["extras"] = extras + return payload + + def test_health_endpoint(self, app_client): + """Test that the health endpoint is reachable.""" + response = app_client.get("/v1.0/health") + assert response.status_code == 200 + + def test_initial_multi_objective_run_returns_pareto_data(self, app_client, api_key): + """Test that an initial multi-objective run includes pareto_data in plots.""" + payload = self.build_request( + data=MULTI_OBJECTIVE_DATA, + optimizer_config=MULTI_OBJECTIVE_CONFIG, + extras={ + "experimentSuggestionCount": 1, + "graphs": ["pareto", "single"], + "graphFormat": "json", + }, + ) + + response = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload, + content_type="application/json", + ) + + assert response.status_code == 200 + result = response.get_json() + + # Verify response structure + assert "plots" in result + assert "result" in result + assert "next" in result["result"] + assert "pickled" in result["result"] + + # Verify pareto_data plot is present + plot_ids = [p["id"] for p in result["plots"]] + assert "pareto_data" in plot_ids, "pareto_data plot should be present in multi-objective response" + + # Verify pareto_data has expected structure + pareto_plot = next(p for p in result["plots"] if p["id"] == "pareto_data") + pareto_data = json.loads(pareto_plot["plot"]) + assert "front_y_data" in pareto_data + assert "front_x_data" in pareto_data + assert "best_idx" in pareto_data + + def test_pareto_front_x_data_matches_input_dimensions(self, app_client, api_key): + """Test that front_x_data points have the same dimensionality as the space.""" + payload = self.build_request( + data=MULTI_OBJECTIVE_DATA, + optimizer_config=MULTI_OBJECTIVE_CONFIG, + extras={ + "graphs": ["pareto"], + "graphFormat": "json", + }, + ) + + response = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload, + content_type="application/json", + ) + + assert response.status_code == 200 + result = response.get_json() + + pareto_plot = next(p for p in result["plots"] if p["id"] == "pareto_data") + pareto_data = json.loads(pareto_plot["plot"]) + + space_dim = len(MULTI_OBJECTIVE_CONFIG["space"]) + for point in pareto_data["front_x_data"]: + assert len(point) == space_dim, ( + f"front_x_data point has {len(point)} dimensions, expected {space_dim}" + ) + + def test_selectedPoint_highlights_correct_point(self, app_client, api_key): + """Test that selectedPoint correctly highlights a specific point in plots.""" + # First, get the pareto front + payload = self.build_request( + data=MULTI_OBJECTIVE_DATA, + optimizer_config=MULTI_OBJECTIVE_CONFIG, + extras={ + "graphs": ["pareto", "single"], + "graphFormat": "json", + "includeModel": "true", + }, + ) + + response = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload, + content_type="application/json", + ) + assert response.status_code == 200 + run1 = response.get_json() + + # Extract a point from the pareto front + pareto_plot = next(p for p in run1["plots"] if p["id"] == "pareto_data") + pareto_data = json.loads(pareto_plot["plot"]) + selected_point = pareto_data["front_x_data"][0] + + # Now request with selectedPoint + payload_with_selection = self.build_request( + data=MULTI_OBJECTIVE_DATA, + optimizer_config=MULTI_OBJECTIVE_CONFIG, + extras={ + "graphs": ["single"], + "graphFormat": "json", + "selectedPoint": selected_point, + "includeModel": "false", + }, + ) + + response = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload_with_selection, + content_type="application/json", + ) + assert response.status_code == 200 + run2 = response.get_json() + + # Verify selectedPoint is reflected in objective_1_0 plot (dimension 0). + # In get_Brownie_Bee_1d_plot's output shape, index 3 is the x-coord of + # the highlight point for the dimension. + obj1_dim0 = next(p for p in run2["plots"] if p["id"] == "objective_1_0") + plot_data = json.loads(obj1_dim0["plot"]) + assert plot_data["data"][3] == selected_point[0], ( + "selectedPoint should be reflected in the single plot highlight" + ) + + def test_full_pareto_exploration_workflow(self, app_client, api_key): + """ + Test the complete pareto exploration workflow: + 1. Initial run to get pareto front and pickled state + 2. User selects a point from pareto front + 3. Re-request with selectedPoint + pickled for fast response + 4. Verify pickledUsed is True (fast path engaged) + """ + # Step 1: Initial run + payload = self.build_request( + data=MULTI_OBJECTIVE_DATA, + optimizer_config=MULTI_OBJECTIVE_CONFIG, + extras={ + "experimentSuggestionCount": 1, + "graphs": ["pareto", "single"], + "graphFormat": "json", + "includeModel": "true", + }, + ) + + response = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload, + content_type="application/json", + ) + assert response.status_code == 200 + run1 = response.get_json() + + # Verify initial run used full training (not cache) + assert run1["result"]["extras"]["pickledUsed"] is False, ( + "Initial run should not use pickled cache" + ) + + # Extract pickled and a point from pareto front + pickled_value = run1["result"]["pickled"] + assert len(pickled_value) > 0, "pickled should be present in response" + + pareto_plot = next(p for p in run1["plots"] if p["id"] == "pareto_data") + pareto_data = json.loads(pareto_plot["plot"]) + selected_point = pareto_data["front_x_data"][0] + + # Step 2: Re-request with selectedPoint + pickled + payload_with_cache = self.build_request( + data=MULTI_OBJECTIVE_DATA, + optimizer_config=MULTI_OBJECTIVE_CONFIG, + extras={ + "experimentSuggestionCount": 1, + "graphs": ["pareto", "single"], + "graphFormat": "json", + "selectedPoint": selected_point, + "pickled": pickled_value, + "includeModel": "true", + }, + ) + + response = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload_with_cache, + content_type="application/json", + ) + assert response.status_code == 200 + run2 = response.get_json() + + # Verify fast path was used + assert run2["result"]["extras"]["pickledUsed"] is True, ( + "Second run with pickled should use fast path" + ) + + # Verify response is valid and selectedPoint is applied + assert "next" in run2["result"] + assert len(run2["result"]["next"]) > 0 + + # Index 3 is the highlight x-coord in get_Brownie_Bee_1d_plot's output. + obj1_dim0 = next(p for p in run2["plots"] if p["id"] == "objective_1_0") + plot_data = json.loads(obj1_dim0["plot"]) + assert plot_data["data"][3] == selected_point[0] + + def test_equivalence_with_and_without_pickled(self, app_client, api_key): + """ + Test that pickled vs no-pickled produce identical plots when + selectedPoint, data, and config are the same. + """ + base_payload = self.build_request( + data=MULTI_OBJECTIVE_DATA, + optimizer_config=MULTI_OBJECTIVE_CONFIG, + extras={ + "graphs": ["single", "pareto"], + "graphFormat": "json", + "selectedPoint": [50, 833, 150, 60, "Whipped cream"], + "includeModel": "true", + }, + ) + + # Run without pickled + response_no_pickle = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=base_payload, + content_type="application/json", + ) + assert response_no_pickle.status_code == 200 + result_no_pickle = response_no_pickle.get_json() + + # Get pickled from the response for the next request + pickled_value = result_no_pickle["result"]["pickled"] + + # Run with pickled + payload_with_pickle = copy.deepcopy(base_payload) + payload_with_pickle["extras"]["pickled"] = pickled_value + + response_with_pickle = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload_with_pickle, + content_type="application/json", + ) + assert response_with_pickle.status_code == 200 + result_with_pickle = response_with_pickle.get_json() + + # Verify fast path was engaged + assert result_with_pickle["result"]["extras"]["pickledUsed"] is True + assert result_no_pickle["result"]["extras"]["pickledUsed"] is False + + # Equivalence contract (docs §8): same plots and same next. + assert result_no_pickle["result"]["next"] == result_with_pickle["result"]["next"], ( + "result.next should be identical with and without pickled" + ) + + plots_no_pickle = {p["id"]: p["plot"] for p in result_no_pickle["plots"]} + plots_with_pickle = {p["id"]: p["plot"] for p in result_with_pickle["plots"]} + + assert set(plots_no_pickle.keys()) == set(plots_with_pickle.keys()), ( + "Plot IDs should match between pickled and non-pickled runs" + ) + + for plot_id in plots_no_pickle: + assert plots_no_pickle[plot_id] == plots_with_pickle[plot_id], ( + f"Plot {plot_id} content should be identical with and without pickled" + ) + + def test_pareto_data_structure(self, app_client, api_key): + """Test the detailed structure of pareto_data response.""" + payload = self.build_request( + data=MULTI_OBJECTIVE_DATA, + optimizer_config=MULTI_OBJECTIVE_CONFIG, + extras={ + "graphs": ["pareto"], + "graphFormat": "json", + }, + ) + + response = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload, + content_type="application/json", + ) + assert response.status_code == 200 + result = response.get_json() + + pareto_plot = next(p for p in result["plots"] if p["id"] == "pareto_data") + pareto_data = json.loads(pareto_plot["plot"]) + + # All fields emit_pareto_data writes (docs §7 "per-objective uncertainty" + # corresponds to obj1_error / obj2_error). + assert set(pareto_data.keys()) >= { + "front_x_data", + "front_y_data", + "obj1_error", + "obj2_error", + "best_idx", + } + + # front_y_data is a list of [y_obj1, y_obj2] for each pareto point + assert isinstance(pareto_data["front_y_data"], list) + assert len(pareto_data["front_y_data"]) > 0 # At least one point on pareto front + num_front_points = len(pareto_data["front_y_data"]) + for point in pareto_data["front_y_data"]: + assert isinstance(point, list) + assert len(point) == 2, f"Each point should have 2 objectives, got {len(point)}" + + # front_x_data should have shape (num_points, num_dimensions) + assert isinstance(pareto_data["front_x_data"], list) + assert len(pareto_data["front_x_data"]) == num_front_points, ( + "front_x_data and front_y_data should have same number of points" + ) + for point in pareto_data["front_x_data"]: + assert len(point) == len(MULTI_OBJECTIVE_CONFIG["space"]) + + # Per-objective uncertainty arrays line up with the front + assert isinstance(pareto_data["obj1_error"], list) + assert isinstance(pareto_data["obj2_error"], list) + assert len(pareto_data["obj1_error"]) == num_front_points + assert len(pareto_data["obj2_error"]) == num_front_points + + # best_idx should be a valid index + assert 0 <= pareto_data["best_idx"] < num_front_points + + def test_pareto_with_png_format(self, app_client, api_key): + """PNG mode never emits pareto_data (it is JSON-only).""" + payload = self.build_request( + data=MULTI_OBJECTIVE_DATA, + optimizer_config=MULTI_OBJECTIVE_CONFIG, + extras={ + "graphs": ["pareto", "single"], + "graphFormat": "png", + }, + ) + + response = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload, + content_type="application/json", + ) + assert response.status_code == 200 + result = response.get_json() + + plot_ids = [p["id"] for p in result["plots"]] + # PNG path produces single plots per model, never a pareto_data entry. + assert "pareto_data" not in plot_ids + assert len(result["plots"]) > 0 + assert "next" in result["result"] + + def test_empty_data_with_pareto_request(self, app_client, api_key): + """Empty data + pareto in graphs returns initial-point suggestions without crashing. + + Note: with data=[] there are no yi vectors, so n_objectives defaults to 1 + and the pareto code path is never entered. This test only guards the + empty-data path; the pareto-specific behavior is covered elsewhere. + """ + payload = self.build_request( + data=[], # No data yet + optimizer_config=MULTI_OBJECTIVE_CONFIG, + extras={ + "graphs": ["pareto", "single"], + "graphFormat": "json", + }, + ) + + response = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload, + content_type="application/json", + ) + + # Should return 200 with initial point suggestions + assert response.status_code == 200 + result = response.get_json() + assert "result" in result + assert "next" in result["result"] + + def test_missing_api_key_returns_401(self, app_client): + """Test that missing API key is rejected with 401 Unauthorized.""" + payload = self.build_request( + data=MULTI_OBJECTIVE_DATA, + optimizer_config=MULTI_OBJECTIVE_CONFIG, + extras={"graphs": ["pareto"]}, + ) + + # No apikey query parameter + response = app_client.post( + self.BASE_URL, + json=payload, + content_type="application/json", + ) + + assert response.status_code == 401 + + def test_invalid_optimizer_config_returns_400(self, app_client, api_key): + """Missing required optimizerConfig fields are rejected at validation time as 400.""" + payload = { + "data": MULTI_OBJECTIVE_DATA, + "optimizerConfig": { + "invalid_field": "value", + # Missing required fields (space, baseEstimator, ...) + }, + } + + response = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload, + content_type="application/json", + ) + + assert response.status_code == 400 + result = response.get_json() + # Connexion returns RFC 7807 problem+json + assert "title" in result and "detail" in result + assert result.get("status") == 400 + + def test_single_objective_does_not_include_pareto(self, app_client, api_key): + """Test that single-objective optimization doesn't include pareto_data.""" + single_objective_data = [ + {"xi": [651, 56, 722, "Ræv"], "yi": [1]}, + {"xi": [651, 42, 722, "Ræv"], "yi": [0.2]}, + ] + + single_objective_config = { + "baseEstimator": "GP", + "acqFunc": "gp_hedge", + "initialPoints": 2, + "kappa": 1.96, + "xi": 0.012, + "space": [ + {"type": "discrete", "name": "Sukker", "from": 0, "to": 1000}, + {"type": "continuous", "name": "Peber", "from": 0, "to": 1000}, + {"type": "continuous", "name": "Hvedemel", "from": 0, "to": 1000}, + {"type": "category", "name": "Kunde", "categories": ["Mus", "Ræv"]}, + ], + } + + payload = self.build_request( + data=single_objective_data, + optimizer_config=single_objective_config, + extras={ + "graphs": ["single", "pareto"], # Requesting pareto but it's single-objective + "graphFormat": "json", + }, + ) + + response = app_client.post( + f"{self.BASE_URL}?apikey={api_key}", + json=payload, + content_type="application/json", + ) + + assert response.status_code == 200 + result = response.get_json() + + # pareto_data should NOT be present for single-objective + plot_ids = [p["id"] for p in result["plots"]] + assert "pareto_data" not in plot_ids, ( + "pareto_data should not be present for single-objective optimization" + ) diff --git a/tests/test_no_prints.py b/tests/test_no_prints.py new file mode 100644 index 0000000..eac01e1 --- /dev/null +++ b/tests/test_no_prints.py @@ -0,0 +1,35 @@ +"""Lint test: no stray print() in production code. + +Tests in this repo configure their own logging; production code MUST +route everything through ``logging`` so operators can filter, level- +gate, and route messages in production. ``print`` calls bypass that. +""" + +import ast +import pathlib + +import pytest + +OPTIMIZERAPI_ROOT = pathlib.Path(__file__).parent.parent / "optimizerapi" + + +def _all_python_files() -> list[pathlib.Path]: + return [p for p in OPTIMIZERAPI_ROOT.rglob("*.py") if p.is_file()] + + +@pytest.mark.parametrize("path", _all_python_files(), ids=lambda p: str(p.relative_to(OPTIMIZERAPI_ROOT.parent))) +def test_no_print_calls(path): + """No `print()` call is allowed under optimizerapi/.""" + source = path.read_text(encoding="utf-8") + tree = ast.parse(source, filename=str(path)) + offenders = [ + node.lineno + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "print" + ] + assert not offenders, ( + f"{path}: found print() calls at line(s) {offenders}. " + "Use logging.getLogger(__name__) instead." + ) diff --git a/tests/test_optimizer.py b/tests/test_optimizer.py index 94a6c81..75a2425 100644 --- a/tests/test_optimizer.py +++ b/tests/test_optimizer.py @@ -5,12 +5,9 @@ from unittest.mock import patch import copy import collections.abc +import json from optimizerapi import optimizer_handler as optimizer - -# {'data': [{'xi': [651, 56, 722, 'Ræv'], 'yi': 1}, {'xi': [651, 42, 722, 'Ræv'], 'yi': 0.2}], 'optimizerConfig': {'baseEstimator': 'GP', 'acqFunc': 'gp_hedge', 'initialPoints': 5, 'kappa': 1.96, 'xi': 0.012, 'space': [{'type': 'numeric', 'name': 'Sukker', 'from': 0, 'to': 1000}, {'type': 'numeric', 'name': 'Peber', 'from': 0, 'to': 1000}, {'type': 'numeric', 'name': 'Hvedemel', 'from': 0, 'to': 1000}, {'type': 'category', 'name': 'Kunde', 'categories': ['Mus', 'Ræv']}]}} -# 'data': [{'xi': [0, 5, 'Rød'], 'yi': 10}, {'xi': [5, 8.33, 'Hvid'], 'yi': 3}, {'xi': [10, 1.66, 'Rød'], 'yi': 5}], -# 'optimizerConfig': {'baseEstimator': 'GP', 'acqFunc': 'gp_hedge', 'initialPoints': 3, 'kappa': 1.96, 'xi': 0.01, -# 'space': [{'type': 'discrete', 'name': 'Alkohol', 'from': 0, 'to': 10}, {'type': 'continuous', 'name': 'Vand', 'from': 0, 'to': 10}, {'type': 'category', 'name': 'Farve', 'categories': ['Rød', 'Hvid']}]}} Received extras {'experimentSuggestionCount': 2} +from optimizerapi.securepickle import get_crypto, pickleToString, unpickleFromString sampleData = [ {"xi": [651, 56, 722, "Ræv"], "yi": [1]}, @@ -111,32 +108,31 @@ def validateResult(result): def test_can_be_run_without_data(): - result = optimizer.run(body={"data": [], "optimizerConfig": sampleConfig}) + result = optimizer.run_optimizer(body={"data": [], "optimizerConfig": sampleConfig}) validateResult(result) assert len(result["plots"]) == 0 def test_generates_plots_when_run_with_more_than_initialPoints_samples(): - result = optimizer.run(body={"data": sampleData, "optimizerConfig": sampleConfig}) + result = optimizer.run_optimizer(body={"data": sampleData, "optimizerConfig": sampleConfig}) validateResult(result) assert len(result["result"]["models"]) > 0 - assert len(result["plots"]) == 2 + assert len(result["plots"]) == 7 def test_generates_convergence_plots(): convergence_config = copy.deepcopy(sampleConfig) - convergence_config["extras"] = {"graphs": ["convergence"]} - result = optimizer.run( - body={"data": sampleData, "optimizerConfig": convergence_config} + result = optimizer.run_optimizer( + body={"data": sampleData, "optimizerConfig": convergence_config, "extras": {"graphs": ["convergence"]}} ) validateResult(result) assert len(result["result"]["models"]) > 0 - assert len(result["plots"]) == 2 + assert len(result["plots"]) == 1 assert result["plots"][0]["id"] == "convergence_0" def test_specifying_png_plots(): - result = optimizer.run( + result = optimizer.run_optimizer( body={ "data": sampleData, "optimizerConfig": sampleConfig, @@ -145,21 +141,62 @@ def test_specifying_png_plots(): ) validateResult(result) assert len(result["result"]["models"]) > 0 - assert len(result["plots"]) == 2 + assert len(result["plots"]) == 7 + + +def test_specifying_json_single_plots(): + result = optimizer.run_optimizer( + body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"graphFormat": "json", "graphs": ["single"], "includeModel": "false"}, + } + ) + # Don't call validateResult() because includeModel is false, so pickled model won't be included + assert len(result["result"]["models"]) > 0 + assert len(result["plots"]) == 5 + + plot_ids = [p["id"] for p in result["plots"]] + + assert "single_0_0" in plot_ids + assert "single_0_1" in plot_ids + assert "single_0_2" in plot_ids + assert "single_0_3" in plot_ids + assert "single_0_4" in plot_ids + assert "single_0" not in plot_ids + + for dim_idx in range(4): + plot_id = f"single_0_{dim_idx}" + plot_entry = next(p for p in result["plots"] if p["id"] == plot_id) + plot_data = json.loads(plot_entry["plot"]) + + assert "data" in plot_data + assert isinstance(plot_data["data"], list) + assert len(plot_data["data"]) == 4 + + histogram_entry = next(p for p in result["plots"] if p["id"] == "single_0_4") + histogram_data = json.loads(histogram_entry["plot"]) + + assert "histogram" in histogram_data + assert isinstance(histogram_data["histogram"], dict) + assert "mean" in histogram_data["histogram"] + assert "std" in histogram_data["histogram"] + assert isinstance(histogram_data["histogram"]["mean"], (int, float)) + assert isinstance(histogram_data["histogram"]["std"], (int, float)) def test_specifying_empty_extras_preserve_legacy_plotting(): - result = optimizer.run( + result = optimizer.run_optimizer( body={"data": sampleData, "optimizerConfig": sampleConfig, "extras": {}} ) validateResult(result) assert len(result["result"]["models"]) > 0 - assert len(result["plots"]) == 2 + assert len(result["plots"]) == 7 def test_deselecting_plots(): # If graphFormat is none, no plots should be returned. This should be faster. - result = optimizer.run( + result = optimizer.run_optimizer( body={ "data": sampleData, "optimizerConfig": sampleConfig, @@ -172,17 +209,79 @@ def test_deselecting_plots(): def test_can_accept_multi_objective_data(): - result = optimizer.run( - body={"data": sampleMultiObjectiveData, "optimizerConfig": sampleConfig} + result = optimizer.run_optimizer( + body={ + "data": sampleMultiObjectiveData, + "optimizerConfig": sampleConfig, + "extras": { + "experimentSuggestionCount": 1, + "graphFormat": "json", + "graphs": ["pareto", "single"], + "objectivePars": "expected_minimum", + }, + } ) validateResult(result) assert len(result["result"]["models"]) > 1 - assert len(result["plots"]) == 5 + assert "pareto_data" in [x["id"] for x in result["plots"]] + assert len(result["plots"]) == 11 + + +def test_multi_objective_json_single_plots(): + result = optimizer.run_optimizer( + body={ + "data": sampleMultiObjectiveData, + "optimizerConfig": sampleConfig, + "extras": { + "graphFormat": "json", + "graphs": ["single", "pareto"], + "includeModel": "false", + "experimentSuggestionCount": 1, + "objectivePars": "expected_minimum", + }, + } + ) + assert len(result["result"]["models"]) > 1 + plot_ids = [x["id"] for x in result["plots"]] + + assert len(result["plots"]) == 11 + + assert "pareto_data" in plot_ids + + assert "objective_1_data" not in plot_ids + assert "objective_2_data" not in plot_ids + + for objective_prefix in ["objective_1", "objective_2"]: + for idx in range(5): + plot_id = f"{objective_prefix}_{idx}" + assert plot_id in plot_ids, f"Expected {plot_id} in plot IDs" + + for objective_prefix in ["objective_1", "objective_2"]: + for dim_idx in range(4): + plot_id = f"{objective_prefix}_{dim_idx}" + plot_entry = next(x for x in result["plots"] if x["id"] == plot_id) + plot_data = json.loads(plot_entry["plot"]) + + assert "data" in plot_data + assert isinstance(plot_data["data"], list) + assert len(plot_data["data"]) == 4 + + for objective_prefix in ["objective_1", "objective_2"]: + histogram_id = f"{objective_prefix}_4" + histogram_entry = next(x for x in result["plots"] if x["id"] == histogram_id) + histogram_data = json.loads(histogram_entry["plot"]) + + assert "histogram" in histogram_data + assert isinstance(histogram_data["histogram"], dict) + assert "mean" in histogram_data["histogram"] + assert "std" in histogram_data["histogram"] + assert isinstance(histogram_data["histogram"]["mean"], (int, float)) + assert isinstance(histogram_data["histogram"]["std"], (int, float)) def test_deselecting_pickled_model(): # If includeModel is false, pickled data should not be included in result - result = optimizer.run( + result = optimizer.run_optimizer( body={ "data": sampleData, "optimizerConfig": sampleConfig, @@ -195,7 +294,7 @@ def test_deselecting_pickled_model(): def test_selecting_pickled_model(): # If includeModel is true, pickled data should be included in result - result = optimizer.run( + result = optimizer.run_optimizer( body={ "data": sampleData, "optimizerConfig": sampleConfig, @@ -207,7 +306,7 @@ def test_selecting_pickled_model(): def test_expected_minimum_contains_std_deviation(): - result = optimizer.run(body={"data": sampleData, "optimizerConfig": sampleConfig}) + result = optimizer.run_optimizer(body={"data": sampleData, "optimizerConfig": sampleConfig}) assert "expected_minimum" in result["result"] expected_minimum = result["result"]["expected_minimum"] assert isinstance(expected_minimum[1], collections.abc.Sequence) @@ -217,7 +316,7 @@ def test_expected_minimum_contains_std_deviation(): def test_when_using_constraints_set_constraints_should_be_called(mock): instance = mock.return_value request = brownie_with_constraints - optimizer.run(body=request) + optimizer.run_optimizer(body=request) instance.set_constraints.assert_called_once() @@ -225,7 +324,7 @@ def test_when_using_constraints_set_constraints_should_be_called(mock): def test_when_not_using_constraints_set_constraints_should_not_be_called(mock): instance = mock.return_value request = brownie_without_constraints - optimizer.run(body=request) + optimizer.run_optimizer(body=request) instance.set_constraints.assert_not_called() @@ -233,7 +332,7 @@ def test_when_not_using_constraints_set_constraints_should_not_be_called(mock): def test_when_using_constraints_strategy_cl_min_should_be_used(mock): instance = mock.return_value request = brownie_with_constraints - optimizer.run(body=request) + optimizer.run_optimizer(body=request) instance.ask.assert_called_once_with(n_points=3, strategy="cl_min") @@ -241,5 +340,467 @@ def test_when_using_constraints_strategy_cl_min_should_be_used(mock): def test_when_not_using_constraints_standard_strategy_should_be_used(mock): instance = mock.return_value request = brownie_without_constraints - optimizer.run(body=request) + optimizer.run_optimizer(body=request) instance.ask.assert_called_once_with(n_points=3) + + +def test_selectedPoint_single_objective_json(): + default_result = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"graphFormat": "json", "graphs": ["single"], "includeModel": "false"}, + }) + default_dim0_plot = next(p for p in default_result["plots"] if p["id"] == "single_0_0") + default_x_highlight = json.loads(default_dim0_plot["plot"])["data"][3] + selected_point = [100, 200, 300, "Mus"] + assert selected_point[0] != default_x_highlight, ( + "selected_point[0] must differ from default highlight for this test to be meaningful" + ) + result = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": { + "graphFormat": "json", + "graphs": ["single"], + "includeModel": "false", + "selectedPoint": selected_point, + }, + }) + assert len(result["plots"]) == len(default_result["plots"]) + dim0_plot = next(p for p in result["plots"] if p["id"] == "single_0_0") + plot_data = json.loads(dim0_plot["plot"]) + assert plot_data["data"][3] == selected_point[0] + + +def test_selectedPoint_multi_objective_json(): + selected_point = [651, 56, 722, "Ræv"] + result = optimizer.run_optimizer(body={ + "data": sampleMultiObjectiveData, + "optimizerConfig": sampleConfig, + "extras": { + "graphFormat": "json", + "graphs": ["single"], + "includeModel": "false", + "experimentSuggestionCount": 1, + "selectedPoint": selected_point, + }, + }) + obj1_dim0 = next(p for p in result["plots"] if p["id"] == "objective_1_0") + plot_data = json.loads(obj1_dim0["plot"]) + assert plot_data["data"][3] == selected_point[0] + obj2_dim0 = next(p for p in result["plots"] if p["id"] == "objective_2_0") + plot_data2 = json.loads(obj2_dim0["plot"]) + assert plot_data2["data"][3] == selected_point[0] + + +def test_selectedPoint_with_png_logs_warning_and_is_ignored(caplog): + import logging as _logging + + with caplog.at_level(_logging.WARNING, logger="optimizerapi.optimizer"): + result = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": { + "graphFormat": "png", + "selectedPoint": [651, 56, 722, "Ræv"], + "includeModel": "false", + }, + }) + + assert any("selectedPoint ignored on png path" in r.message for r in caplog.records) + # No crash, response is valid. + assert "plots" in result + assert "next" in result["result"] + + +def test_no_selectedPoint_preserves_default(): + result = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"graphFormat": "json", "graphs": ["single"], "includeModel": "false"}, + }) + assert len(result["plots"]) > 0 + single_plots = [p for p in result["plots"] if p["id"].startswith("single_")] + assert len(single_plots) > 0 + for p in single_plots: + plot_data = json.loads(p["plot"]) + if "histogram" not in plot_data: + assert "data" in plot_data + assert len(plot_data["data"]) == 4 + + +def test_selectedPoint_does_not_change_expected_minimum(): + default_result = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"graphFormat": "json", "graphs": ["single"], "includeModel": "false"}, + }) + selected_point = [651, 56, 722, "Ræv"] + result_with_selection = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": { + "graphFormat": "json", + "graphs": ["single"], + "includeModel": "false", + "selectedPoint": selected_point, + }, + }) + assert ( + default_result["result"]["models"][0]["expected_minimum"] + == result_with_selection["result"]["models"][0]["expected_minimum"] + ) + + +def test_pickled_consumption_skips_training(): + """Test that providing extras.pickled skips model retraining (tell() not called)""" + # First run to get pickled + first_result = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"includeModel": "true"}, + }) + pickled_value = first_result["result"]["pickled"] + assert len(pickled_value) > 0 + + # Second run with pickled - tell() should NOT be called + with patch("optimizerapi.optimizer.Optimizer.tell") as mock_tell: + second_result = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"pickled": pickled_value, "includeModel": "false"}, + }) + mock_tell.assert_not_called() + + assert "result" in second_result + assert "next" in second_result["result"] + assert len(second_result["result"]["next"]) > 0 + + +def test_old_format_pickled_falls_back(caplog): + """A pickled payload that decrypts but isn't the new dict shape falls through.""" + import logging as _logging + + old_format_data = ["some", "old", "data"] + old_pickled = pickleToString(old_format_data, get_crypto()) + + with caplog.at_level(_logging.WARNING, logger="optimizerapi.pickled_state"): + result = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"pickled": old_pickled, "includeModel": "false", "graphFormat": "json"}, + }) + + assert "result" in result + assert len(result["result"]["next"]) > 0 + assert result["result"]["extras"]["pickledUsed"] is False + assert any("bad_structure" in r.message for r in caplog.records) + + +def test_invalid_pickled_falls_back(caplog): + """An undecodable pickled string falls through to a full run.""" + import logging as _logging + + with caplog.at_level(_logging.WARNING, logger="optimizerapi.pickled_state"): + result = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"pickled": "this_is_not_valid_pickled_data_at_all", "includeModel": "false", "graphFormat": "json"}, + }) + + assert "result" in result + assert len(result["result"]["next"]) > 0 + assert result["result"]["extras"]["pickledUsed"] is False + assert any("decrypt_failed" in r.message for r in caplog.records) + + +def test_pickled_response_is_dict_format(): + """Test that pickled response is a dict with keys fingerprint, result, next, optimizer""" + result = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"includeModel": "true", "graphFormat": "json"}, + }) + pickled_value = result["result"]["pickled"] + assert len(pickled_value) > 0 + unpickled = unpickleFromString(pickled_value, get_crypto()) + assert isinstance(unpickled, dict), f"Expected dict, got {type(unpickled)}" + assert set(unpickled.keys()) >= {"fingerprint", "result", "next", "optimizer"} + assert isinstance(unpickled["fingerprint"], str) and len(unpickled["fingerprint"]) == 64 + + +def test_pickled_round_trip(): + """Test that pickled from run 1 can be sent as extras.pickled in run 2""" + # Run 1 + first_result = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"includeModel": "true"}, + }) + pickled_value = first_result["result"]["pickled"] + assert len(pickled_value) > 0 + + # Run 2 with pickled - should produce a valid response + second_result = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"pickled": pickled_value, "includeModel": "false"}, + }) + assert "result" in second_result + assert "next" in second_result["result"] + assert len(second_result["result"]["next"]) > 0 + # Verify plot structure is valid + assert "plots" in second_result + + +# Multi-objective 5-dim data from sample-multi.curl +sampleMultiObjective5DimData = [ + {"xi": [16.7, 500, 250, 20, "None"], "yi": [-2, -17]}, + {"xi": [50, 833, 150, 60, "Whipped cream"], "yi": [-3, -6]}, + {"xi": [58.3, 22, 85, 6, "Frosting"], "yi": [-6, -25]}, +] + +sampleMultiObjective5DimConfig = { + "baseEstimator": "GP", + "acqFunc": "EI", + "initialPoints": 3, + "kappa": 1.96, + "xi": 2, + "space": [ + {"type": "continuous", "name": "Sugar", "from": 0, "to": 100}, + {"type": "continuous", "name": "Flour", "from": 0, "to": 1000}, + {"type": "discrete", "name": "Temperature", "from": 0, "to": 300}, + {"type": "discrete", "name": "Time", "from": 0, "to": 120}, + {"type": "category", "name": "Finish", "categories": ["None", "Frosting", "Whipped cream"]}, + ], + "constraints": [], +} + + +def test_pickled_with_selectedPoint(): + """Integration test: pickled from run 1 consumed in run 2 with selectedPoint from pareto front""" + # Run 1: multi-objective JSON request — get pickled + run1_result = optimizer.run_optimizer(body={ + "data": sampleMultiObjective5DimData, + "optimizerConfig": sampleMultiObjective5DimConfig, + "extras": { + "experimentSuggestionCount": 1, + "graphs": ["pareto", "single"], + "graphFormat": "json", + "includeModel": "true", + }, + }) + assert "result" in run1_result + pickled_value = run1_result["result"]["pickled"] + assert len(pickled_value) > 0 + + # Extract a point from pareto front_x_data + pareto_plot_entry = next(p for p in run1_result["plots"] if p["id"] == "pareto_data") + pareto_data = json.loads(pareto_plot_entry["plot"]) + front_x_data = pareto_data["front_x_data"] + assert len(front_x_data) > 0 + selected_point = front_x_data[0] + + # Run 2: same request + pickled + selectedPoint + run2_result = optimizer.run_optimizer(body={ + "data": sampleMultiObjective5DimData, + "optimizerConfig": sampleMultiObjective5DimConfig, + "extras": { + "experimentSuggestionCount": 1, + "graphs": ["pareto", "single"], + "graphFormat": "json", + "pickled": pickled_value, + "selectedPoint": selected_point, + }, + }) + + # Assert valid response + assert "result" in run2_result + assert "plots" in run2_result + assert len(run2_result["plots"]) > 0 + assert "next" in run2_result["result"] + assert len(run2_result["result"]["next"]) > 0 + + # Assert new pickled is present + new_pickled = run2_result["result"]["pickled"] + assert len(new_pickled) > 0 + + # Assert selectedPoint is reflected in objective_1_0 plot + obj1_dim0 = next(p for p in run2_result["plots"] if p["id"] == "objective_1_0") + plot_data = json.loads(obj1_dim0["plot"]) + assert plot_data["data"][3] == selected_point[0] + + +def test_pickled_consumption_with_include_model_false(): + """Integration test: pickled consumed + includeModel=false → pickled suppressed in response""" + # Run 1: get pickled + run1_result = optimizer.run_optimizer(body={ + "data": sampleMultiObjective5DimData, + "optimizerConfig": sampleMultiObjective5DimConfig, + "extras": { + "experimentSuggestionCount": 1, + "graphs": ["pareto", "single"], + "graphFormat": "json", + "includeModel": "true", + }, + }) + pickled_value = run1_result["result"]["pickled"] + assert len(pickled_value) > 0 + + # Run 2: consume pickled + includeModel=false + run2_result = optimizer.run_optimizer(body={ + "data": sampleMultiObjective5DimData, + "optimizerConfig": sampleMultiObjective5DimConfig, + "extras": { + "experimentSuggestionCount": 1, + "graphs": ["pareto", "single"], + "graphFormat": "json", + "pickled": pickled_value, + "includeModel": "false", + }, + }) + + # includeModel=false suppresses pickled + assert "result" in run2_result + assert "pickled" in run2_result["result"] + assert run2_result["result"]["pickled"] == "" + + # Response is otherwise valid + assert "next" in run2_result["result"] + assert len(run2_result["result"]["next"]) > 0 + assert "plots" in run2_result + assert len(run2_result["plots"]) > 0 + + +def test_pickled_used_flag_round_trip(): + """First run has pickledUsed=False; second run with returned pickled has pickledUsed=True.""" + first = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"includeModel": "true", "graphFormat": "json"}, + }) + assert first["result"]["extras"]["pickledUsed"] is False + pickled_value = first["result"]["pickled"] + assert len(pickled_value) > 0 + + second = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"includeModel": "true", "pickled": pickled_value, "graphFormat": "json"}, + }) + assert second["result"]["extras"]["pickledUsed"] is True + + +def test_selectedPoint_with_no_data(): + """Integration test: selectedPoint with empty data — server handles gracefully""" + selected_point = [50, 833, 150, 60, "Whipped cream"] + result = optimizer.run_optimizer(body={ + "data": [], + "optimizerConfig": sampleMultiObjective5DimConfig, + "extras": { + "experimentSuggestionCount": 1, + "graphs": ["pareto", "single"], + "graphFormat": "json", + "selectedPoint": selected_point, + }, + }) + + # Server handles gracefully — no exception, valid response + assert "result" in result + # No data means no models trained, so no plots + assert "plots" in result + + +def test_equivalence_with_and_without_pickled_multi_objective(): + """Same selectedPoint, same data: pickled vs no-pickled produce identical plots (all ids).""" + base_body = { + "data": sampleMultiObjective5DimData, + "optimizerConfig": sampleMultiObjective5DimConfig, + "extras": { + "graphs": ["single", "pareto"], + "graphFormat": "json", + "selectedPoint": [50, 833, 150, 60, "Whipped cream"], + "includeModel": "true", + }, + } + + # Seed: one full run to capture pickled. + seed = optimizer.run_optimizer(body=copy.deepcopy({ + **base_body, + "extras": {**base_body["extras"], "selectedPoint": None}, + })) + pickled_value = seed["result"]["pickled"] + assert len(pickled_value) > 0 + + # Run A: no pickled (full retrain), with selectedPoint. + body_no_pickle = copy.deepcopy(base_body) + result_no_pickle = optimizer.run_optimizer(body=body_no_pickle) + + # Run B: same request + pickled. + body_with_pickle = copy.deepcopy(base_body) + body_with_pickle["extras"]["pickled"] = pickled_value + result_with_pickle = optimizer.run_optimizer(body=body_with_pickle) + + # Sanity: fast path actually engaged. + assert result_with_pickle["result"]["extras"]["pickledUsed"] is True + assert result_no_pickle["result"]["extras"]["pickledUsed"] is False + + # Compare every plot entry except the pickled string itself (which is + # not in plots) — identical plot ids and identical plot bodies. + plots_no_pickle = {p["id"]: p["plot"] for p in result_no_pickle["plots"]} + plots_with_pickle = {p["id"]: p["plot"] for p in result_with_pickle["plots"]} + assert set(plots_no_pickle) == set(plots_with_pickle) + for plot_id in plots_no_pickle: + assert plots_no_pickle[plot_id] == plots_with_pickle[plot_id], ( + f"divergence on plot {plot_id}" + ) + + +def test_pickled_fingerprint_mismatch_falls_through(caplog): + """A pickled produced from one data set is ignored when data changes.""" + import logging as _logging + + seed = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"includeModel": "true", "graphFormat": "json"}, + }) + pickled_value = seed["result"]["pickled"] + assert len(pickled_value) > 0 + + altered_data = sampleData + [{"xi": [100, 100, 100, "Mus"], "yi": [0.5]}] + + with caplog.at_level(_logging.WARNING, logger="optimizerapi.pickled_state"): + result = optimizer.run_optimizer(body={ + "data": altered_data, + "optimizerConfig": sampleConfig, + "extras": {"includeModel": "true", "pickled": pickled_value, "graphFormat": "json"}, + }) + + assert result["result"]["extras"]["pickledUsed"] is False + assert any("fingerprint_mismatch" in r.message for r in caplog.records) + # Response still valid — server fell through to a full run. + assert "next" in result["result"] + assert len(result["result"]["next"]) > 0 + + +def test_includeModel_false_with_pickled_logs_chain_break_warning(caplog): + import logging as _logging + + seed = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"includeModel": "true", "graphFormat": "json"}, + }) + pickled_value = seed["result"]["pickled"] + + with caplog.at_level(_logging.WARNING, logger="optimizerapi.optimizer"): + second = optimizer.run_optimizer(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"pickled": pickled_value, "includeModel": "false", "graphFormat": "json"}, + }) + + assert any("includeModel=false with extras.pickled" in r.message for r in caplog.records) + # Contract preserved: empty pickled returned. + assert second["result"]["pickled"] == "" diff --git a/tests/test_pickled_state.py b/tests/test_pickled_state.py new file mode 100644 index 0000000..918dad2 --- /dev/null +++ b/tests/test_pickled_state.py @@ -0,0 +1,114 @@ +"""Tests for pickled_state: fingerprint + pack/unpack helpers.""" + +import logging + +from optimizerapi.pickled_state import compute_fingerprint, pack, unpack_if_valid +from optimizerapi.securepickle import get_crypto, pickleToString + + +SAMPLE_DATA = [ + {"xi": [651, 56, 722, "Ræv"], "yi": [1]}, + {"xi": [651, 42, 722, "Ræv"], "yi": [0.2]}, +] + +SAMPLE_CONFIG = { + "baseEstimator": "GP", + "acqFunc": "gp_hedge", + "initialPoints": 2, + "kappa": 1.96, + "xi": 0.012, + "space": [ + {"type": "discrete", "name": "Sukker", "from": 0, "to": 1000}, + {"type": "continuous", "name": "Peber", "from": 0, "to": 1000}, + {"type": "continuous", "name": "Hvedemel", "from": 0, "to": 1000}, + {"type": "category", "name": "Kunde", "categories": ["Mus", "Ræv"]}, + ], +} + + +def test_fingerprint_is_deterministic(): + fp1 = compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) + fp2 = compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) + assert fp1 == fp2 + assert isinstance(fp1, str) + assert len(fp1) == 64 # sha256 hex + + +def test_fingerprint_is_independent_of_dict_key_order(): + reordered_config = { + "space": SAMPLE_CONFIG["space"], + "xi": SAMPLE_CONFIG["xi"], + "kappa": SAMPLE_CONFIG["kappa"], + "initialPoints": SAMPLE_CONFIG["initialPoints"], + "acqFunc": SAMPLE_CONFIG["acqFunc"], + "baseEstimator": SAMPLE_CONFIG["baseEstimator"], + } + assert compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) == compute_fingerprint( + SAMPLE_DATA, reordered_config + ) + + +def test_fingerprint_changes_when_data_changes(): + other_data = SAMPLE_DATA + [{"xi": [100, 100, 100, "Mus"], "yi": [0.5]}] + assert compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) != compute_fingerprint( + other_data, SAMPLE_CONFIG + ) + + +def test_fingerprint_changes_when_config_changes(): + other_config = dict(SAMPLE_CONFIG, kappa=2.0) + assert compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) != compute_fingerprint( + SAMPLE_DATA, other_config + ) + + +def _crypto(): + return get_crypto() + + +def test_pack_unpack_round_trip(): + crypto = _crypto() + fingerprint = compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) + blob = pack(result="result-sentinel", next_points=["next-sentinel"], + optimizer="opt-sentinel", fingerprint=fingerprint, crypto=crypto) + assert isinstance(blob, str) and len(blob) > 0 + + payload = unpack_if_valid(blob, expected_fingerprint=fingerprint, crypto=crypto) + assert payload is not None + assert payload["result"] == "result-sentinel" + assert payload["next"] == ["next-sentinel"] + assert payload["optimizer"] == "opt-sentinel" + assert payload["fingerprint"] == fingerprint + + +def test_unpack_returns_none_on_fingerprint_mismatch(caplog): + crypto = _crypto() + fingerprint = compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) + blob = pack(result="r", next_points=[], optimizer="o", + fingerprint=fingerprint, crypto=crypto) + + with caplog.at_level(logging.WARNING, logger="optimizerapi.pickled_state"): + result = unpack_if_valid(blob, expected_fingerprint="0" * 64, crypto=crypto) + assert result is None + assert any("fingerprint_mismatch" in record.message for record in caplog.records) + + +def test_unpack_returns_none_on_decrypt_failure(caplog): + crypto = _crypto() + with caplog.at_level(logging.WARNING, logger="optimizerapi.pickled_state"): + result = unpack_if_valid("not-a-valid-blob", expected_fingerprint="x" * 64, + crypto=crypto) + assert result is None + assert any("decrypt_failed" in record.message for record in caplog.records) + + +def test_unpack_returns_none_on_bad_structure(caplog): + crypto = _crypto() + # Encrypt a non-dict payload using the same machinery. + bogus_blob = pickleToString(["not", "a", "dict"], crypto) + + with caplog.at_level(logging.WARNING, logger="optimizerapi.pickled_state"): + result = unpack_if_valid(bogus_blob, expected_fingerprint="x" * 64, + crypto=crypto) + assert result is None + assert any("bad_structure" in record.message for record in caplog.records) diff --git a/tests/test_plot_emitters.py b/tests/test_plot_emitters.py new file mode 100644 index 0000000..f242509 --- /dev/null +++ b/tests/test_plot_emitters.py @@ -0,0 +1,67 @@ +"""Tests for the plot-emitters module.""" + +import json + +import numpy +import pytest + +from optimizerapi.plot_emitters import emit_json_single_plots + + +class _FakePlotData: + """Mimics ProcessOptimizer's get_Brownie_Bee_1d_plot return value.""" + + @staticmethod + def make(n_dims: int): + # Each dimension yields a 4-element series; the last entry is a + # histogram (mean, std) pair. + dims = [[[1.0, 2.0, 3.0, 4.0] for _ in range(4)] for _ in range(n_dims)] + histogram = (numpy.array([42.0]), numpy.array([1.5])) + return dims + [histogram] + + +@pytest.fixture +def fake_brownie_1d(monkeypatch): + """Patch _get_brownie_bee_1d_plot_safe so this test doesn't need ProcessOptimizer.""" + def _stub(result, x_eval=None): + return _FakePlotData.make(n_dims=3) + monkeypatch.setattr( + "optimizerapi.plot_emitters._get_brownie_bee_1d_plot_safe", _stub + ) + + +def test_emit_json_single_plots_writes_one_entry_per_dim_plus_histogram(fake_brownie_1d): + plots: list = [] + emit_json_single_plots(plots, result=object(), prefix="single_0", selected_point=None) + ids = [p["id"] for p in plots] + assert ids == ["single_0_0", "single_0_1", "single_0_2", "single_0_3"] + # Last entry is the histogram payload. + histogram_payload = json.loads(plots[-1]["plot"]) + assert "histogram" in histogram_payload + assert histogram_payload["histogram"]["mean"] == 42.0 + assert histogram_payload["histogram"]["std"] == 1.5 + + +def test_emit_json_single_plots_supports_different_prefixes(fake_brownie_1d): + plots: list = [] + emit_json_single_plots(plots, result=object(), prefix="objective_2", selected_point=None) + ids = [p["id"] for p in plots] + assert ids[0] == "objective_2_0" + assert ids[-1] == "objective_2_3" + + +def test_emit_json_single_plots_passes_selected_point_through(monkeypatch): + captured = {} + + def _stub(result, x_eval=None): + captured["x_eval"] = x_eval + return _FakePlotData.make(n_dims=2) + + monkeypatch.setattr( + "optimizerapi.plot_emitters._get_brownie_bee_1d_plot_safe", _stub + ) + + plots: list = [] + sp = [50, 833, "Whipped cream"] + emit_json_single_plots(plots, result=object(), prefix="single_0", selected_point=sp) + assert captured["x_eval"] == sp diff --git a/tests/test_types.py b/tests/test_types.py new file mode 100644 index 0000000..46026e3 --- /dev/null +++ b/tests/test_types.py @@ -0,0 +1,65 @@ +"""Sanity checks for the boundary type definitions.""" + +from optimizerapi.types import ( # noqa: F401 + CachePayload, + Constraint, + DataPoint, + Dimension, + Extras, + OptimizerConfig, + Plot, + RequestBody, + ResponseEnvelope, + ResponseResult, + ResponseResultExtras, +) + + +def test_request_body_accepts_realistic_request(): + """A typical request body matches the RequestBody shape at runtime. + + TypedDicts don't enforce at runtime; we use isinstance(d, dict) as a + proxy and let mypy catch shape mismatches statically. + """ + body: RequestBody = { + "data": [{"xi": [50, 833, 150, 60, "Whipped cream"], "yi": [-3, -6]}], + "optimizerConfig": { + "baseEstimator": "GP", + "acqFunc": "EI", + "initialPoints": 3, + "kappa": 1.96, + "xi": 2, + "space": [ + {"type": "continuous", "name": "Sugar", "from": 0, "to": 100}, + {"type": "category", "name": "Finish", + "categories": ["None", "Whipped cream"]}, + ], + "constraints": [], + }, + "extras": {"graphFormat": "json", "includeModel": "true"}, + } + assert isinstance(body, dict) + assert body["data"][0]["xi"][-1] == "Whipped cream" + + +def test_response_envelope_shape(): + response: ResponseEnvelope = { + "plots": [{"id": "single_0_0", "plot": "{}"}], + "result": { + "next": [[1, 2, 3]], + "models": [], + "pickled": "", + "extras": {"pickledUsed": False}, + }, + } + assert response["result"]["extras"]["pickledUsed"] is False + + +def test_cache_payload_shape(): + payload: CachePayload = { + "fingerprint": "a" * 64, + "result": "opaque", + "next": [[1, 2]], + "optimizer": "opaque", + } + assert payload["fingerprint"] == "a" * 64 From 959422085366461872d7f2b7ae1090f85837890e Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Sat, 30 May 2026 07:57:36 +0000 Subject: [PATCH 07/14] docs: AGENTS.md as source of truth, API usage guide, and audit plans Add AGENTS.md (with CLAUDE.md pointing to it) as the canonical setup, workflow, and style reference; add an end-to-end API usage guide and current-context handoff notes; refresh the README; and capture the 5-phase audit and pareto-extras redesign plans/spec under docs/. --- AGENTS.md | 48 + CLAUDE.md | 5 + README.md | 34 +- current-context.md | 74 ++ docs/api-usage.md | 344 ++++++ .../plans/2026-05-18-audit-overview.md | 45 + .../2026-05-18-audit-phase-1-security.md | 698 +++++++++++ .../plans/2026-05-18-audit-phase-2-logging.md | 417 +++++++ ...2026-05-18-audit-phase-3-boundary-types.md | 648 ++++++++++ ...8-audit-phase-4-refactor-process-result.md | 1050 +++++++++++++++++ .../plans/2026-05-18-audit-phase-5-cleanup.md | 646 ++++++++++ .../2026-05-18-pareto-extras-redesign.md | 993 ++++++++++++++++ ...026-05-18-pareto-extras-redesign-design.md | 130 ++ 13 files changed, 5119 insertions(+), 13 deletions(-) create mode 100644 AGENTS.md create mode 100644 CLAUDE.md create mode 100644 current-context.md create mode 100644 docs/api-usage.md create mode 100644 docs/superpowers/plans/2026-05-18-audit-overview.md create mode 100644 docs/superpowers/plans/2026-05-18-audit-phase-1-security.md create mode 100644 docs/superpowers/plans/2026-05-18-audit-phase-2-logging.md create mode 100644 docs/superpowers/plans/2026-05-18-audit-phase-3-boundary-types.md create mode 100644 docs/superpowers/plans/2026-05-18-audit-phase-4-refactor-process-result.md create mode 100644 docs/superpowers/plans/2026-05-18-audit-phase-5-cleanup.md create mode 100644 docs/superpowers/plans/2026-05-18-pareto-extras-redesign.md create mode 100644 docs/superpowers/specs/2026-05-18-pareto-extras-redesign-design.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..07e47f3 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,48 @@ +# AGENTS + +- Before large refactors, run `python -m pytest`, `flake8 . --max-line-length=127`, and `mypy optimizerapi` to match CI. +- Consult `README.md` alongside this file for end-user setup and deployment details. +- Dev server runs on port 9090; Swagger UI at `/v1.0/ui/`. +- Follow existing style: double quotes for JSON-like payload keys, snake_case for functions/variables, CapWords for classes. +- For development dependencies: `uv pip install -e ".[dev]"` +- Handle errors explicitly; avoid bare `except:` and log or re-raise with context where appropriate. +- Keep functions small and focused; share logic between API handlers and tests instead of duplicating. +- Keep module-level constants UPPER_SNAKE_CASE; avoid one-letter names except simple indices. +- Lint locally with `flake8 . --max-line-length=127` (matches CI workflow). +- No Cursor or GitHub Copilot instruction files are present; if added later, update this AGENTS file to reference them. +- Prefer pure functions where practical; avoid side effects in import time except for necessary configuration. +- Prefer standard-library imports, then third-party, then local; keep tests mirroring src layout. +- Preserve current behavior around environment variables (e.g. `FLASK_ENV`, `CORS_ORIGIN`, `USE_WORKER`). +- Run a single test by node id, e.g. `python -m pytest tests/test_optimizer.py::test_can_be_run_without_data`. +- Run a single test module via `python -m pytest tests/test_optimizer.py`. +- Run the full test suite with `python -m pytest` (watch mode: `ptw`). +- Run the worker (requires Redis) with `python -m optimizerapi.worker`. +- Start the dev server with `python -m optimizerapi.server` (see README.md). +- Type-check locally with `mypy optimizerapi` (matches CI workflow); see `[tool.mypy]` in `pyproject.toml`. +- Type hints are required on new or changed public functions; keep signatures simple and ensure `mypy optimizerapi` passes. +- Use Python 3.13 (pinned in `mise.toml`); with [mise](https://mise.jdx.dev/) installed, `mise install` provisions Python, `uv`, and `.venv` automatically. Without mise, install Python 3.13 and `uv` manually, then `uv pip install -e .`. +- When adding dependencies, update `pyproject.toml` in the `dependencies` array + (or `dev` in `optional-dependencies` for dev tools) and verify with `git diff` + (see README.md). +- When modifying the API, update `optimizerapi/openapi/specification.yml` and keep handler names consistent. +- Write tests under `tests/` using `pytest` style; use `unittest.mock.patch` for external effects as in `tests/test_optimizer.py`. + +## Architecture + +This is an OpenAPI-first REST API wrapping [ProcessOptimizer](https://github.com/novonordisk-research/ProcessOptimizer) (Bayesian optimization). The API has a single main endpoint: `POST /optimizer`. + +**Request flow:** Connexion validates requests against `optimizerapi/openapi/specification.yml`, routes to handler via `operationId`, handler dispatches to optimizer core logic. + +Key modules: +- `optimizerapi/server.py` — Flask/Connexion app init, CORS, Waitress (prod) vs Flask dev server +- `optimizerapi/openapi/specification.yml` — OpenAPI 3.0 spec; defines all schemas and wires handlers via `operationId` +- `optimizerapi/optimizer_handler.py` — HTTP handler; optionally routes work through Redis/RQ job queue (`USE_WORKER=true`) with SHA256-based dedup +- `optimizerapi/optimizer.py` — Core logic: runs optimizer, generates plots (base64 PNG or JSON), handles pickled model reuse, multi-objective support +- `optimizerapi/securepickle/` — Fernet encryption for pickled optimizer state (`PICKLE_KEY` env var) +- `optimizerapi/auth.py` — Optional Keycloak OIDC or static API key auth + +**Multi-objective:** When `yi` arrays have >1 element, the optimizer runs in multi-objective mode with separate per-objective plots (`objective_1_*`, `objective_2_*`) and pareto front data. + +**Pickled model flow:** Client sends `extras.pickled` to skip expensive GP retraining. If unpickling fails (corrupt, wrong key, schema change), falls back to full run with a warning log. + +**Plot formats:** `extras.graphFormat` controls output — `"png"` returns base64 PNG, `"json"` returns structured plot data. JSON mode uses `_get_brownie_bee_1d_plot_safe()` which works around a ProcessOptimizer bug with categorical dimensions. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..ea46917 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,5 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +Source of truth for setup, workflow, style, and architecture: @AGENTS.md. diff --git a/README.md b/README.md index c2dec66..c871591 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,20 @@ This project expose a REST based API for [ProcessOptimizer](https://github.com/n If you have Docker installed the API can be started locally, in development mode, by running the script `build-and-run.sh` -Alternatively the project can be build and run with the following commands: +Alternatively the project can be built and run locally. - python3 -m venv env - source env/bin/activate - pip install --upgrade pip +The toolchain (Python 3.13 + [`uv`](https://github.com/astral-sh/uv)) is +pinned in `mise.toml`. With [mise](https://mise.jdx.dev/) installed +(see the [getting started guide](https://mise.jdx.dev/getting-started.html)), +`cd` into the repo provisions Python, `uv`, and a `.venv` automatically — +just run `mise install` once. Without mise, install Python 3.13 and `uv` +yourself (`brew install uv` or `pip install uv`) and create the venv +manually: - pip install -r requirements-freeze.txt + python3 -m venv .venv + source .venv/bin/activate + + uv pip install -e . python -m optimizerapi.server Now open [http://localhost:9090/v1.0/ui/](http://localhost:9090/v1.0/ui/) in a browser to explore the API through Swagger UI @@ -108,20 +115,21 @@ Keycloak is configured using the following environement variables # Adding or updating dependencies -When adding a new dependency, you should manually add it to `requirements.txt` and then run the following commands: +This project uses `pyproject.toml` for dependency management following PEP 621 standards. - pip install -r requirements.txt - pip freeze | grep --invert-match pkg_resources > requirements-freeze.txt +When adding a new dependency: -Now you should check if the freeze operation resulted in unwanted upates by running: +1. Add it to the `dependencies` array in `pyproject.toml` with a version constraint +2. For development dependencies, add to `[project.optional-dependencies]` under `dev` +3. Install the updated dependencies: - git diff requirements-freeze.txt + uv pip install -e . -After manually fixing any dependencies, you should run: + Or for development dependencies: - pip install -r requirements-freeze.txt + uv pip install -e ".[dev]" -Remember to commit both the changed `requirements.txt` and `requirements-freeze.txt` files. +4. Commit the changed `pyproject.toml` file # Updating the change log diff --git a/current-context.md b/current-context.md new file mode 100644 index 0000000..234944f --- /dev/null +++ b/current-context.md @@ -0,0 +1,74 @@ +# Current Context — pareto branch, design checkpoint + +> Temporary working doc. Captures where the `pareto` branch sits today (2026-05-18) and what the next brainstorm needs to settle. Delete after the brainstorm produces an updated plan. + +## TL;DR + +The `pareto-point-selection` plan was fully implemented and QA-approved on 2026-03-27, but the `pareto` branch has **not** been merged. A UI is being built in parallel and will inform changes to the API. Before merging, we need a design brainstorm focused on the **ergonomics of `selectedPoint` / `pickled` / `includeModel`** — driven by one north-star principle the user has now articulated: + +> **The pickled model is an *efficiency* feature only. The full round-trip (user clicks a pareto point → server re-renders single plots) must work without pickled, just slower.** + +This principle was not stated in the original plan and contradicts at least one of its design decisions (see "Tensions" below). + +## What's on the branch today + +- Two features shipped (commits `5e13f92` → `590b09e`): + - `extras.selectedPoint` — list of X-space coords; overrides `x_eval` in `_get_brownie_bee_1d_plot_safe` at 3 sites in `optimizer.py`. JSON graphFormat only. + - `extras.pickled` — round-trip string; payload format `{"result", "next", "optimizer"}`. Silent fallback to full run on bad/old-format input. +- OpenAPI schema updated with both fields under `extras`. +- `_get_brownie_bee_1d_plot_safe()` wraps a known ProcessOptimizer v1.1.1 bug where `get_Brownie_Bee_1d_plot(x_eval=...)` with categoricals creates a string-dtype numpy array. +- `includeModel` is unchanged from before: `extras.includeModel` is a **stringly-typed** boolean parsed via `json.loads(extras.get("includeModel", "true").lower())` at `optimizer.py:224`. Controls whether the response `pickled` field is populated. +- 29 tests pass. One pre-existing failure (`test_multi_objective_json_single_plots`) is unrelated and explicitly out of scope. + +Full trace lives under `.sisyphus/`: `plans/pareto-point-selection.md`, `notepads/pareto-point-selection/{decisions,issues,learnings}.md`, and `evidence/` (including `final-qa/qa-summary.txt`). + +## UI flow driving the redesign + +User clicks a point on the pareto plot → UI re-requests the optimizer endpoint with that point set as `selectedPoint` → wants single plots refreshed quickly. `pickled` is the speed lever, not a correctness requirement. + +## Loose ends in the working tree + +- `scripts/multi-blank-first-run.curl`, `scripts/multi-blank-second-run.curl` — untracked. 4-dim space (water/temperature/angle/color, categorical), first has empty `data`, second has 5 points × 2 objectives. Hand-rolled smoke tests, not bug repros. Different shape from the existing `sample-multi.curl` (5-dim) and `sample-multi-with-selection.curl`. +- `result.json` — untracked, 0 bytes. Placeholder. +- `CLAUDE.md` — untracked. +- Branch is unmerged; QA approval is six weeks old; behavior may need to shift based on UI findings. + +## Tensions to resolve in the brainstorm + +These are the concrete things to argue out — not yet decided. + +1. **"Pickled overrides data" vs "pickled is efficiency only"** + Original plan decision #23 said pickled overrides the request `data` field. The current implementation actually passes `data` through to `process_result` regardless, so the contract is unclear. Under the new principle, the same request *with* and *without* `extras.pickled` should produce equivalent output — just at different speeds. That means `data` must be authoritative and the client should always send it. Confirm and document; possibly enforce. + +2. **`includeModel=false` + `extras.pickled` sent in** + Today this consumes the incoming state but returns an empty `pickled` field — silently breaking the next iteration's fast path. Options: warn, auto-override to true, forbid, or accept as-is and document. + +3. **`includeModel` typing** + String `"true"`/`"false"` parsed via `json.loads`. Cheap to fix to a real boolean in the OpenAPI spec; would touch every existing client. Decide whether to bundle with the redesign or leave alone. + +4. **`selectedPoint` as raw coordinates vs index** + UI currently has to read `pareto_data.front_x_data`, pick an element, and round-trip the coordinate list. An index reference would be tighter but couples the request to the prior response shape. Open. + +5. **Pickled payload size and shape** + The pickled string is large enough that `sample-multi-with-selection.curl` documents it as "paste manually". Round-tripping it through the browser is workable but ugly. A server-side session cache + short ID is an option; not requested, but worth flagging since the UI will surface this pain. + +6. **PNG path has no `selectedPoint` support** + Deliberate exclusion in the original plan. If the UI ever wants PNG fallback, this becomes a gap. + +## Out of scope for the brainstorm (do not reopen) + +- Fixing the pre-existing `test_multi_objective_json_single_plots` failure. +- Modifying `ProcessOptimizer` source. +- Changing `expected_minimum` computation. +- The `_get_brownie_bee_1d_plot_safe` workaround — it works. + +## Open questions for the user (to bring into the brainstorm) + +- What does the UI need from a *failed* fast path? Silent fallback (current), explicit error, or a "stale" flag in the response? +- Is the long-term plan to keep optimizer state strictly client-managed (stateless server), or is a session/cache acceptable? +- Are there UI affordances already designed that lock us into a particular extras shape (e.g., is the UI already sending `selectedPoint` as coordinates)? +- What's the merge target / timeline once the brainstorm settles? + +## Suggested next move + +Run `superpowers:brainstorming` against this doc as the seed. Goal: produce a revised plan that replaces the relevant TODOs in `.sisyphus/plans/pareto-point-selection.md` (or a fresh plan file) and clarifies the principle above in writing before any more code lands. diff --git a/docs/api-usage.md b/docs/api-usage.md new file mode 100644 index 0000000..046b143 --- /dev/null +++ b/docs/api-usage.md @@ -0,0 +1,344 @@ +# Using the Process Optimizer API + +This document walks through the end-to-end process of driving an experiment with the Process Optimizer API. It covers the request shape, the response shape, the suggested experiment loop, multi-objective workflows with pareto-point exploration, and the performance contract around cached model state. + +The API is OpenAPI-first. For machine-readable details and an interactive sandbox, see the Swagger UI at `http://localhost:9090/v1.0/ui/`. This document describes the *process* of using the API — what to send, what comes back, and how successive calls fit together. + +## 1. What the API does + +The Process Optimizer API wraps [ProcessOptimizer](https://github.com/novonordisk-research/ProcessOptimizer) — a Bayesian optimization library — behind a single REST endpoint: + +``` +POST /v1.0/optimizer +``` + +You describe an experiment space and the data you've collected so far, and the server returns the next experiment(s) to run, plus diagnostic plots of the model's current beliefs. Each call is stateless: the full history is sent in the request body. A cache hint (`extras.pickled`) lets you skip retraining on repeat calls without affecting correctness. + +The endpoint is the same for single- and multi-objective experiments — multi-objective is inferred from the shape of `yi`. + +## 2. Running the server + +For development: + +```bash +uv pip install -e . +python -m optimizerapi.server +``` + +The server listens on port `9090` with Swagger UI at `/v1.0/ui/`. See `README.md` for production deployment, Redis-backed job queueing, CORS, and auth configuration. + +## 3. Authentication + +The endpoint accepts either of: + +- **Static API key** as the `apikey` query parameter. Set `AUTH_API_KEY` on the server; clients add `?apikey=` to the URL. +- **Keycloak OIDC** bearer token in `Authorization: Bearer `. See `README.md` for the relevant `AUTH_*` env vars. + +If neither is configured, the server still requires the `apikey` query parameter — pass any value (e.g. `?apikey=none`) for local development. + +## 4. The request body + +The endpoint accepts a JSON object with three top-level fields: + +```json +{ + "data": [ ... measurement history ... ], + "optimizerConfig": { ... search space and BO hyperparameters ... }, + "extras": { ... output and caching options ... } +} +``` + +`data` and `optimizerConfig` are **authoritative inputs** — together they define what the server computes. `extras` shapes the output (which plots, which format) and provides a cache hint to make repeat calls faster; it never affects correctness. + +### 4.1 `optimizerConfig` — the search space and BO hyperparameters + +```jsonc +{ + "baseEstimator": "GP", // "GP" or another ProcessOptimizer base estimator + "acqFunc": "EI", // "EI" | "PI" | "LCB" | "gp_hedge" ... + "initialPoints": 3, // # of random samples before model takes over + "kappa": 1.96, // exploration weight for LCB + "xi": 0.01, // improvement threshold for EI / PI + "space": [ ... ], // list of dimensions, see below + "constraints": [ ... ] // optional, see below +} +``` + +Each dimension in `space` has one of three types: + +```jsonc +// continuous (float) +{ "type": "continuous", "name": "Sugar", "from": 0, "to": 100 } + +// discrete (integer) +{ "type": "discrete", "name": "Temperature", "from": 0, "to": 300 } + +// categorical +{ "type": "category", "name": "Finish", "categories": ["None", "Frosting", "Whipped cream"] } +``` + +`constraints` is an optional list of sum-constraints across dimensions: + +```jsonc +[ + { "type": "sum", "dimensions": [0, 1], "value": 200 } // dim[0] + dim[1] == 200 +] +``` + +### 4.2 `data` — the measurement history + +A list of `{xi, yi}` pairs. `xi` is one experiment's coordinates in the same order as `space`. `yi` is a **list** of measured outcomes — length 1 for single-objective, length ≥ 2 for multi-objective. The optimizer minimizes each `yi` component. + +```jsonc +{ + "data": [ + { "xi": [50, 833, 150, 60, "Whipped cream"], "yi": [-3, -6] }, + { "xi": [16.7, 500, 250, 20, "None"], "yi": [-2, -17] } + ] +} +``` + +For the very first call you have no measurements yet — send `"data": []`. The server returns initial-point suggestions instead. + +> **Maximize instead of minimize?** Negate the value client-side. The optimizer always minimizes. + +### 4.3 `extras` — output and cache options + +All fields are optional. The most useful ones: + +| Field | Meaning | Default | +| ----------------------------- | --------------------------------------------------------------------------------------------- | --------------------------------------------- | +| `experimentSuggestionCount` | How many next-experiments to return | `1` | +| `graphFormat` | `"png"` (base64-encoded images) or `"json"` (structured plot data) | `"png"` | +| `graphs` | Which plots to compute: subset of `["objective", "convergence", "pareto", "single"]` | `["objective", "convergence", "pareto", "single"]` | +| `maxQuality` | Render quality for PNG plots | `5` | +| `objectivePars` | `"result"` or `"expected_minimum"` — where to evaluate the objective plot | `"result"` | +| `includeModel` | String `"true"` / `"false"` — whether to populate `result.pickled` in the response | `"true"` | +| `selectedPoint` | Override the highlight point in single plots with explicit X-space coordinates. JSON only. | none | +| `pickled` | Cache hint from a previous response. The server validates it against the current request. | none | + +Notes on a few less-obvious knobs: + +- `graphFormat` controls the plot encoding. `"png"` produces classic image plots and is intended for human-facing UIs that display PNGs. `"json"` returns structured per-dimension series for clients that render plots themselves (e.g. a React-based UI). +- `selectedPoint` is honored **only** on the JSON path. On the PNG path it is logged-and-ignored. +- `includeModel: "false"` skips serializing the model into the response, saving bandwidth. The next call then has to do a full retrain (no cache available) — the server logs a warning when it sees `includeModel: "false"` together with an incoming `pickled`, since this combination silently breaks the cache chain. + +## 5. The response body + +```jsonc +{ + "plots": [ + { "id": "single_0_0", "plot": "" }, + { "id": "convergence_0", "plot": "<...>" }, + { "id": "pareto_data", "plot": "<...>" } + // ... more + ], + "result": { + "next": [ [50, 833, 150, 60, "Whipped cream"] ], // suggested experiments + "models": [ { "expected_minimum": [...], "extras": {} } ], + "expected_minimum": [...], + "pickled": "", + "extras": { + "pickledUsed": false, // fast path engaged? + "parameters": { ... echo of the request inputs ... }, + "version": "..." + } + } +} +``` + +Plot ids by mode: + +| Mode | Plot ids | +| ------------------------------- | ----------------------------------------------------------------------------- | +| Single-objective, `png` | `single_0`, `convergence_0`, `objective` plots (when available) | +| Single-objective, `json` | `single_0_0`, `single_0_1`, ..., `single_0_` (per-dim series), `single_0_` (histogram entry) | +| Multi-objective | `pareto_data` plus per-objective single/objective plots (`objective_1_*`, `objective_2_*`) | + +The `next` field is what you feed back into the next iteration: run those experiments, record the results, append to `data`, and call again. + +## 6. The optimization loop + +A typical sequence: + +1. **Round 1 — no data yet.** POST with `"data": []` and your `optimizerConfig`. The server returns one or more initial-point suggestions in `result.next`. +2. **Run the experiments.** Measure `yi` for each suggested `xi`. +3. **Round 2..N.** POST again with the accumulated `data` (all prior `{xi, yi}` pairs). The server fits a model, returns plots, and proposes the next experiment in `result.next`. +4. **(Optional) Pass the cache hint.** On rounds 2..N, include the previous response's `result.pickled` as `extras.pickled`. The server validates that the data and config still match what produced that cache; if they match, model fitting is skipped and the response comes back faster. +5. **Stop when you're satisfied** — when convergence flattens, your budget is exhausted, or `result.expected_minimum` is good enough. + +A minimal curl call for round 1: + +```bash +curl 'http://localhost:9090/v1.0/optimizer?apikey=none' \ + -X POST \ + -H 'Content-Type: application/json' \ + --data-raw '{ + "extras": {"experimentSuggestionCount": 1}, + "data": [], + "optimizerConfig": { + "baseEstimator": "GP", + "acqFunc": "gp_hedge", + "initialPoints": 3, + "kappa": 1.96, + "xi": 0.01, + "space": [ + {"type": "continuous", "name": "Red", "from": 0, "to": 255}, + {"type": "continuous", "name": "Green", "from": 0, "to": 255}, + {"type": "discrete", "name": "Blue", "from": 0, "to": 255} + ] + } + }' +``` + +Pre-built sample requests live in `scripts/`: + +- `sample.curl` — minimal single-objective, JSON plot format +- `sample-multi.curl` — multi-objective with multiple data points +- `sample-multi-with-selection.curl` — multi-objective with `selectedPoint` + `pickled` (the pareto-exploration flow) + +## 7. Multi-objective and pareto-point exploration + +When `yi` is a list of length ≥ 2, the optimizer runs in multi-objective mode. The response then includes: + +- A `pareto_data` plot entry with the pareto front in objective space (`front_y_data`), the corresponding points in X-space (`front_x_data`), per-objective uncertainty, and a `best_idx` highlighting a recommended compromise. +- Per-objective single plots (`objective_1_*`, `objective_2_*`). + +A common UI flow: + +1. Render the pareto front from `pareto_data`. +2. The user clicks a point on the pareto front. The UI looks up the corresponding `front_x_data[i]`. +3. The UI re-requests the optimizer endpoint with that coordinate list as `extras.selectedPoint` (and `graphFormat: "json"`). The server re-renders the single plots highlighting that point. +4. To keep this responsive, the UI also sends back the previous response's `result.pickled` as `extras.pickled` — the server skips the GP retraining step. + +The full process, end-to-end, is the curl pair shown in `scripts/sample-multi-with-selection.curl`. + +## 8. The pickled cache contract + +The `pickled` field is a performance optimization. Two guarantees: + +- **Equivalence.** For any request `R`, the response to `R` and to `R ∪ {extras.pickled: P}` are observably equivalent — same plots, same `next`, same `expected_minimum` — provided `P` was produced by an earlier response whose `data` and `optimizerConfig` matched `R`. Only `result.pickled` itself and the timing differ. +- **Fingerprint-validated.** The server embeds a hash of `(data, optimizerConfig)` inside the pickled blob at pack time. On a subsequent request the server recomputes the fingerprint and compares it to the one inside the blob. If they don't match — because the client sent a stale cache or changed the config — the cache is silently ignored and a full run is performed. + +`result.extras.pickledUsed` tells you which path the server took: `true` for the fast path, `false` for a full run. The UI can use this to surface latency expectations without measuring. + +Three things can cause a fall-through to a full run (each logs a single `WARNING`): + +| Reason | When | Logger / tag | +| ----------------------- | ----------------------------------------------------------------------------------- | ------------------------------------- | +| `decrypt_failed` | Blob cannot be decrypted (rotated `PICKLE_KEY`, corrupted bytes) | `optimizerapi.pickled_state` | +| `bad_structure` | Blob decrypts but isn't the expected `{fingerprint, result, next, optimizer}` shape | `optimizerapi.pickled_state` | +| `fingerprint_mismatch` | Blob decrypts and is well-formed, but the embedded fingerprint differs from the request | `optimizerapi.pickled_state` | + +Two other warnings, not cache fall-throughs, surface useful misuse signals: + +- `optimizerapi.optimizer`: `"selectedPoint ignored on png path"` — `selectedPoint` was sent with `graphFormat: "png"`. The field is JSON-only; on the PNG path it does nothing. +- `optimizerapi.optimizer`: `"includeModel=false with extras.pickled — next call will pay the full cost"` — the client used the cache to take the fast path but suppressed the new cache from the response, so the next call cannot reuse it. + +None of these conditions are errors. The HTTP response is always valid in all five cases. + +> **Why fingerprinting?** It guarantees the equivalence property mechanically: a client can never accidentally short-circuit the optimizer with a cache that was produced from different data, because such a cache is detected and ignored. + +## 9. Error handling + +The endpoint returns three status codes: + +- `200 OK` — Success. Body is the result schema described in §5. +- `400 Bad Request` — Validation, type, or I/O error in the request. +- `500 Internal Server Error` — Unexpected server error. + +Both error responses use the RFC 7807 problem+json shape produced by Connexion: + +```jsonc +{ + "title": "Bad request", // or "Internal server error" + "detail": "", // exception message or validator output + "status": 400, // or 500 + "type": "about:blank" +} +``` + +For health checks: `GET /v1.0/health` returns `200` if the service is reachable. + +## 10. End-to-end example: pareto-point exploration + +This is the same flow as `scripts/sample-multi-with-selection.curl`, narrated step by step. + +### Step 1 — initial multi-objective run + +The client sends 3 measurements and asks for JSON pareto and single plots. The server returns plots, a next-experiment suggestion, and a `result.pickled` cache blob. + +```bash +curl 'http://localhost:9090/v1.0/optimizer?apikey=none' \ + -X POST \ + -H 'Content-Type: application/json' \ + --data-raw '{ + "extras": { + "experimentSuggestionCount": 1, + "graphs": ["pareto", "single"], + "graphFormat": "json" + }, + "data": [ + {"xi": [16.7, 500, 250, 20, "None"], "yi": [-2, -17]}, + {"xi": [50, 833, 150, 60, "Whipped cream"], "yi": [-3, -6]}, + {"xi": [58.3, 22, 85, 6, "Frosting"], "yi": [-6, -25]} + ], + "optimizerConfig": { + "baseEstimator": "GP", + "acqFunc": "EI", + "initialPoints": 3, + "kappa": 1.96, + "xi": 2, + "space": [ + {"type": "continuous", "name": "Sugar", "from": 0, "to": 100}, + {"type": "continuous", "name": "Flour", "from": 0, "to": 1000}, + {"type": "discrete", "name": "Temperature", "from": 0, "to": 300}, + {"type": "discrete", "name": "Time", "from": 0, "to": 120}, + {"type": "category", "name": "Finish", "categories": ["None", "Frosting", "Whipped cream"]} + ], + "constraints": [] + } + }' +``` + +The response contains, among other things: + +- A `pareto_data` plot entry the UI uses to render the pareto front. +- `result.pickled` — store this for the next call. +- `result.extras.pickledUsed: false` — this was a full run. + +### Step 2 — the user clicks a pareto point + +The UI takes the coordinates of the clicked point (read from `pareto_data.front_x_data[i]`) and re-requests with `selectedPoint` set, including the previous `pickled` for speed: + +```bash +curl 'http://localhost:9090/v1.0/optimizer?apikey=none' \ + -X POST \ + -H 'Content-Type: application/json' \ + --data-raw '{ + "extras": { + "experimentSuggestionCount": 1, + "graphs": ["pareto", "single"], + "graphFormat": "json", + "selectedPoint": [50, 833, 150, 60, "Whipped cream"], + "pickled": "" + }, + "data": [ ... same as step 1 ... ], + "optimizerConfig": { ... same as step 1 ... } + }' +``` + +If `data` and `optimizerConfig` match what produced the cache, `result.extras.pickledUsed` comes back `true` and the single plots are re-rendered with the new highlight point — without retraining the GP. If anything has drifted, the server silently falls through to a full run and the response is still correct (just slower). + +That's the loop: keep accumulating measurements, keep round-tripping `pickled`, and reach for `selectedPoint` whenever a UI interaction needs a fresh single-plot view. + +## 11. Quick reference + +- **Endpoint:** `POST /v1.0/optimizer` +- **Auth:** `?apikey=` or `Authorization: Bearer ` +- **Request:** `{data, optimizerConfig, extras}` +- **Response:** `{plots[], result: {next, models, expected_minimum, pickled, extras: {pickledUsed, parameters, version}}}` +- **Health:** `GET /v1.0/health` +- **Swagger UI:** `http://localhost:9090/v1.0/ui/` +- **Sample requests:** `scripts/sample.curl`, `scripts/sample-multi.curl`, `scripts/sample-multi-with-selection.curl` +- **Design notes:** `docs/superpowers/specs/2026-05-18-pareto-extras-redesign-design.md` diff --git a/docs/superpowers/plans/2026-05-18-audit-overview.md b/docs/superpowers/plans/2026-05-18-audit-overview.md new file mode 100644 index 0000000..4f80b09 --- /dev/null +++ b/docs/superpowers/plans/2026-05-18-audit-overview.md @@ -0,0 +1,45 @@ +# Codebase Audit — 5-Phase Improvement Plan + +This is the index for the audit-driven improvements identified after the Pareto extras redesign landed. The audit covered structure, idioms, typing, security, logging, and tests. Findings are split into five phases that each ship independent, testable software. + +## Ordering rationale + +The phases are ordered by risk × dependency: + +1. **Security & dependency hygiene** — low blast radius, highest urgency. CVE bumps + fix auth issues. Lays no traps for later phases. +2. **Logging migration** — mechanical sweep; produces no behavior change but cleans up the surface that Phase 1 partly touched. +3. **Boundary types** — adds `TypedDict`s and a static type checker. Acts as a safety net for Phase 4. +4. **Refactor `process_result`** — the biggest behavior-preserving change. Splits ~250 lines into focused helpers. Types from Phase 3 catch mistakes that would otherwise need runtime to surface. +5. **Opportunistic cleanup** — small wins (fixtures, idiomatic Python, dead code) that benefit from the structural work above. + +Each phase has its own plan file in this directory. + +## Phase plans + +| # | Plan | Touches | +| - | --------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------ | +| 1 | [`2026-05-18-audit-phase-1-security.md`](./2026-05-18-audit-phase-1-security.md) | `auth.py`, `securepickle/`, `pyproject.toml` | +| 2 | [`2026-05-18-audit-phase-2-logging.md`](./2026-05-18-audit-phase-2-logging.md) | `server.py`, `auth.py`, `secure.py`, `optimizer_handler.py`, `optimizer.py` | +| 3 | [`2026-05-18-audit-phase-3-boundary-types.md`](./2026-05-18-audit-phase-3-boundary-types.md) | new `types.py`, all module signatures, mypy config | +| 4 | [`2026-05-18-audit-phase-4-refactor-process-result.md`](./2026-05-18-audit-phase-4-refactor-process-result.md) | `optimizer.py`, new `plot_emitters.py` | +| 5 | [`2026-05-18-audit-phase-5-cleanup.md`](./2026-05-18-audit-phase-5-cleanup.md) | tests, `optimizer_handler.py`, `auth.py` | + +## Out of scope across all phases + +- Flask 2 → 3 / Connexion 2 → 3 migration. Connexion 3 is a rewrite (async) — this is its own project, not a cleanup. Phase 1 limits the dependency bumps to libraries that don't ripple through the framework. +- ProcessOptimizer source changes. +- Switching auth providers; redesigning the API contract; multi-tenant features. +- Performance work (async/background queue model). Phase 5 notes the issue but does not fix it. + +## Acceptance for the audit as a whole + +When all five phases are done: + +- `flake8 optimizerapi tests --max-line-length=127` is fully clean (no pre-existing exceptions remain). +- `mypy optimizerapi` (or `pyright`) is clean. +- `python -m pytest` is 43+ passed (no regressions; new tests for security fixes). +- `optimizer.py` is under ~250 lines; `process_result` is under ~80 lines. +- `auth.py` uses `secrets.compare_digest` and never prints tokens. +- `cryptography` is on a current major. +- No `print()` calls in any module under `optimizerapi/`. +- The OpenAPI surface is unchanged (response shape and field names stay the same). diff --git a/docs/superpowers/plans/2026-05-18-audit-phase-1-security.md b/docs/superpowers/plans/2026-05-18-audit-phase-1-security.md new file mode 100644 index 0000000..282cf53 --- /dev/null +++ b/docs/superpowers/plans/2026-05-18-audit-phase-1-security.md @@ -0,0 +1,698 @@ +# Audit Phase 1 — Security & Dependency Hygiene + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Bump out-of-date crypto dependencies, eliminate auth-related anti-patterns (timing-attack-vulnerable equality, token-logging, default-API-key acceptance), and stop the `securepickle` module from mutating `os.environ` as a side effect. + +**Architecture:** Single-shot changes to three files: `pyproject.toml` (dependency bumps), `optimizerapi/auth.py` (constant-time comparison + fail-closed + drop token log), `optimizerapi/securepickle/secure.py` (`is None` + no env mutation + warning). One new test module added: `tests/test_auth.py`. No API contract changes. + +**Tech Stack:** Python 3.13 (just bumped), `cryptography`, Flask 2.2.x, connexion 2.14.x, pytest. `secrets.compare_digest` from stdlib for constant-time string comparison. + +**Audit reference:** §5 of the audit findings. Specifically `cryptography==3.4.7` (2021, has CVEs), `apikey_handler`'s `==` comparison, the `print(access_token)` line in `auth.py:31`, and the `os.environ` mutation in `securepickle/secure.py`. + +--- + +## File Map + +- **Modify:** `pyproject.toml` — bump `cryptography`, `jsonschema`. +- **Modify:** `optimizerapi/auth.py` — `secrets.compare_digest`, drop token print, fail-closed default behavior. +- **Modify:** `optimizerapi/securepickle/secure.py` — `is None`, drop `os.environ` mutation, use `logging` for the no-key-set message. +- **Create:** `tests/test_auth.py` — unit tests for `apikey_handler` covering correct key, wrong key, and fail-closed default. + +All work happens on branch `langdal/ai-restructure`. Test commands run via the existing `env/` venv (Python 3.13.13). + +--- + +### Task 1: Bump `cryptography` to a current major + +**Files:** +- Modify: `pyproject.toml:32` + +- [ ] **Step 1: Check the dependency tree for hidden cryptography constraints** + +Run: +```bash +env/bin/pip show python-keycloak python-jose | grep -i "Requires\|cryptography" +``` +Expected: lists what other packages need from `cryptography`. Note the highest minor version implied by any constraint. + +- [ ] **Step 2: Edit `pyproject.toml`** + +Find the line: +``` + "cryptography==3.4.7", +``` + +Replace with: +``` + "cryptography>=44,<46", +``` + +The lower bound is the current LTS series; the upper guards against major-version surprises while still letting patch updates land. + +- [ ] **Step 3: Reinstall** + +Run: +```bash +env/bin/pip install -e ".[dev]" 2>&1 | tail -5 +``` +Expected: `Successfully installed ... cryptography-44.x.x ...` and no error. + +- [ ] **Step 4: Run securepickle tests** + +Run: +```bash +env/bin/python -m pytest tests/test_securepickle.py -v +``` +Expected: PASS. + +- [ ] **Step 5: Run the full suite** + +Run: +```bash +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: 43 passed. + +- [ ] **Step 6: Commit** + +```bash +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -am "$(cat <<'EOF' +build: bump cryptography to current major (>=44) + +cryptography 3.4.7 is from 2021 and has known CVEs. Fernet API surface +has not changed across 3.x→44.x so this is a drop-in bump. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +If Step 3 surfaces an irreconcilable conflict (some dep pins `cryptography=4.21,<5", +``` + +- [ ] **Step 2: Reinstall** + +```bash +env/bin/pip install -e ".[dev]" 2>&1 | tail -5 +``` +Expected: jsonschema upgraded, no errors. + +- [ ] **Step 3: Run the full suite** + +```bash +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: 43 passed. + +If pytest fails, revert the bump (`git checkout pyproject.toml`) and pin to the highest minor that works. Connexion 2.14.2 uses jsonschema for OpenAPI validation — a bad bump shows up immediately as a validation error on every request. + +- [ ] **Step 4: Commit** + +```bash +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -am "$(cat <<'EOF' +build: bump jsonschema to 4.21+ + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 3: Constant-time API-key comparison + +**Files:** +- Modify: `optimizerapi/auth.py:44-55` +- Create: `tests/test_auth.py` + +- [ ] **Step 1: Write the failing test** + +Create `tests/test_auth.py`: + +```python +"""Tests for the API-key handler.""" + +from unittest.mock import patch + + +def test_correct_apikey_returns_scope_dict(): + """A request with the configured API key returns a scope dict.""" + with patch("optimizerapi.auth.AUTH_API_KEY", "secret-key"), \ + patch("optimizerapi.auth.AUTH_SERVER", None): + from optimizerapi import auth + result = auth.apikey_handler("secret-key") + assert result == {"scope": []} + + +def test_wrong_apikey_returns_none(): + """An incorrect API key is rejected.""" + with patch("optimizerapi.auth.AUTH_API_KEY", "secret-key"), \ + patch("optimizerapi.auth.AUTH_SERVER", None): + from optimizerapi import auth + result = auth.apikey_handler("not-the-key") + assert result is None + + +def test_apikey_handler_uses_constant_time_comparison(): + """The handler must compare keys via secrets.compare_digest.""" + from optimizerapi import auth + import secrets + + with patch("optimizerapi.auth.AUTH_API_KEY", "secret-key"), \ + patch("optimizerapi.auth.AUTH_SERVER", None), \ + patch.object(secrets, "compare_digest", wraps=secrets.compare_digest) as mock_cmp: + auth.apikey_handler("secret-key") + assert mock_cmp.called, "apikey_handler must use secrets.compare_digest" +``` + +- [ ] **Step 2: Run test to verify some fail** + +Run: +```bash +env/bin/python -m pytest tests/test_auth.py -v +``` +Expected: the first two PASS (current `==` logic works for those), the third FAILS with `AssertionError: apikey_handler must use secrets.compare_digest`. + +- [ ] **Step 3: Edit `optimizerapi/auth.py`** + +At the top of the file, the existing imports are: + +```python +import os +from keycloak import KeycloakOpenID +``` + +Add `secrets` and `logging`: + +```python +import logging +import os +import secrets + +from keycloak import KeycloakOpenID +``` + +Then replace the existing `apikey_handler`: + +```python +def apikey_handler(access_token) -> dict: + """Verify API key based on environment variable + + Returns + ------- + dict + a dictionary containing sub and scope + None in case of invalid token + """ + if not AUTH_SERVER and AUTH_API_KEY == access_token: + return {"scope": []} + return None +``` + +with: + +```python +def apikey_handler(access_token: str) -> dict | None: + """Verify the API key passed by the client. + + Returns + ------- + dict + ``{"scope": []}`` if the supplied key matches the configured + ``AUTH_API_KEY``. + None + If the key is wrong, or if no static-key auth is configured. + """ + if AUTH_SERVER: + # OIDC is configured; static-key path is disabled. + return None + expected = AUTH_API_KEY.encode("utf-8") + provided = (access_token or "").encode("utf-8") + if secrets.compare_digest(expected, provided): + return {"scope": []} + return None +``` + +- [ ] **Step 4: Run tests** + +Run: +```bash +env/bin/python -m pytest tests/test_auth.py -v +``` +Expected: all 3 PASS. + +Run the full suite: +```bash +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: 46 passed (43 + 3 new). + +- [ ] **Step 5: Commit** + +```bash +git add optimizerapi/auth.py tests/test_auth.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +fix(auth): use secrets.compare_digest for API-key comparison + +Constant-time comparison protects against timing-side-channel attacks +when the configured AUTH_API_KEY is non-trivial. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 4: Remove access-token logging in `token_info` + +**Files:** +- Modify: `optimizerapi/auth.py:22-41` + +- [ ] **Step 1: Edit `optimizerapi/auth.py`** + +Replace the existing `token_info`: + +```python +def token_info(access_token) -> dict: + """Verify token with authentication server + + Returns + ------- + dict + a dictionary containing sub and scope + None in case of invalid token + """ + print(access_token) + if not AUTH_SERVER: + return {"scope": []} + token = access_token + token_data = keycloak_openid.introspect(token) + if "active" in token_data and token_data["active"]: + print("OK") + return token_data + print("NOT OK") + print(token_data) + return None +``` + +with: + +```python +_LOG = logging.getLogger(__name__) + + +def token_info(access_token: str) -> dict | None: + """Verify a bearer token against the configured Keycloak server. + + Returns + ------- + dict + Token data from Keycloak introspection when the token is active. + If no OIDC server is configured, returns ``{"scope": []}``. + None + If the server reports the token as inactive. + """ + if not AUTH_SERVER: + return {"scope": []} + token_data = keycloak_openid.introspect(access_token) + if token_data.get("active"): + _LOG.debug("token accepted") + return token_data + _LOG.warning("token rejected by Keycloak") + return None +``` + +The four `print` lines (including the one that leaked the raw access token) are gone. `_LOG.debug` / `_LOG.warning` use the standard logger; the token itself is never logged. + +- [ ] **Step 2: Run the full suite** + +```bash +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: 46 passed. + +- [ ] **Step 3: Verify no `print` remains in `auth.py`** + +Run: +```bash +grep -n "print(" optimizerapi/auth.py +``` +Expected: no output. + +- [ ] **Step 4: Commit** + +```bash +git add optimizerapi/auth.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +fix(auth): stop logging raw access tokens to stdout + +token_info previously printed the access_token on every request, which +leaks bearer credentials to whatever scrapes the API server's stdout. +Replaced with structured debug/warning logs that don't reference the +token value. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 5: Fail closed when neither auth mechanism is configured + +The current default (`AUTH_API_KEY = "none"`, `AUTH_SERVER = None`) means a request with `?apikey=none` succeeds. Convenient in dev, dangerous in prod if either env var is dropped on deploy. + +**Files:** +- Modify: `optimizerapi/auth.py` +- Modify: `tests/test_auth.py` + +- [ ] **Step 1: Add a failing test** + +Append to `tests/test_auth.py`: + +```python +def test_default_apikey_value_is_rejected_in_production_mode(): + """When AUTH_API_KEY is the default 'none' and FLASK_ENV is 'production', + even matching the default value must be rejected.""" + with patch("optimizerapi.auth.AUTH_API_KEY", "none"), \ + patch("optimizerapi.auth.AUTH_SERVER", None), \ + patch("optimizerapi.auth.FLASK_ENV", "production"): + from optimizerapi import auth + result = auth.apikey_handler("none") + assert result is None + + +def test_default_apikey_value_is_accepted_in_development_mode(): + """In development mode (FLASK_ENV != 'production'), the legacy + behaviour of accepting AUTH_API_KEY='none' is preserved for ergonomics.""" + with patch("optimizerapi.auth.AUTH_API_KEY", "none"), \ + patch("optimizerapi.auth.AUTH_SERVER", None), \ + patch("optimizerapi.auth.FLASK_ENV", "development"): + from optimizerapi import auth + result = auth.apikey_handler("none") + assert result == {"scope": []} +``` + +- [ ] **Step 2: Run tests to confirm one fails** + +Run: +```bash +env/bin/python -m pytest tests/test_auth.py -v +``` +Expected: `test_default_apikey_value_is_rejected_in_production_mode` FAILS (current code accepts it). The other new test PASSES. + +- [ ] **Step 3: Edit `optimizerapi/auth.py`** + +In the module-level constants block, add `FLASK_ENV`: + +Find: +```python +AUTH_API_KEY = os.getenv("AUTH_API_KEY", "none") +AUTH_SERVER = os.getenv("AUTH_SERVER", None) +``` + +Replace with: +```python +AUTH_API_KEY = os.getenv("AUTH_API_KEY", "none") +AUTH_SERVER = os.getenv("AUTH_SERVER", None) +FLASK_ENV = os.getenv("FLASK_ENV", "development") + +_DEFAULT_APIKEY = "none" # sentinel: matches AUTH_API_KEY default; never accept in production +``` + +Update `apikey_handler`: + +```python +def apikey_handler(access_token: str) -> dict | None: + """Verify the API key passed by the client. + + Returns + ------- + dict + ``{"scope": []}`` if the supplied key matches the configured + ``AUTH_API_KEY``. + None + If the key is wrong, OIDC is configured (OIDC handles this path), + or the operator has not configured a real key in production. + """ + if AUTH_SERVER: + return None + # In production we refuse the unconfigured default value, even if it + # technically matches the request — operators should set a real key. + if FLASK_ENV == "production" and AUTH_API_KEY == _DEFAULT_APIKEY: + return None + expected = AUTH_API_KEY.encode("utf-8") + provided = (access_token or "").encode("utf-8") + if secrets.compare_digest(expected, provided): + return {"scope": []} + return None +``` + +- [ ] **Step 4: Run tests** + +```bash +env/bin/python -m pytest tests/test_auth.py -v +``` +Expected: all 5 PASS. + +```bash +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: 48 passed (46 + 2 new). + +- [ ] **Step 5: Commit** + +```bash +git add optimizerapi/auth.py tests/test_auth.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +fix(auth): refuse default AUTH_API_KEY='none' in production + +Previously, a server deployed with FLASK_ENV=production but no explicit +AUTH_API_KEY would still accept ?apikey=none. Now that's rejected; +operators must set a real key to enable the static-key path in prod. +Development mode keeps the legacy behaviour for ergonomics. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 6: Fix `== None` in `securepickle/secure.py` + +Pre-existing flake8 E711 violations. Trivial fix while we're in this file anyway. + +**Files:** +- Modify: `optimizerapi/securepickle/secure.py:6,8` + +- [ ] **Step 1: Edit** + +Replace: +```python +def get_crypto(key=None): + if key == None: + key = os.getenv("PICKLE_KEY", None) + if key == None: + print("No key found, generating new key") + key = Fernet.generate_key() + os.environ["PICKLE_KEY"] = key.decode("utf-8") + print("To reuse key for future server runs, set environment variable PICKLE_KEY=" + + os.environ["PICKLE_KEY"]) + return Fernet(key) +``` + +with (this also lands Task 7's changes — see step note): + +```python +def get_crypto(key=None): + if key is None: + key = os.getenv("PICKLE_KEY", None) + if key is None: + print("No key found, generating new key") + key = Fernet.generate_key() + os.environ["PICKLE_KEY"] = key.decode("utf-8") + print("To reuse key for future server runs, set environment variable PICKLE_KEY=" + + os.environ["PICKLE_KEY"]) + return Fernet(key) +``` + +**Only the two `== None` → `is None` changes here.** Task 7 finishes the rest of the cleanup. + +- [ ] **Step 2: Verify lint** + +```bash +env/bin/flake8 optimizerapi/securepickle/secure.py --max-line-length=127 +``` +Expected: only the warning-free output (no E711). May still show other style issues; those are Task 7's job. + +- [ ] **Step 3: Commit** + +```bash +git add optimizerapi/securepickle/secure.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +style(securepickle): use 'is None' instead of '== None' + +Resolves long-standing flake8 E711 violations. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 7: `get_crypto` no longer mutates `os.environ`, logs instead of prints + +**Files:** +- Modify: `optimizerapi/securepickle/secure.py` + +- [ ] **Step 1: Edit** + +Replace the whole file: + +```python +import logging +import os + +from cryptography.fernet import Fernet + +_LOG = logging.getLogger(__name__) + + +def get_crypto(key=None): + """Return a Fernet crypto handle for the given key. + + Resolution order: + 1. ``key`` argument (highest priority). + 2. ``PICKLE_KEY`` environment variable. + 3. Generate a new ephemeral key and warn the operator. + + The ephemeral path is intended for local development only. In any + environment where pickled state must survive a restart, ``PICKLE_KEY`` + must be set explicitly. This function no longer mutates ``os.environ`` + so caller environments are unchanged. + """ + if key is None: + key = os.getenv("PICKLE_KEY") + if key is None: + key = Fernet.generate_key() + _LOG.warning( + "No PICKLE_KEY set; generated an ephemeral key. Set " + "PICKLE_KEY=%s in the environment to persist pickled state " + "across restarts.", + key.decode("utf-8"), + ) + return Fernet(key) +``` + +Changes from the previous version: + +- `os.environ["PICKLE_KEY"] = ...` removed — `get_crypto` is now side-effect-free except for logging. +- `print` → `_LOG.warning` — visible in production logs, can be filtered/routed. +- The warning text is unified into a single message; before it printed two lines. +- Imports reordered (stdlib, then third-party). + +- [ ] **Step 2: Run the securepickle tests** + +```bash +env/bin/python -m pytest tests/test_securepickle.py -v +``` +Expected: PASS. + +- [ ] **Step 3: Run the full suite** + +```bash +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: 48 passed. + +- [ ] **Step 4: Verify behaviour by smoke-testing the server** + +```bash +timeout 5 env/bin/python -m optimizerapi.server 2>&1 | head -15 || true +``` +Expected: the `_LOG.warning` line about generating a key appears (this used to be two `print` lines). The Flask startup line appears below it. + +- [ ] **Step 5: Commit** + +```bash +git add optimizerapi/securepickle/secure.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +refactor(securepickle): use logging, drop os.environ mutation + +get_crypto previously printed to stdout and wrote PICKLE_KEY back into +os.environ as a side effect. Both are surprising for a "getter": +- print bypasses log routing/filtering +- env mutation makes the caller's environment opaque to itself + +Now it logs a single WARNING and leaves os.environ alone. The behaviour +contract (return a Fernet handle, optionally backed by a freshly +generated key) is unchanged. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 8: Final lint + test pass + +**Files:** none. + +- [ ] **Step 1: flake8** + +```bash +env/bin/flake8 optimizerapi tests --max-line-length=127 +``` +Expected: fewer violations than before. `optimizerapi/securepickle/secure.py` should be fully clean (no E711). `optimizerapi/auth.py` should be clean. `optimizerapi/securepickle/__init__.py` F401s (re-export warnings) and `tests/context.py` F401s remain (those are deferred to Phase 5). `tests/test_optimizer.py:12,15` E501s remain (Phase 5). + +- [ ] **Step 2: pytest** + +```bash +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: 48 passed. + +- [ ] **Step 3: No additional commit required.** + +--- + +## Out of scope (for Phase 1) + +- Flask 2 → 3 migration (requires Connexion 3 rewrite; see overview). +- ProcessOptimizer pinning to a release tag (deferred; need a release to exist first). +- `print` cleanup outside `auth.py` and `secure.py` — that's Phase 2. +- `typing` annotations beyond what fits naturally into the auth signatures (`dict | None`) — Phase 3. +- Pre-existing flake8 violations outside the touched files — Phase 5. + +## Acceptance criteria + +- `cryptography>=44` installed; `tests/test_securepickle.py` passes. +- `apikey_handler` uses `secrets.compare_digest`; a unit test asserts the call. +- `apikey_handler` refuses `AUTH_API_KEY="none"` in production. +- `token_info` no longer prints the access token or any other stdout output. +- `get_crypto` no longer mutates `os.environ` and no longer prints; emits one WARNING when generating an ephemeral key. +- `flake8 optimizerapi/auth.py optimizerapi/securepickle/secure.py` is clean. +- Full suite: 48 passed (43 prior + 5 new auth tests). diff --git a/docs/superpowers/plans/2026-05-18-audit-phase-2-logging.md b/docs/superpowers/plans/2026-05-18-audit-phase-2-logging.md new file mode 100644 index 0000000..31d9488 --- /dev/null +++ b/docs/superpowers/plans/2026-05-18-audit-phase-2-logging.md @@ -0,0 +1,417 @@ +# Audit Phase 2 — Logging Migration + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace every `print()` call in `optimizerapi/` with a properly-scoped `logging` call routed through a single `basicConfig` set up at server startup. Remove dead `# print(...)` comments. + +**Architecture:** Centralize log configuration in `optimizerapi/server.py` (run before any handlers fire). Each module uses `logging.getLogger(__name__)`, so loggers fall into the natural module hierarchy (`optimizerapi.server`, `optimizerapi.optimizer_handler`, etc.) and can be filtered/routed individually in production. + +**Tech Stack:** Standard library `logging`. + +**Audit reference:** §4 of the audit. `print` calls remain in `server.py`, `optimizer_handler.py`, and dead-code comments in `optimizer.py`. Phase 1 already removed prints from `auth.py` and `secure.py`. + +**Prereq:** Phase 1 complete (`auth.py` and `secure.py` are already on `logging`). + +--- + +## File Map + +- **Modify:** `optimizerapi/server.py` — add `logging.basicConfig`, convert 3 prints. +- **Modify:** `optimizerapi/optimizer_handler.py` — convert ~4 prints. +- **Modify:** `optimizerapi/optimizer.py` — delete 2 dead `# print(...)` comments. +- **Modify:** `tests/test_no_prints.py` (new) — enforce the no-print rule under `optimizerapi/`. + +--- + +### Task 1: Centralize log config in `server.py` + +**Files:** +- Modify: `optimizerapi/server.py` + +- [ ] **Step 1: Edit `optimizerapi/server.py`** + +The existing file starts with: + +```python +""" +Main server +""" +import os +import re +import connexion +from waitress import serve +from flask_cors import CORS +from .securepickle import get_crypto +``` + +Replace with: + +```python +""" +Main server +""" +import logging +import os +import re + +import connexion +from flask_cors import CORS +from waitress import serve + +from .securepickle import get_crypto + +_LOG = logging.getLogger(__name__) + + +def _configure_logging() -> None: + """Configure root logging once, before any handler is dispatched. + + Level defaults to INFO. Override with the LOG_LEVEL environment + variable (e.g. LOG_LEVEL=DEBUG for local debugging). + """ + level_name = os.getenv("LOG_LEVEL", "INFO").upper() + level = getattr(logging, level_name, logging.INFO) + logging.basicConfig( + level=level, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) +``` + +Then, inside the existing `if __name__ == "__main__":` block, add a call to `_configure_logging()` as the very first statement: + +Find: +```python +if __name__ == "__main__": + # Initialize crypto + get_crypto() +``` + +Replace with: +```python +if __name__ == "__main__": + _configure_logging() + # Initialize crypto + get_crypto() +``` + +Then replace the three CORS-related `print` lines. + +Find: +```python + print("CORS: " + cors_origin) + except re.error: + print("CORS: failed - the regex might be malformed.") + else: + print("CORS: disabled") +``` + +Replace with: +```python + _LOG.info("CORS: %s", cors_origin) + except re.error: + _LOG.warning("CORS: failed - the regex might be malformed.") + else: + _LOG.info("CORS: disabled") +``` + +- [ ] **Step 2: Smoke-test the server boots and emits log lines** + +```bash +LOG_LEVEL=INFO timeout 5 env/bin/python -m optimizerapi.server 2>&1 | head -15 || true +``` + +Expected: At least one line of the form `2026-... INFO optimizerapi.server: CORS: ...` appears. The Phase-1 `_LOG.warning("No PICKLE_KEY set...")` line from `securepickle.secure` is also routed through this config and appears with the right format. + +- [ ] **Step 3: Run the full suite** + +```bash +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: 48 passed (unchanged from end of Phase 1). + +- [ ] **Step 4: Commit** + +```bash +git add optimizerapi/server.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +refactor(server): central logging.basicConfig + CORS prints to logger + +LOG_LEVEL env var (default INFO) controls verbosity. All module loggers +inherit this single basicConfig, so per-module routing is now possible +in production. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 2: Convert prints in `optimizer_handler.py` + +**Files:** +- Modify: `optimizerapi/optimizer_handler.py` + +- [ ] **Step 1: Audit the existing prints** + +Run: +```bash +grep -n "print(" optimizerapi/optimizer_handler.py +``` + +Expected output (lines may shift slightly): +``` +25:print("Connecting to" + REDIS_URL) +67: print("Found existing job") +69: print(f"Creating new job (WORKER_TIMEOUT={WORKER_TIMEOUT})") +80: print(f"Client disconnected, cancelling job {job.id}") +``` + +- [ ] **Step 2: Edit the file** + +At the top, the existing import block looks like: + +```python +import os +import time +import json +import traceback +import hashlib +from rq import Queue +from rq.job import Job +``` + +Add `import logging` (alphabetical, with the other stdlib imports): + +```python +import hashlib +import json +import logging +import os +import time +import traceback + +from rq import Queue +from rq.job import Job +``` + +Right after the imports, add the module logger and replace the module-level connection print. + +Find: +```python +if "REDIS_URL" in os.environ: + REDIS_URL = os.environ["REDIS_URL"] +else: + REDIS_URL = "redis://localhost:6379" +print("Connecting to" + REDIS_URL) +redis = Redis.from_url(REDIS_URL) +``` + +Replace with: +```python +_LOG = logging.getLogger(__name__) + +REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379") +_LOG.info("Connecting to %s", REDIS_URL) +redis = Redis.from_url(REDIS_URL) +``` + +(The `os.environ.get(..., default)` collapse and the formatted-arg log are idiomatic; the latter also fixes the missing space between "to" and the URL in the previous line.) + +Then inside `run(body)`, replace: + +```python + try: + job = Job.fetch(job_id, connection=redis) + + print("Found existing job") + except NoSuchJobError: + print(f"Creating new job (WORKER_TIMEOUT={WORKER_TIMEOUT})") +``` + +with: + +```python + try: + job = Job.fetch(job_id, connection=redis) + _LOG.info("Found existing job %s", job_id) + except NoSuchJobError: + _LOG.info("Creating new job (WORKER_TIMEOUT=%s)", WORKER_TIMEOUT) +``` + +And the disconnect print: + +```python + print(f"Client disconnected, cancelling job {job.id}") +``` + +with: + +```python + _LOG.warning("Client disconnected, cancelling job %s", job.id) +``` + +- [ ] **Step 3: Verify no print remains** + +```bash +grep -n "print(" optimizerapi/optimizer_handler.py +``` +Expected: no output. + +- [ ] **Step 4: Run the full suite** + +```bash +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: 48 passed. + +- [ ] **Step 5: Commit** + +```bash +git add optimizerapi/optimizer_handler.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +refactor(handler): use logging in place of print + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 3: Remove dead `# print(...)` comments in `optimizer.py` + +**Files:** +- Modify: `optimizerapi/optimizer.py` + +- [ ] **Step 1: Find the dead comments** + +```bash +grep -n "# print(" optimizerapi/optimizer.py +``` +Expected output: +``` +: # print(str(response)) +: # print("IMAGE: " + str(pic_hash, "utf-8")) +``` + +- [ ] **Step 2: Delete those two lines.** + +Edit the file and remove exactly those two `# print(...)` lines. Do not touch surrounding code. + +- [ ] **Step 3: Confirm clean** + +```bash +grep -n "print(" optimizerapi/optimizer.py +``` +Expected: no output. + +- [ ] **Step 4: Run the full suite** + +```bash +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: 48 passed. + +- [ ] **Step 5: Commit** + +```bash +git add optimizerapi/optimizer.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +chore(optimizer): drop two dead # print debug comments + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 4: Enforce no-print rule with a unit test + +This locks in the migration so future regressions are caught. + +**Files:** +- Create: `tests/test_no_prints.py` + +- [ ] **Step 1: Write the test** + +Create `tests/test_no_prints.py`: + +```python +"""Lint test: no stray print() in production code. + +Tests in this repo configure their own logging; production code MUST +route everything through ``logging`` so operators can filter, level- +gate, and route messages in production. ``print`` calls bypass that. +""" + +import ast +import pathlib + +import pytest + +OPTIMIZERAPI_ROOT = pathlib.Path(__file__).parent.parent / "optimizerapi" + + +def _all_python_files() -> list[pathlib.Path]: + return [p for p in OPTIMIZERAPI_ROOT.rglob("*.py") if p.is_file()] + + +@pytest.mark.parametrize("path", _all_python_files(), ids=lambda p: str(p.relative_to(OPTIMIZERAPI_ROOT.parent))) +def test_no_print_calls(path): + """No `print()` call is allowed under optimizerapi/.""" + source = path.read_text(encoding="utf-8") + tree = ast.parse(source, filename=str(path)) + offenders = [ + node.lineno + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "print" + ] + assert not offenders, ( + f"{path}: found print() calls at line(s) {offenders}. " + "Use logging.getLogger(__name__) instead." + ) +``` + +- [ ] **Step 2: Run the test** + +```bash +env/bin/python -m pytest tests/test_no_prints.py -v +``` +Expected: PASS for every file under `optimizerapi/`. + +If anything fails, find the missed `print` and convert it. The error message will say which file and line. + +- [ ] **Step 3: Run the full suite** + +```bash +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: 48+N passed where N is the number of `.py` files under `optimizerapi/` (the parametrize generates one test per file). + +- [ ] **Step 4: Commit** + +```bash +git add tests/test_no_prints.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +test: enforce no print() under optimizerapi/ + +Locks in the logging migration with an AST-based lint test so future +regressions fail loudly. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Acceptance criteria + +- `grep -rn "print(" optimizerapi/` returns nothing. +- `tests/test_no_prints.py` passes for every module. +- Server still boots; log lines appear under the new `2026-... LEVEL name: message` format. +- Full pytest run is green. diff --git a/docs/superpowers/plans/2026-05-18-audit-phase-3-boundary-types.md b/docs/superpowers/plans/2026-05-18-audit-phase-3-boundary-types.md new file mode 100644 index 0000000..dee9670 --- /dev/null +++ b/docs/superpowers/plans/2026-05-18-audit-phase-3-boundary-types.md @@ -0,0 +1,648 @@ +# Audit Phase 3 — Boundary Types & Static Checking + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add `TypedDict` definitions for the request/response/cache shapes, annotate every public function signature in `optimizerapi/`, and add `mypy` to the dev workflow as a CI-grade type check. Acts as a safety net before the Phase 4 refactor. + +**Architecture:** One new module — `optimizerapi/types.py` — owns every boundary `TypedDict`. Modules import the relevant types and use them in signatures only (the runtime structures stay plain dicts, since the request validation lives in Connexion + OpenAPI, not Python). `mypy` is configured in `pyproject.toml` and runs against `optimizerapi/` only (tests are not type-checked yet). + +**Tech Stack:** Python 3.13 stdlib `typing` (`TypedDict`, `Literal`, `NotRequired`, `Self`), `mypy`. + +**Audit reference:** §2 of the audit. `body: dict` is the only thing keeping the contract honest today; adding types catches mistakes at edit time and makes Phase 4 safer. + +**Prereq:** Phase 1 & 2 complete (clean baseline of 48+ tests, no `print` in `optimizerapi/`). + +--- + +## File Map + +- **Create:** `optimizerapi/types.py` — all `TypedDict` definitions for request, response, cache. +- **Modify:** `pyproject.toml` — add `mypy>=1.13` to dev deps; add `[tool.mypy]` section. +- **Modify:** `optimizerapi/pickled_state.py` — annotate signatures using the new types. +- **Modify:** `optimizerapi/optimizer_handler.py` — annotate signatures. +- **Modify:** `optimizerapi/optimizer.py` — annotate `run` and `process_result` signatures. +- **Modify:** `AGENTS.md`/`CLAUDE.md` — document `mypy` in the dev loop. + +--- + +### Task 1: Create `optimizerapi/types.py` + +**Files:** +- Create: `optimizerapi/types.py` +- Create: `tests/test_types.py` — sanity check the module imports and the types are usable. + +- [ ] **Step 1: Write the failing test** + +Create `tests/test_types.py`: + +```python +"""Sanity checks for the boundary type definitions.""" + +from optimizerapi.types import ( + CachePayload, + Constraint, + DataPoint, + Dimension, + Extras, + OptimizerConfig, + Plot, + RequestBody, + ResponseEnvelope, + ResponseResult, + ResponseResultExtras, +) + + +def test_request_body_accepts_realistic_request(): + """A typical request body matches the RequestBody shape at runtime. + + TypedDicts don't enforce at runtime; we use isinstance(d, dict) as a + proxy and let mypy catch shape mismatches statically. + """ + body: RequestBody = { + "data": [{"xi": [50, 833, 150, 60, "Whipped cream"], "yi": [-3, -6]}], + "optimizerConfig": { + "baseEstimator": "GP", + "acqFunc": "EI", + "initialPoints": 3, + "kappa": 1.96, + "xi": 2, + "space": [ + {"type": "continuous", "name": "Sugar", "from": 0, "to": 100}, + {"type": "category", "name": "Finish", + "categories": ["None", "Whipped cream"]}, + ], + "constraints": [], + }, + "extras": {"graphFormat": "json", "includeModel": "true"}, + } + assert isinstance(body, dict) + assert body["data"][0]["xi"][-1] == "Whipped cream" + + +def test_response_envelope_shape(): + response: ResponseEnvelope = { + "plots": [{"id": "single_0_0", "plot": "{}"}], + "result": { + "next": [[1, 2, 3]], + "models": [], + "pickled": "", + "extras": {"pickledUsed": False}, + }, + } + assert response["result"]["extras"]["pickledUsed"] is False + + +def test_cache_payload_shape(): + payload: CachePayload = { + "fingerprint": "a" * 64, + "result": "opaque", + "next": [[1, 2]], + "optimizer": "opaque", + } + assert payload["fingerprint"] == "a" * 64 +``` + +- [ ] **Step 2: Run test to verify it fails** + +```bash +env/bin/python -m pytest tests/test_types.py -v +``` +Expected: FAIL with `ModuleNotFoundError: No module named 'optimizerapi.types'`. + +- [ ] **Step 3: Create `optimizerapi/types.py`** + +```python +"""Boundary types for the optimizer API. + +These ``TypedDict``s describe the shape of the request body, response +envelope, and internal cache payload. They are *static* contracts — +they exist to be checked by ``mypy`` and to document the API to +readers. Runtime validation of incoming requests lives in Connexion + +OpenAPI; we do not duplicate that here. + +Note on the ``Dimension`` field name ``from``: + ``from`` is a Python keyword, so the class-based ``TypedDict`` + syntax cannot declare it. We use the alternative call-based + syntax to define ``Dimension``. +""" + +from typing import Any, Literal, NotRequired, TypedDict + + +# --- Request types --------------------------------------------------------- + +Dimension = TypedDict( + "Dimension", + { + "type": Literal["continuous", "discrete", "category"], + "name": str, + "from": NotRequired[float], + "to": NotRequired[float], + "categories": NotRequired[list[str]], + }, +) + + +class Constraint(TypedDict): + type: Literal["sum"] + dimensions: list[int] + value: float + + +class OptimizerConfig(TypedDict): + baseEstimator: str + acqFunc: str + initialPoints: int + kappa: float + xi: float + space: list[Dimension] + constraints: NotRequired[list[Constraint]] + + +class DataPoint(TypedDict): + xi: list[str | float] + yi: list[float] + + +GraphFormat = Literal["png", "json", "none"] +GraphName = Literal["objective", "convergence", "pareto", "single"] +ObjectivePars = Literal["result", "expected_minimum"] + + +class Extras(TypedDict, total=False): + objectivePars: ObjectivePars + graphFormat: GraphFormat + experimentSuggestionCount: int + maxQuality: int + graphs: list[GraphName] + selectedPoint: list[str | float] + pickled: str + # NB: stringly-typed; see audit §3 for the rationale for not migrating yet. + includeModel: Literal["true", "false"] + useActualMeasurementHistogram: Literal["true", "false"] + + +class RequestBody(TypedDict): + data: list[DataPoint] + optimizerConfig: OptimizerConfig + extras: NotRequired[Extras] + + +# --- Response types -------------------------------------------------------- + +class Plot(TypedDict): + id: str + plot: str # base64 PNG OR JSON-serialised plot data + + +class ResponseResultExtras(TypedDict, total=False): + pickledUsed: bool + parameters: dict[str, Any] + libraries: list[str] + pythonVersion: str + apiVersion: str + timeOfExecution: str + + +class ResponseModel(TypedDict, total=False): + expected_minimum: list[list[str | float]] + extras: dict[str, Any] + + +class ResponseResult(TypedDict, total=False): + next: list[list[str | float]] + models: list[ResponseModel] + pickled: str + extras: ResponseResultExtras + expected_minimum: list[list[str | float] | float] + + +class ResponseEnvelope(TypedDict): + plots: list[Plot] + result: ResponseResult + + +# --- Cache payload (pickled_state) ----------------------------------------- + +class CachePayload(TypedDict): + fingerprint: str + # ProcessOptimizer's OptimizerResult or list thereof — kept opaque + # so types.py does not depend on the optimizer library. + result: Any + next: list[list[str | float]] + # ProcessOptimizer.Optimizer — also opaque. + optimizer: Any +``` + +- [ ] **Step 4: Run tests** + +```bash +env/bin/python -m pytest tests/test_types.py -v +``` +Expected: PASS, 3 tests. + +- [ ] **Step 5: Run the full suite** + +```bash +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: 51+N passed (48 + 3 new). (`N` is the no-print parametrize tests from Phase 2.) + +- [ ] **Step 6: Commit** + +```bash +git add optimizerapi/types.py tests/test_types.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +feat(types): add TypedDicts for request, response, and cache payload + +Static-only contracts for mypy; runtime validation continues to live in +Connexion + OpenAPI. The new types.py module is the single source of +truth for what the boundary shapes look like in Python. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 2: Install and configure `mypy` + +**Files:** +- Modify: `pyproject.toml` + +- [ ] **Step 1: Add `mypy` to dev deps** + +Find: +``` +dev = ["pytest==8.3.3", "pytest-watch==4.2.0", "flake8>=7"] +``` + +Replace with: +``` +dev = ["pytest==8.3.3", "pytest-watch==4.2.0", "flake8>=7", "mypy>=1.13"] +``` + +- [ ] **Step 2: Add `[tool.mypy]` configuration** + +Append to the end of `pyproject.toml`: + +```toml +[tool.mypy] +files = ["optimizerapi"] +python_version = "3.13" +strict = false +# Start permissive; we tighten as Phase 4 lands. +warn_unused_ignores = true +warn_redundant_casts = true +warn_return_any = true +disallow_any_unimported = false +check_untyped_defs = true +# Third-party libraries that don't ship type stubs. +ignore_missing_imports = true +``` + +- [ ] **Step 3: Install** + +```bash +env/bin/pip install -e ".[dev]" 2>&1 | tail -3 +``` +Expected: `Successfully installed ... mypy-1.x.x ...`. + +- [ ] **Step 4: Capture the baseline** + +```bash +env/bin/mypy optimizerapi 2>&1 | tail -20 +``` + +Don't try to fix things yet — this is the baseline that subsequent tasks will whittle down. The expected number of errors is "moderate" (most signatures are untyped today). Take note of the count for reference. + +- [ ] **Step 5: Commit** + +```bash +git add pyproject.toml +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +build: add mypy to dev deps with permissive baseline config + +Subsequent tasks tighten signatures module-by-module. Permissive mode +keeps the gate green during the migration. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 3: Annotate `pickled_state.py` + +**Files:** +- Modify: `optimizerapi/pickled_state.py` + +- [ ] **Step 1: Edit** + +The current functions are untyped. Add precise signatures using the new types. + +Find: +```python +def compute_fingerprint(data, optimizer_config): + """Return sha256 hex of the canonical-JSON of (data, optimizerConfig). +``` + +Replace with: +```python +def compute_fingerprint( + data: list["DataPoint"], + optimizer_config: "OptimizerConfig", +) -> str: + """Return sha256 hex of the canonical-JSON of (data, optimizerConfig). +``` + +Find: +```python +def pack(*, result, next_points, optimizer, fingerprint, crypto): + """Encrypt a pickled cache payload for the given fingerprint.""" +``` + +Replace with: +```python +def pack( + *, + result: object, + next_points: list[list[str | float]], + optimizer: object, + fingerprint: str, + crypto: "Fernet", +) -> str: + """Encrypt a pickled cache payload for the given fingerprint.""" +``` + +Find: +```python +def unpack_if_valid(blob, *, expected_fingerprint, crypto): + """Decrypt and validate a pickled cache payload. +``` + +Replace with: +```python +def unpack_if_valid( + blob: str, + *, + expected_fingerprint: str, + crypto: "Fernet", +) -> "CachePayload | None": + """Decrypt and validate a pickled cache payload. +``` + +Add the imports at the top of the file (after the existing `import logging`): + +```python +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from cryptography.fernet import Fernet + + from .types import CachePayload, DataPoint, OptimizerConfig +``` + +Using `TYPE_CHECKING` keeps import-time light and avoids circular-import risk during the Phase 4 refactor when `optimizer.py` will also start importing from `types.py`. + +- [ ] **Step 2: Run mypy on the module** + +```bash +env/bin/mypy optimizerapi/pickled_state.py 2>&1 | tail -10 +``` +Expected: 0 errors (or the same module-local errors mypy reported in the baseline minus the ones the new annotations resolved). + +- [ ] **Step 3: Run the full suite** + +```bash +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: unchanged (tests don't change behavior). + +- [ ] **Step 4: Commit** + +```bash +git add optimizerapi/pickled_state.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +types: annotate pickled_state public surface + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 4: Annotate `optimizer_handler.py` + +**Files:** +- Modify: `optimizerapi/optimizer_handler.py` + +- [ ] **Step 1: Edit signatures** + +Find: +```python +def run(body) -> dict: +``` + +Replace with: +```python +def run(body: "RequestBody") -> "ResponseEnvelope | tuple[dict[str, str], int]": +``` + +The return-shape is the existing `dict | (dict, status_code)` mix that `do_run_work` produces. (Phase 5 normalizes this.) + +Find: +```python +def do_run_work(body) -> dict: + """ "Handle the run request""" +``` + +Replace with: +```python +def do_run_work(body: "RequestBody") -> "ResponseEnvelope | tuple[dict[str, str], int]": + """Handle the run request.""" +``` + +Add at the top of the file: +```python +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .types import RequestBody, ResponseEnvelope +``` + +- [ ] **Step 2: Run mypy on the module** + +```bash +env/bin/mypy optimizerapi/optimizer_handler.py 2>&1 | tail -10 +``` +Expected: 0 new errors. + +- [ ] **Step 3: Run full suite** + +```bash +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: unchanged. + +- [ ] **Step 4: Commit** + +```bash +git add optimizerapi/optimizer_handler.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +types: annotate optimizer_handler.run and do_run_work + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 5: Annotate `optimizer.py` public entry points + +We only annotate `run` and `process_result` here. Helpers stay untyped for now — Phase 4 splits them into smaller pieces and annotates them as it goes. + +**Files:** +- Modify: `optimizerapi/optimizer.py` + +- [ ] **Step 1: Edit `run` signature** + +Find: +```python +def run(body) -> dict: + """ "Handle the run request""" +``` + +Replace with: +```python +def run(body: "RequestBody") -> dict: + """Handle the run request. + + Returns the response envelope as a plain ``dict`` — the json_tricks + round-trip at the bottom of the function dropping NumPy types is + why we cannot return ``ResponseEnvelope`` directly without an + explicit cast. + """ +``` + +- [ ] **Step 2: Edit `process_result` signature** + +Find: +```python +def process_result(result, optimizer, dimensions, cfg, extras, data, space, + *, request_fingerprint, pickled_used): +``` + +Replace with: +```python +def process_result( + result: object, + optimizer: object, + dimensions: list[str], + cfg: "OptimizerConfig", + extras: "Extras", + data: list[tuple[list[str | float], list[float]]], + space: list, + *, + request_fingerprint: str, + pickled_used: bool, +) -> dict: +``` + +- [ ] **Step 3: Add the imports** + +At the top of the file, near the existing imports, add: + +```python +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .types import Extras, OptimizerConfig, RequestBody +``` + +- [ ] **Step 4: Run mypy and full suite** + +```bash +env/bin/mypy optimizerapi/optimizer.py 2>&1 | tail -10 +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: mypy may surface in-body issues; ignore them for now (they belong to Phase 4). The `def`-line types are what matter here. Pytest is unchanged. + +- [ ] **Step 5: Commit** + +```bash +git add optimizerapi/optimizer.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +types: annotate optimizer.run and process_result signatures + +Body-level annotations and helper functions land in Phase 4 along with +the structural refactor that splits process_result into focused units. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 6: Document the type-check command + +**Files:** +- Modify: `AGENTS.md` +- Modify: `CLAUDE.md` + +- [ ] **Step 1: Add a line to `AGENTS.md`** + +Open `AGENTS.md`. After the line: +``` +- Lint locally with `flake8 . --max-line-length=127` (matches CI workflow). +``` + +Insert (alphabetical placement): +``` +- Type-check locally with `mypy optimizerapi` (matches CI workflow); see `[tool.mypy]` in `pyproject.toml`. +``` + +- [ ] **Step 2: Add the same to `CLAUDE.md`** + +In the `## Commands` block of `CLAUDE.md`, after the flake8 line, add: + +```bash +# Type-check (matches CI) +mypy optimizerapi +``` + +- [ ] **Step 3: Commit** + +```bash +git add AGENTS.md CLAUDE.md +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +docs: document mypy in dev-loop instructions + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +## Acceptance criteria + +- `optimizerapi/types.py` exists and exports the documented TypedDicts. +- `tests/test_types.py` passes (3 sanity-check assertions). +- `mypy optimizerapi` runs (it does not need to be error-free yet; Phase 4 cleans up the body-level issues). +- Public function signatures in `pickled_state.py`, `optimizer_handler.py`, and `optimizer.py` have types referencing the new module. +- Full pytest run is green. +- `AGENTS.md` and `CLAUDE.md` mention `mypy optimizerapi`. + +## Out of scope + +- Annotating every helper inside `optimizer.py` (Phase 4). +- Making `mypy strict = true` (deferred until after Phase 4). +- Type-checking the test suite (low ROI; tests are dynamic by nature). +- Switching to `pyright` (mypy chosen because it's the most familiar tooling). diff --git a/docs/superpowers/plans/2026-05-18-audit-phase-4-refactor-process-result.md b/docs/superpowers/plans/2026-05-18-audit-phase-4-refactor-process-result.md new file mode 100644 index 0000000..d381bce --- /dev/null +++ b/docs/superpowers/plans/2026-05-18-audit-phase-4-refactor-process-result.md @@ -0,0 +1,1050 @@ +# Audit Phase 4 — Refactor `process_result` + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Split `optimizer.py:process_result` (~250 lines, 4-way branching) into focused helpers. Pull plot-rendering code into a new `plot_emitters.py` module. Eliminate the three near-duplicate JSON single-plot loops via a single helper that takes a `prefix` argument. Behavior is unchanged — the equivalence test from the pareto redesign is the binding correctness gate. + +**Architecture:** Two refactors stacked: + +1. **Internal extraction in `optimizer.py`:** small helpers (`_parse_extras`, `_compute_next_experiments`, `_flatten_expected_minima`, `_set_expected_minimum`) live in the same file as private functions. +2. **External extraction:** the plot-emission code moves to `optimizerapi/plot_emitters.py`. The three duplicated single-plot loops collapse into `emit_json_single_plots(plots, model, *, prefix, selected_point)`. PNG emission moves to `emit_png_plots`. + +After both steps, `process_result` becomes a ~60-line dispatcher. + +**Tech Stack:** Python 3.13, mypy from Phase 3 catches refactor mistakes, the equivalence test (`test_equivalence_with_and_without_pickled_multi_objective`) catches behavior drift. + +**Audit reference:** §1 of the audit. The `process_result` function is the single largest source of structural debt; the refactor pays back the typing investment from Phase 3. + +**Prereq:** Phases 1, 2, 3 complete. `mypy optimizerapi/optimizer.py` runs (errors permitted; this phase reduces them). + +--- + +## File Map + +- **Modify:** `optimizerapi/optimizer.py` — extract helpers, shrink `process_result`. +- **Create:** `optimizerapi/plot_emitters.py` — `emit_png_plots`, `emit_json_single_plots`, `emit_pareto_data`. +- **Create:** `tests/test_plot_emitters.py` — focused unit tests for the new module. + +### Behavioral safety net + +Every task ends with `python -m pytest 2>&1 | tail -3`. The CRITICAL invariant is that `test_equivalence_with_and_without_pickled_multi_objective` passes after every commit — that's the binding contract from the pareto redesign. If it fails at any point, STOP and report BLOCKED with the divergence details. Do NOT modify the test to make it pass. + +--- + +### Task 1: Extract `_parse_extras` and the PNG-warning move + +The top of `process_result` reads extras keys and emits the PNG + selectedPoint warning. Pull both into a private helper that returns a small dataclass. + +**Files:** +- Modify: `optimizerapi/optimizer.py` + +- [ ] **Step 1: Add the dataclass and helper near the top of `optimizer.py`** + +Add at the top of the file (after the existing imports + version-info helpers, but before `process_result`): + +```python +from dataclasses import dataclass + + +@dataclass(frozen=True) +class _ParsedExtras: + graph_format: str + max_quality: int + graphs_to_return: list[str] + objective_pars: str + include_model: bool + selected_point: list[str | float] | None + experiment_suggestion_count: int + + +def _parse_bool(value: object, default: bool = True) -> bool: + """Coerce extras' stringly-typed boolean fields. + + Accepts the literals "true" / "false" (case-insensitive), real bools, + and JSON-style ``true``/``false``. Anything else falls back to *default*. + """ + if isinstance(value, bool): + return value + if value is None: + return default + text = str(value).strip().lower() + if text in ("true", "1", "yes"): + return True + if text in ("false", "0", "no"): + return False + return default + + +def _parse_extras(extras: "Extras", logger: "logging.Logger") -> _ParsedExtras: + """Read the request ``extras`` block into a typed view. + + Emits the PNG + selectedPoint warning here so callers don't need to + duplicate the check. + """ + graph_format = extras.get("graphFormat", "png") + selected_point = extras.get("selectedPoint") + if selected_point is not None and graph_format != "json": + logger.warning( + "selectedPoint ignored on png path (graphFormat=%s)", graph_format + ) + return _ParsedExtras( + graph_format=graph_format, + max_quality=int(extras.get("maxQuality", 5)), + graphs_to_return=extras.get( + "graphs", ["objective", "convergence", "pareto", "single"] + ), + objective_pars=extras.get("objectivePars", "result"), + include_model=_parse_bool(extras.get("includeModel", "true")), + selected_point=selected_point, + experiment_suggestion_count=int(extras.get("experimentSuggestionCount", 1)), + ) +``` + +- [ ] **Step 2: Use it inside `process_result`** + +Inside `process_result`, replace the block that currently reads the extras keys: + +```python + graph_format = extras.get("graphFormat", "png") + max_quality = int(extras.get("maxQuality", "5")) + graphs_to_return = extras.get( + "graphs", ["objective", "convergence", "pareto", "single"] + ) + + objective_pars = extras.get("objectivePars", "result") + + pickle_model = json.loads(extras.get("includeModel", "true").lower()) + selected_point = extras.get("selectedPoint") + if selected_point is not None and graph_format != "json": + logging.getLogger(__name__).warning( + "selectedPoint ignored on png path (graphFormat=%s)", graph_format + ) + + experiment_suggestion_count = 1 + if "experimentSuggestionCount" in extras: + experiment_suggestion_count = extras["experimentSuggestionCount"] +``` + +with: + +```python + parsed = _parse_extras(extras, logging.getLogger(__name__)) + graph_format = parsed.graph_format + max_quality = parsed.max_quality + graphs_to_return = parsed.graphs_to_return + objective_pars = parsed.objective_pars + pickle_model = parsed.include_model + selected_point = parsed.selected_point + experiment_suggestion_count = parsed.experiment_suggestion_count +``` + +The reassignment to local names preserves the existing body's references — minimising the diff and keeping the refactor incremental. Future tasks can drop the local aliases as the body shrinks. + +- [ ] **Step 3: Run the equivalence test** + +```bash +env/bin/python -m pytest tests/test_optimizer.py::test_equivalence_with_and_without_pickled_multi_objective -v +``` +Expected: PASS. + +- [ ] **Step 4: Run the full suite** + +```bash +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: unchanged from end of Phase 3. + +- [ ] **Step 5: Commit** + +```bash +git add optimizerapi/optimizer.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +refactor(optimizer): extract _parse_extras + _parse_bool helper + +Single source of truth for extras parsing (including the PNG+selectedPoint +warning). The local aliases preserve the rest of the function body for now; +subsequent tasks drop them as the body shrinks. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 2: Extract `_compute_next_experiments` and `_flatten_expected_minima` + +These two are small but ugly. Pulling them out reduces noise in `process_result`. + +**Files:** +- Modify: `optimizerapi/optimizer.py` + +- [ ] **Step 1: Add the helpers** + +Below `_parse_extras`, add: + +```python +def _compute_next_experiments( + optimizer: object, + cfg: "OptimizerConfig", + n_points: int, +) -> list[list[str | float]]: + """Ask the optimizer for the next N experiments, normalising the shape. + + ``optimizer.ask`` can return either a single experiment (flat list) or + a list of experiments. We always return a list of lists. + """ + constraints = cfg.get("constraints", []) + if constraints: + next_exp = optimizer.ask(n_points=n_points, strategy="cl_min") + else: + next_exp = optimizer.ask(n_points=n_points) + if next_exp and not any(isinstance(x, list) for x in next_exp): + next_exp = [next_exp] + return round_to_length_scales(next_exp, optimizer.space) + + +def _flatten_expected_minima(models: list[dict]) -> None: + """In-place flatten of nested ``expected_minimum`` entries on each model. + + The pre-flatten shape from ``process_model`` can be a list of mixed + scalars and lists; the response contract is a single flat list inside + a one-element outer list. This function normalises that. + """ + for model in models: + flat = [] + for x in model["expected_minimum"]: + if isinstance(x, list): + flat.extend(x) + else: + flat.append(x) + model["expected_minimum"] = [flat] +``` + +- [ ] **Step 2: Use them inside `process_result`** + +Replace the existing next-experiment block: + +```python + if "constraints" in cfg and len(cfg["constraints"]) > 0: + next_exp = optimizer.ask( + n_points=experiment_suggestion_count, strategy="cl_min" + ) + else: + next_exp = optimizer.ask(n_points=experiment_suggestion_count) + if len(next_exp) > 0 and not any(isinstance(x, list) for x in next_exp): + next_exp = [next_exp] + result_details["next"] = round_to_length_scales(next_exp, optimizer.space) +``` + +with: + +```python + result_details["next"] = _compute_next_experiments( + optimizer, cfg, experiment_suggestion_count + ) +``` + +Replace the trailing flatten block (currently 3 levels of nested list comprehension): + +```python + org_models = response["result"]["models"] + for model in org_models: + # Flatten expected minimum entries + model["expected_minimum"] = [ + [ + item + for sublist in [ + x if isinstance(x, list) else [x] for x in model["expected_minimum"] + ] + for item in sublist + ] + ] + return response +``` + +with: + +```python + _flatten_expected_minima(response["result"]["models"]) + return response +``` + +- [ ] **Step 3: Run the equivalence test + full suite** + +```bash +env/bin/python -m pytest tests/test_optimizer.py::test_equivalence_with_and_without_pickled_multi_objective -v +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: equivalence PASS; full suite unchanged. + +- [ ] **Step 4: Commit** + +```bash +git add optimizerapi/optimizer.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +refactor(optimizer): extract _compute_next_experiments, _flatten_expected_minima + +Both were small but inline-ugly: the next-experiment shape normalisation +and the 3-deep list comprehension for flattening models. Each now reads +as a single function call at the call site. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 3: Create `plot_emitters.py` with `emit_json_single_plots` + +The biggest DRY win. The same single-plot emission code is currently inlined three times in `process_result` (single-obj JSON, multi-obj objective_1, multi-obj objective_2), each with a different prefix. We replace all three with one helper. + +**Files:** +- Create: `optimizerapi/plot_emitters.py` +- Create: `tests/test_plot_emitters.py` + +- [ ] **Step 1: Write the failing test** + +Create `tests/test_plot_emitters.py`: + +```python +"""Tests for the plot-emitters module.""" + +import json + +import numpy +import pytest + +from optimizerapi.plot_emitters import emit_json_single_plots + + +class _FakePlotData: + """Mimics ProcessOptimizer's get_Brownie_Bee_1d_plot return value.""" + + @staticmethod + def make(n_dims: int): + # Each dimension yields a 4-element series; the last entry is a + # histogram (mean, std) pair. + dims = [[[1.0, 2.0, 3.0, 4.0] for _ in range(4)] for _ in range(n_dims)] + histogram = (numpy.array([42.0]), numpy.array([1.5])) + return dims + [histogram] + + +@pytest.fixture +def fake_brownie_1d(monkeypatch): + """Patch _get_brownie_bee_1d_plot_safe so this test doesn't need ProcessOptimizer.""" + def _stub(result, x_eval=None): + return _FakePlotData.make(n_dims=3) + monkeypatch.setattr( + "optimizerapi.plot_emitters._get_brownie_bee_1d_plot_safe", _stub + ) + + +def test_emit_json_single_plots_writes_one_entry_per_dim_plus_histogram(fake_brownie_1d): + plots: list = [] + emit_json_single_plots(plots, result=object(), prefix="single_0", selected_point=None) + ids = [p["id"] for p in plots] + assert ids == ["single_0_0", "single_0_1", "single_0_2", "single_0_3"] + # Last entry is the histogram payload. + histogram_payload = json.loads(plots[-1]["plot"]) + assert "histogram" in histogram_payload + assert histogram_payload["histogram"]["mean"] == 42.0 + assert histogram_payload["histogram"]["std"] == 1.5 + + +def test_emit_json_single_plots_supports_different_prefixes(fake_brownie_1d): + plots: list = [] + emit_json_single_plots(plots, result=object(), prefix="objective_2", selected_point=None) + ids = [p["id"] for p in plots] + assert ids[0] == "objective_2_0" + assert ids[-1] == "objective_2_3" + + +def test_emit_json_single_plots_passes_selected_point_through(monkeypatch): + captured = {} + + def _stub(result, x_eval=None): + captured["x_eval"] = x_eval + return _FakePlotData.make(n_dims=2) + + monkeypatch.setattr( + "optimizerapi.plot_emitters._get_brownie_bee_1d_plot_safe", _stub + ) + + plots: list = [] + sp = [50, 833, "Whipped cream"] + emit_json_single_plots(plots, result=object(), prefix="single_0", selected_point=sp) + assert captured["x_eval"] == sp +``` + +- [ ] **Step 2: Run test to verify it fails** + +```bash +env/bin/python -m pytest tests/test_plot_emitters.py -v +``` +Expected: FAIL — `ModuleNotFoundError: No module named 'optimizerapi.plot_emitters'`. + +- [ ] **Step 3: Create the module** + +Create `optimizerapi/plot_emitters.py`: + +```python +"""Plot emission for the optimizer API response. + +Two output formats are supported: +- ``emit_png_plots`` — base64 PNG plots, used by the legacy UI. +- ``emit_json_single_plots`` / ``emit_pareto_data`` — structured JSON + plot data, used by the React-based UI. +""" + +from typing import TYPE_CHECKING, Any + +import json_tricks +import numpy + +# Imported from optimizer.py — the private helper that wraps a ProcessOptimizer +# bug with categorical x_eval values. Phase 5 considers moving it here too. +from .optimizer import _get_brownie_bee_1d_plot_safe + +if TYPE_CHECKING: + from .types import Plot + + +def emit_json_single_plots( + plots: list["Plot"], + *, + result: Any, + prefix: str, + selected_point: list[str | float] | None, +) -> None: + """Append one plot entry per dimension plus a final histogram entry. + + Parameters + ---------- + plots: + The mutable list of plot entries to append to. + result: + A ProcessOptimizer ``OptimizerResult``-like object. + prefix: + Plot-id prefix. The emitted ids are ``{prefix}_0``, ``{prefix}_1``, + ..., with the final id being ``{prefix}_`` for the histogram. + selected_point: + X-space coordinates to highlight, or ``None`` to use the default. + """ + one_d_data = _get_brownie_bee_1d_plot_safe(result, x_eval=selected_point) + histogram_entry = one_d_data[-1] + for i, dim_data in enumerate(one_d_data[:-1]): + plots.append( + {"id": f"{prefix}_{i}", "plot": json_tricks.dumps({"data": dim_data})} + ) + plots.append( + { + "id": f"{prefix}_{len(one_d_data) - 1}", + "plot": json_tricks.dumps( + { + "histogram": { + "mean": float(numpy.ravel(histogram_entry[0])[0]), + "std": float(numpy.ravel(histogram_entry[1])[0]), + } + } + ), + } + ) +``` + +- [ ] **Step 4: Run tests** + +```bash +env/bin/python -m pytest tests/test_plot_emitters.py -v +``` +Expected: 3 PASS. + +- [ ] **Step 5: Replace the three duplicated loops in `process_result`** + +In `optimizer.py:process_result`, find the single-objective JSON loop: + +```python + elif graph_format == "json": + for idx, model in enumerate(result): + if "single" in graphs_to_return and optimizer.n_objectives != 2: + obj1_1D_data = _get_brownie_bee_1d_plot_safe(result[idx], x_eval=selected_point) + histogram_entry = obj1_1D_data[-1] + for i, dim_data in enumerate(obj1_1D_data[:-1]): + plots.append( + { + "id": f"single_{idx}_{i}", + "plot": json_tricks.dumps({"data": dim_data}), + } + ) + plots.append( + { + "id": f"single_{idx}_{len(obj1_1D_data) - 1}", + "plot": json_tricks.dumps( + { + "histogram": { + "mean": float( + numpy.ravel(histogram_entry[0])[0] + ), + "std": float( + numpy.ravel(histogram_entry[1])[0] + ), + } + } + ), + } + ) + if "convergence" in graphs_to_return: + pass + # skip plotting convergence data in json format + + if "objective" in graphs_to_return: + pass + # skip plotting objective data in json format +``` + +Replace with: + +```python + elif graph_format == "json": + for idx, model in enumerate(result): + if "single" in graphs_to_return and optimizer.n_objectives != 2: + emit_json_single_plots( + plots, + result=result[idx], + prefix=f"single_{idx}", + selected_point=selected_point, + ) + # convergence and objective plots are PNG-only; nothing to emit here. +``` + +Then find the multi-objective objective_1 loop (a near-copy): + +```python + if optimizer.n_objectives == 2 and "single" in graphs_to_return: + obj1_1D_data = _get_brownie_bee_1d_plot_safe(result[0], x_eval=selected_point) + histogram_entry = obj1_1D_data[-1] + for i, dim_data in enumerate(obj1_1D_data[:-1]): + plots.append( + { + "id": f"objective_1_{i}", + "plot": json_tricks.dumps({"data": dim_data}), + } + ) + plots.append( + { + "id": f"objective_1_{len(obj1_1D_data) - 1}", + "plot": json_tricks.dumps( + { + "histogram": { + "mean": float(numpy.ravel(histogram_entry[0])[0]), + "std": float(numpy.ravel(histogram_entry[1])[0]), + } + } + ), + } + ) + + obj2_1D_data = _get_brownie_bee_1d_plot_safe(result[1], x_eval=selected_point) + histogram_entry2 = obj2_1D_data[-1] + for i, dim_data in enumerate(obj2_1D_data[:-1]): + plots.append( + { + "id": f"objective_2_{i}", + "plot": json_tricks.dumps({"data": dim_data}), + } + ) + plots.append( + { + "id": f"objective_2_{len(obj2_1D_data) - 1}", + "plot": json_tricks.dumps( + { + "histogram": { + "mean": float(numpy.ravel(histogram_entry2[0])[0]), + "std": float(numpy.ravel(histogram_entry2[1])[0]), + } + } + ), + } + ) +``` + +Replace with: + +```python + if optimizer.n_objectives == 2 and "single" in graphs_to_return: + emit_json_single_plots( + plots, + result=result[0], + prefix="objective_1", + selected_point=selected_point, + ) + emit_json_single_plots( + plots, + result=result[1], + prefix="objective_2", + selected_point=selected_point, + ) +``` + +Add the import at the top of `optimizer.py` (after the existing local imports): + +```python +from .plot_emitters import emit_json_single_plots +``` + +- [ ] **Step 6: Run the equivalence test** + +```bash +env/bin/python -m pytest tests/test_optimizer.py::test_equivalence_with_and_without_pickled_multi_objective -v +``` +Expected: PASS. This test is the BINDING gate — if it fails, the prefix or argument order is wrong somewhere. + +- [ ] **Step 7: Run the full suite** + +```bash +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: all green, ~3 more tests than before (the new plot_emitters tests). + +- [ ] **Step 8: Commit** + +```bash +git add optimizerapi/plot_emitters.py optimizerapi/optimizer.py tests/test_plot_emitters.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +refactor(optimizer): unify three JSON single-plot loops into emit_json_single_plots + +Same body was inlined three times in process_result with different +prefix strings: single_{idx}, objective_1, objective_2. They are now +one function call each, in a new optimizerapi/plot_emitters.py module. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 4: Move PNG plot emission to `plot_emitters.py` + +The PNG path inside `process_result` is its own ~25-line block. Move it out so the dispatcher has fewer branches to read. + +**Files:** +- Modify: `optimizerapi/plot_emitters.py` +- Modify: `optimizerapi/optimizer.py` + +- [ ] **Step 1: Add `emit_png_plots` to `plot_emitters.py`** + +Append to `optimizerapi/plot_emitters.py`: + +```python +import base64 +import io + +import matplotlib.pyplot as plt +from ProcessOptimizer.plots import ( + plot_brownie_bee_frontend, + plot_convergence, + plot_objective, +) + + +def emit_png_plots( + plots: list["Plot"], + *, + result: list, + dimensions: list[str], + graphs: list[str], + max_quality: int, + objective_pars: str, +) -> None: + """Append base64-encoded PNG plots for each model in ``result``.""" + for idx, model in enumerate(result): + if "single" in graphs: + bb_plots = plot_brownie_bee_frontend(model, max_quality=max_quality) + for i, plot in enumerate(bb_plots): + plots.append( + {"id": f"single_{idx}_{i}", "plot": _figure_to_b64(plot)} + ) + plt.close(plot) + if "convergence" in graphs: + plot_convergence(model) + _emit_current_figure(plots, f"convergence_{idx}") + if "objective" in graphs: + plot_objective( + model, + dimensions=dimensions, + usepartialdependence=False, + show_confidence=True, + pars=objective_pars, + ) + _emit_current_figure(plots, f"single_{idx}") + + +def _figure_to_b64(figure) -> str: + buf = io.BytesIO() + figure.savefig(buf, format="png") + buf.seek(0) + return base64.b64encode(buf.read()).decode("utf-8") + + +def _emit_current_figure(plots: list["Plot"], plot_id: str) -> None: + """Snapshot the current matplotlib figure into ``plots`` and clear it.""" + buf = io.BytesIO() + plt.savefig(buf, format="png", bbox_inches="tight") + buf.seek(0) + plots.append({"id": plot_id, "plot": base64.b64encode(buf.read()).decode("utf-8")}) + plt.clf() +``` + +`_emit_current_figure` is the new home for the logic that was in the old `add_plot` helper in `optimizer.py`. The `debug=True` branch of `add_plot` is dropped — it was an unused breakpoint helper. + +- [ ] **Step 2: Use `emit_png_plots` from `process_result`** + +In `optimizer.py:process_result`, find the PNG branch: + +```python + if graph_format == "png": + for idx, model in enumerate(result): + if "single" in graphs_to_return: + bb_plots = plot_brownie_bee_frontend(model, max_quality=max_quality) + for i, plot in enumerate(bb_plots): + pic_io_bytes = io.BytesIO() + plot.savefig(pic_io_bytes, format="png") + pic_io_bytes.seek(0) + pic_hash = base64.b64encode(pic_io_bytes.read()) + plots.append( + {"id": f"single_{idx}_{i}", "plot": str(pic_hash, "utf-8")} + ) + plt.close(plot) + if "convergence" in graphs_to_return: + plot_convergence(model) + add_plot(plots, f"convergence_{idx}") + + if "objective" in graphs_to_return: + plot_objective( + model, + dimensions=dimensions, + usepartialdependence=False, + show_confidence=True, + pars=objective_pars, + ) + add_plot(plots, f"single_{idx}") + + if optimizer.n_objectives == 1: + minimum = expected_minimum(result[0], return_std=True) + result_details["expected_minimum"] = [ + round_to_length_scales(minimum[0], optimizer.space), + minimum[1], + ] +``` + +Replace with: + +```python + if graph_format == "png": + emit_png_plots( + plots, + result=result, + dimensions=dimensions, + graphs=graphs_to_return, + max_quality=max_quality, + objective_pars=objective_pars, + ) + if optimizer.n_objectives == 1: + _set_expected_minimum(result_details, result[0], optimizer.space) +``` + +The `_set_expected_minimum` helper is new — add it near the other private helpers in `optimizer.py`: + +```python +def _set_expected_minimum( + result_details: dict, + single_result: object, + space: object, +) -> None: + """Compute and store the expected minimum (single-objective only).""" + minimum = expected_minimum(single_result, return_std=True) + result_details["expected_minimum"] = [ + round_to_length_scales(minimum[0], space), + minimum[1], + ] +``` + +Then find the duplicated `expected_minimum` block in the JSON single-objective branch: + +```python + if optimizer.n_objectives == 1: + minimum = expected_minimum(result[0], return_std=True) + result_details["expected_minimum"] = [ + round_to_length_scales(minimum[0], optimizer.space), + minimum[1], + ] +``` + +Replace with: + +```python + if optimizer.n_objectives == 1: + _set_expected_minimum(result_details, result[0], optimizer.space) +``` + +Add the import: + +```python +from .plot_emitters import emit_json_single_plots, emit_png_plots +``` + +(Replace the previous import line with the combined import.) + +- [ ] **Step 3: Delete the now-unused `add_plot` helper** + +The old `add_plot` function in `optimizer.py` (the one that writes to `result` and conditionally calls `plt.savefig` with `debug`) is no longer referenced. Delete it. + +```bash +grep -n "^def add_plot\|add_plot(" optimizerapi/optimizer.py +``` +Expected after deletion: no output for `^def add_plot` and no remaining call sites. + +- [ ] **Step 4: Run the equivalence test + full suite** + +```bash +env/bin/python -m pytest tests/test_optimizer.py::test_equivalence_with_and_without_pickled_multi_objective -v +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: equivalence PASS; full suite unchanged. + +- [ ] **Step 5: Commit** + +```bash +git add optimizerapi/plot_emitters.py optimizerapi/optimizer.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +refactor(optimizer): move PNG plot emission to plot_emitters, drop add_plot + +PNG plot rendering, the unused debug=True branch of add_plot, and the +duplicate expected_minimum block are folded into emit_png_plots / +_set_expected_minimum. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 5: Move pareto-data emission to `plot_emitters.py` + +**Files:** +- Modify: `optimizerapi/plot_emitters.py` +- Modify: `optimizerapi/optimizer.py` + +- [ ] **Step 1: Add `emit_pareto_data` to `plot_emitters.py`** + +```python +from ProcessOptimizer.plots import get_Brownie_Bee_Pareto +from ProcessOptimizer.utils.utils import get_Pareto_front_compromise + + +def emit_pareto_data(plots: list["Plot"], optimizer: object) -> None: + """Append the pareto-front payload (multi-objective only).""" + front_x_data, front_y_data, obj1_error, obj2_error = get_Brownie_Bee_Pareto( + optimizer, n_points=200 + ) + best_idx = get_Pareto_front_compromise(front_y_data) + pareto_data = { + "front_x_data": front_x_data.tolist(), + "front_y_data": front_y_data.tolist(), + "obj1_error": obj1_error.tolist(), + "obj2_error": obj2_error.tolist(), + "best_idx": best_idx, + } + plots.append({"id": "pareto_data", "plot": json_tricks.dumps(pareto_data)}) +``` + +Add the `import` to the top of `plot_emitters.py` if not already there. + +- [ ] **Step 2: Use it in `process_result`** + +Find the pareto-computation block: + +```python + if optimizer.n_objectives == 2 and ( + "pareto" in graphs_to_return or "single" in graphs_to_return + ): + front_x_data, front_y_data, obj1_error, obj2_error = ( + get_Brownie_Bee_Pareto(optimizer, n_points=200) + ) + best_idx = get_Pareto_front_compromise(front_y_data) + if "pareto" in graphs_to_return: + pareto_data = { + "front_x_data": front_x_data.tolist(), + "front_y_data": front_y_data.tolist(), + "obj1_error": obj1_error.tolist(), + "obj2_error": obj2_error.tolist(), + "best_idx": best_idx, + } + plots.append( + { + "id": "pareto_data", + "plot": json_tricks.dumps(pareto_data), + } + ) +``` + +Replace with: + +```python + if optimizer.n_objectives == 2 and "pareto" in graphs_to_return: + emit_pareto_data(plots, optimizer) +``` + +**Notice the simplification:** the original `if` was `pareto OR single` but only emitted when `pareto` was actually in the list. The `OR single` was dead — the original outer condition gated a block where only the inner `if "pareto"` actually mattered. We drop the misleading OR. + +(If a future need arises to compute the pareto front for plotting reasons beyond `"pareto"`, this can come back. Right now it doesn't.) + +Update the import line: + +```python +from .plot_emitters import emit_json_single_plots, emit_pareto_data, emit_png_plots +``` + +Remove now-unused imports from `optimizer.py`: + +```bash +grep -n "from ProcessOptimizer.plots import\|get_Pareto_front_compromise" optimizerapi/optimizer.py +``` + +If `get_Brownie_Bee_Pareto`, `plot_brownie_bee_frontend`, `plot_convergence`, `plot_objective`, or `get_Pareto_front_compromise` are no longer used in `optimizer.py`, remove them from the imports. (`_get_brownie_bee_1d_plot_safe` still lives in `optimizer.py` and is re-imported by `plot_emitters.py` — leave that alone for now.) + +- [ ] **Step 3: Run the equivalence test + full suite** + +```bash +env/bin/python -m pytest tests/test_optimizer.py::test_equivalence_with_and_without_pickled_multi_objective -v +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: equivalence PASS; full suite unchanged. + +- [ ] **Step 4: Commit** + +```bash +git add optimizerapi/plot_emitters.py optimizerapi/optimizer.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +refactor(optimizer): move emit_pareto_data to plot_emitters + +Also drops the misleading 'pareto OR single' outer condition — only +'pareto' was ever load-bearing for the actual emission. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 6: Final shrink + mypy + lint + +By this point `process_result` should be close to 60 lines. The body still has the local-alias reassignments from Task 1; drop them now that the function is small enough to use `parsed.X` directly throughout. + +**Files:** +- Modify: `optimizerapi/optimizer.py` + +- [ ] **Step 1: Drop the local-alias block in `process_result`** + +Find: + +```python + parsed = _parse_extras(extras, logging.getLogger(__name__)) + graph_format = parsed.graph_format + max_quality = parsed.max_quality + graphs_to_return = parsed.graphs_to_return + objective_pars = parsed.objective_pars + pickle_model = parsed.include_model + selected_point = parsed.selected_point + experiment_suggestion_count = parsed.experiment_suggestion_count +``` + +Replace with: + +```python + parsed = _parse_extras(extras, logging.getLogger(__name__)) +``` + +Then update every reference in the body from the bare name to `parsed.X`. For example: + +- `graph_format == "png"` → `parsed.graph_format == "png"` +- `experiment_suggestion_count` → `parsed.experiment_suggestion_count` +- `selected_point` → `parsed.selected_point` +- `graphs_to_return` → `parsed.graphs_to_return` +- `max_quality` → `parsed.max_quality` +- `objective_pars` → `parsed.objective_pars` +- `pickle_model` → `parsed.include_model` + +The body should now be small enough to inspect that no references are missed. Run pytest after the substitution. + +- [ ] **Step 2: Run mypy on `optimizer.py`** + +```bash +env/bin/mypy optimizerapi/optimizer.py 2>&1 | tail -20 +``` +Expected: significantly fewer errors than the Phase 3 baseline. The new helpers are typed; the dispatcher uses typed inputs. Fix anything that mypy now flags as a real issue (typically forgotten-import, attribute-access on a wider type than intended). + +- [ ] **Step 3: Run flake8** + +```bash +env/bin/flake8 optimizerapi/optimizer.py optimizerapi/plot_emitters.py --max-line-length=127 +``` +Expected: clean. + +- [ ] **Step 4: Run the equivalence test and full suite one final time** + +```bash +env/bin/python -m pytest tests/test_optimizer.py::test_equivalence_with_and_without_pickled_multi_objective -v +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: equivalence PASS; full suite green. + +- [ ] **Step 5: Commit** + +```bash +git add optimizerapi/optimizer.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +refactor(optimizer): drop local aliases; use parsed.X directly + +Final shrink of process_result after the extractions land. The +dispatcher is now under ~80 lines and walks each path in order: +extras → next → models → graph branches → pickled → extras tail. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +- [ ] **Step 6: Check the LOC delta** + +```bash +wc -l optimizerapi/optimizer.py optimizerapi/plot_emitters.py +``` + +Reasonable target: `optimizer.py` under 350 lines (was 565), `plot_emitters.py` under 200 lines. The total may be similar or slightly larger — the win is in distribution and readability, not raw line count. + +--- + +## Acceptance criteria + +- `optimizerapi/optimizer.py:process_result` is under ~80 lines. +- `optimizerapi/plot_emitters.py` exists and owns all plot emission. +- `emit_json_single_plots` is called three times from `process_result` (single-obj single, multi-obj objective_1, multi-obj objective_2) — i.e. the duplication is gone. +- The PNG path in `process_result` is a single call to `emit_png_plots`. +- `add_plot` and its `debug=True` branch are deleted (no callers remain). +- `_get_brownie_bee_1d_plot_safe`, `round_to_length_scales`, `process_model`, `add_version_info`, `convert_number_type` continue to live in `optimizer.py` — they don't fit the plot-emitters split. +- `test_equivalence_with_and_without_pickled_multi_objective` passes after every commit in this phase. +- Full pytest run is green. +- `mypy optimizerapi/optimizer.py optimizerapi/plot_emitters.py` is clean. +- `flake8 optimizerapi --max-line-length=127` introduces no new violations. + +## Out of scope + +- Annotating every internal helper (most are now small enough to be obvious). +- Removing `_get_brownie_bee_1d_plot_safe` from `optimizer.py` (it's the workaround for the upstream bug; moving it would cost more clarity than it gains). +- Async / streaming responses (a Phase 6+ idea, if it ever happens). +- Changing the OpenAPI response shape (this whole phase is behavior-preserving). diff --git a/docs/superpowers/plans/2026-05-18-audit-phase-5-cleanup.md b/docs/superpowers/plans/2026-05-18-audit-phase-5-cleanup.md new file mode 100644 index 0000000..29b43eb --- /dev/null +++ b/docs/superpowers/plans/2026-05-18-audit-phase-5-cleanup.md @@ -0,0 +1,646 @@ +# Audit Phase 5 — Opportunistic Cleanup + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Finish the long-tail of code-quality items surfaced by the audit: pre-existing flake8 violations, the `optimizer_handler.py` rough edges, the auth typo, and idiomatic Python tidies that the earlier phases left for last. Bring `flake8 optimizerapi tests --max-line-length=127` fully clean. + +**Architecture:** Pure cleanup. No new modules, no new public surface, no behavior changes. + +**Tech Stack:** Standard library `secrets`, `pytest`, mypy from Phase 3. + +**Audit reference:** §3, §6, §7 of the audit, plus the "pre-existing E501s" deferred through Phases 1–4. + +**Prereq:** Phases 1–4 complete. Full pytest is green. + +--- + +## File Map + +- **Modify:** `tests/test_optimizer.py` — delete dead comment lines. +- **Modify:** `optimizerapi/securepickle/__init__.py` — add `__all__` to resolve F401. +- **Modify:** `tests/context.py` — remove unused imports or convert to `__all__`. +- **Modify:** `optimizerapi/optimizer_handler.py` — `USE_WORKER` bool parsing; consolidate `disconnect_check`; normalize return shape. +- **Modify:** `optimizerapi/auth.py` — fix `"Authenitcation"` typo. +- **Modify:** `optimizerapi/optimizer.py` — `enumerate` + `.get()` idiomatic wins. + +--- + +### Task 1: Delete dead 400-character comment lines + +**Files:** +- Modify: `tests/test_optimizer.py:12-15` + +- [ ] **Step 1: Inspect** + +```bash +sed -n '11,16p' tests/test_optimizer.py +``` + +Expected: three comment lines (`# {'data': ...}`, `# 'data': ...`, `# 'space': ...`, `# '...'}}`) totaling ~880 characters. These were sample-payload notes from early development. + +- [ ] **Step 2: Delete the comment block** + +Remove lines 12-15 of `tests/test_optimizer.py`. The block looks like: + +```python +# {'data': [{'xi': [651, 56, 722, 'Ræv'], 'yi': 1}, ... +# 'data': [{'xi': [0, 5, 'Rød'], 'yi': 10}, ... +# 'optimizerConfig': {'baseEstimator': 'GP', ... +# 'space': [{'type': 'discrete', 'name': 'Alkohol', ... +``` + +The information they recorded is now captured properly in `docs/api-usage.md` and `scripts/sample*.curl`. + +- [ ] **Step 3: Verify flake8** + +```bash +env/bin/flake8 tests/test_optimizer.py --max-line-length=127 +``` +Expected: no output. The two E501 violations that have been pre-existing since the redesign are gone. + +- [ ] **Step 4: Run the full suite** + +```bash +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: unchanged. + +- [ ] **Step 5: Commit** + +```bash +git add tests/test_optimizer.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +chore(tests): drop dead 400-char sample-payload comments + +The sample payloads are now documented properly in docs/api-usage.md +and scripts/sample*.curl. The comments at the top of test_optimizer.py +were one-off dev notes that have been silently failing flake8 E501 +for ages. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 2: Fix F401 re-exports in `securepickle/__init__.py` + +The package `__init__.py` imports symbols only to re-export them. flake8 reports F401 because there's no `__all__` declaring intent. + +**Files:** +- Modify: `optimizerapi/securepickle/__init__.py` + +- [ ] **Step 1: Read current state** + +```bash +cat optimizerapi/securepickle/__init__.py +``` + +Expected (current contents): + +```python +""" +Secure pickle module +""" +from .pickler import pickleToString, unpickleFromString +from .secure import get_crypto +``` + +- [ ] **Step 2: Add `__all__`** + +Replace with: + +```python +"""Secure pickle module — Fernet-encrypted pickle round-trip.""" + +from .pickler import pickleToString, unpickleFromString +from .secure import get_crypto + +__all__ = ["get_crypto", "pickleToString", "unpickleFromString"] +``` + +- [ ] **Step 3: Verify flake8** + +```bash +env/bin/flake8 optimizerapi/securepickle/__init__.py --max-line-length=127 +``` +Expected: no output. + +- [ ] **Step 4: Commit** + +```bash +git add optimizerapi/securepickle/__init__.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +chore(securepickle): declare __all__ to resolve F401 on re-exports + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 3: Clean up `tests/context.py` + +**Files:** +- Modify: `tests/context.py` + +- [ ] **Step 1: Read current state** + +```bash +cat tests/context.py +``` + +Decide based on what you find: + +- If `context.py` is an unused / abandoned test helper: delete the file and remove any imports of it. (You'll need to `grep -rn "from .context\|from tests.context\|from context" tests/ optimizerapi/` first.) +- If it is used somewhere: convert its imports into a `__all__` block similar to Task 2. + +- [ ] **Step 2: Apply the fix** + +In practice, this file imports `optimizer` and `securepickle` for tests to use without going through the full package path. If nothing imports `tests.context`, delete the file. Otherwise add: + +```python +"""Test-only convenience re-exports.""" + +from optimizerapi import optimizer, securepickle + +__all__ = ["optimizer", "securepickle"] +``` + +- [ ] **Step 3: Verify** + +```bash +env/bin/flake8 tests/context.py --max-line-length=127 +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: no flake8 output; pytest unchanged. + +- [ ] **Step 4: Commit** + +```bash +git add tests/context.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +chore(tests): tidy tests/context.py (or delete if unused) + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +(If the file was deleted, mention that in the commit body.) + +--- + +### Task 4: Sweep `optimizer_handler.py` + +Three small wins, one commit. + +**Files:** +- Modify: `optimizerapi/optimizer_handler.py` + +- [ ] **Step 1: Fix `USE_WORKER` boolean parsing** + +Find: + +```python + if "USE_WORKER" in os.environ and os.environ["USE_WORKER"]: +``` + +The bug: `os.environ["USE_WORKER"] = "false"` is truthy. Any non-empty string passes this gate. + +Replace with: + +```python + if _parse_env_bool("USE_WORKER"): +``` + +Add this helper at module scope (after the imports, before `_LOG`): + +```python +def _parse_env_bool(name: str, default: bool = False) -> bool: + """Parse a boolean-ish env var. + + Accepts "true", "1", "yes" (case-insensitive) as True. + Anything else — including unset or "false" — is False. + """ + raw = os.environ.get(name, "").strip().lower() + if not raw: + return default + return raw in ("true", "1", "yes") +``` + +- [ ] **Step 2: Consolidate `disconnect_check`** + +The current function defines `disconnect_check` three separate ways depending on which Connexion / Flask state is reachable. Replace: + +```python + try: + if "waitress.client_disconnected" in connexion.request.environ: + disconnect_check = connexion.request.environ["waitress.client_disconnected"] + else: + + def disconnect_check(): + return False + + except RuntimeError: + + def disconnect_check(): + return False +``` + +with: + +```python + disconnect_check = _resolve_disconnect_check() +``` + +And add the helper at module scope: + +```python +def _resolve_disconnect_check(): + """Return a callable that reports whether the client has disconnected. + + Outside of a request context (e.g. unit tests) or when running under + a server that doesn't expose ``waitress.client_disconnected``, this + returns a no-op that always says "still connected". + """ + try: + env = connexion.request.environ + except RuntimeError: + return lambda: False + return env.get("waitress.client_disconnected", lambda: False) +``` + +- [ ] **Step 3: Normalize `do_run_work` return shape** + +Currently the success path returns a `dict` while the error paths return a `(dict, status_code)` tuple. Make all paths return the tuple shape so callers don't branch on it. + +Find: + +```python +def do_run_work(body) -> dict: + """ "Handle the run request""" + try: + return handle_run(body) + except IOError as err: + return ({"message": "I/O error", "error": str(err)}, 400) + except TypeError as err: + return ({"message": "Type error", "error": str(err)}, 400) + except ValueError as err: + return ({"message": "Validation error", "error": str(err)}, 400) + except Exception as err: + # Log unknown exceptions to support debugging + traceback.print_exc() + return ({"message": "Unknown error", "error": str(err)}, 500) +``` + +Replace with: + +```python +def do_run_work(body: "RequestBody") -> "ResponseEnvelope": + """Handle the run request. + + On error we let Connexion translate the exception into the OpenAPI- + declared 400 / 500 response. The handler stops being a tuple-returning + special case. + """ + try: + return handle_run(body) + except (IOError, TypeError, ValueError) as err: + _LOG.warning("client error: %s", err) + raise connexion.problem(400, "Bad request", str(err)) + except Exception as err: + _LOG.exception("unexpected error during optimizer run") + raise connexion.problem(500, "Internal server error", str(err)) +``` + +If the project's Connexion version does not support `connexion.problem` (`2.14.2` does, as `from connexion.exceptions import ProblemException` or similar), substitute the explicit `flask.abort(...)` / raise-with-status path. Test on the dev server before committing. + +If `connexion.problem` returns a `(body, status)` tuple rather than an exception, adapt: + +```python + try: + return handle_run(body) + except (IOError, TypeError, ValueError) as err: + _LOG.warning("client error: %s", err) + return connexion.problem(400, "Bad request", str(err)) + except Exception as err: + _LOG.exception("unexpected error during optimizer run") + return connexion.problem(500, "Internal server error", str(err)) +``` + +Either way, the result is that `traceback.print_exc()` (a print equivalent) is gone — the same diagnostics flow via `_LOG.exception`, which includes the traceback automatically. + +- [ ] **Step 4: Add the imports if missing** + +At the top, ensure `connexion` is imported (it already is — used elsewhere in the file). No new imports needed. + +- [ ] **Step 5: Run the full suite** + +```bash +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: full suite still green. If a test expected the old tuple-return shape, fix it (the test was testing the bug, not the contract). + +- [ ] **Step 6: Smoke-test the server with the worker path off and on** + +```bash +USE_WORKER=false timeout 5 env/bin/python -m optimizerapi.server 2>&1 | head -10 || true +``` + +Expected: no log line saying "USE_WORKER detected" — the explicit `false` is correctly parsed. + +- [ ] **Step 7: Commit** + +```bash +git add optimizerapi/optimizer_handler.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +refactor(handler): _parse_env_bool, single disconnect_check resolver, normalized error path + +Fixes the USE_WORKER='false' truthiness bug, collapses three near- +identical disconnect_check definitions, and routes errors via +connexion.problem so do_run_work returns a single declared shape +instead of a mixed dict/tuple. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 5: Fix `Authenitcation` typo and an idiomatic-Python sweep + +**Files:** +- Modify: `optimizerapi/auth.py` +- Modify: `optimizerapi/optimizer.py` + +- [ ] **Step 1: Fix the auth.py docstring typo** + +Find: + +```python +"""Authenitcation module + +This module will verify tokens provided bt a Keycloak OpenID server +""" +``` + +Replace with: + +```python +"""Authentication module. + +Verifies bearer tokens issued by a Keycloak OpenID server, and the +static API key provided via ``AUTH_API_KEY``. +""" +``` + +(Also fixes "bt" → "by".) + +- [ ] **Step 2: `enumerate` for `round_to_length_scales` in `optimizer.py`** + +Find: + +```python + for dim, i in zip(space.dimensions, range(len(space.dimensions))): +``` + +Replace with: + +```python + for i, dim in enumerate(space.dimensions): +``` + +- [ ] **Step 3: `in (...)` for the dimension-type check in `optimizer.py`** + +Find: + +```python + if (x["type"] == "discrete" or x["type"] == "continuous") +``` + +Replace with: + +```python + if x["type"] in ("discrete", "continuous") +``` + +- [ ] **Step 4: `.get()` for the `extras` and `constraints` parses in `optimizer.run`** + +Find: + +```python + cfg = body["optimizerConfig"] + constraints = cfg["constraints"] if "constraints" in cfg else [] + extras = body["extras"] if "extras" in body else {} +``` + +Replace with: + +```python + cfg = body["optimizerConfig"] + constraints = cfg.get("constraints", []) + extras = body.get("extras", {}) +``` + +- [ ] **Step 5: Verify lint + suite** + +```bash +env/bin/flake8 optimizerapi --max-line-length=127 +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: no new flake8 output; pytest green. + +- [ ] **Step 6: Commit** + +```bash +git add optimizerapi/auth.py optimizerapi/optimizer.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +chore: typo fix + small idiomatic Python sweep + +- auth.py: 'Authenitcation' → 'Authentication', 'bt' → 'by' +- optimizer.py: enumerate; tuple membership for dim type; + .get() defaults for cfg.constraints and body.extras + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 6: Optional — convert sample fixtures to `@pytest.fixture` + +This is offered as a discrete task because it touches many test signatures. Skip if appetite is low; the module-level constants work today and tests mostly `copy.deepcopy` defensively. + +**Files:** +- Modify: `tests/test_optimizer.py` +- Create: `tests/conftest.py` + +- [ ] **Step 1: Create `tests/conftest.py`** + +Pull the read-only fixtures out of `test_optimizer.py`: + +```python +"""Shared fixtures for the optimizer test suite.""" + +import copy + +import pytest + + +@pytest.fixture +def sample_data(): + return copy.deepcopy([ + {"xi": [651, 56, 722, "Ræv"], "yi": [1]}, + {"xi": [651, 42, 722, "Ræv"], "yi": [0.2]}, + ]) + + +@pytest.fixture +def sample_config(): + return copy.deepcopy({ + "baseEstimator": "GP", + "acqFunc": "gp_hedge", + "initialPoints": 2, + "kappa": 1.96, + "xi": 0.012, + "space": [ + {"type": "discrete", "name": "Sukker", "from": 0, "to": 1000}, + {"type": "continuous", "name": "Peber", "from": 0, "to": 1000}, + {"type": "continuous", "name": "Hvedemel", "from": 0, "to": 1000}, + {"type": "category", "name": "Kunde", "categories": ["Mus", "Ræv"]}, + ], + }) + + +@pytest.fixture +def sample_multi_objective_5dim_data(): + return copy.deepcopy([ + {"xi": [16.7, 500, 250, 20, "None"], "yi": [-2, -17]}, + {"xi": [50, 833, 150, 60, "Whipped cream"], "yi": [-3, -6]}, + {"xi": [58.3, 22, 85, 6, "Frosting"], "yi": [-6, -25]}, + ]) + + +@pytest.fixture +def sample_multi_objective_5dim_config(): + return copy.deepcopy({ + "baseEstimator": "GP", + "acqFunc": "EI", + "initialPoints": 3, + "kappa": 1.96, + "xi": 2, + "space": [ + {"type": "continuous", "name": "Sugar", "from": 0, "to": 100}, + {"type": "continuous", "name": "Flour", "from": 0, "to": 1000}, + {"type": "discrete", "name": "Temperature", "from": 0, "to": 300}, + {"type": "discrete", "name": "Time", "from": 0, "to": 120}, + {"type": "category", "name": "Finish", + "categories": ["None", "Frosting", "Whipped cream"]}, + ], + "constraints": [], + }) +``` + +- [ ] **Step 2: Migrate `tests/test_optimizer.py` to use the fixtures** + +For each test that referenced a module-level constant, add the relevant fixture as a parameter: + +```python +# Before +def test_can_be_run_without_data(): + result = optimizer.run(body={"data": [], "optimizerConfig": sampleConfig}) + ... + +# After +def test_can_be_run_without_data(sample_config): + result = optimizer.run(body={"data": [], "optimizerConfig": sample_config}) + ... +``` + +Repeat for every test in the file. This is mechanical but mass — be patient and re-run pytest after each batch of ~5 tests to catch errors early. + +Once every test uses fixtures, delete the module-level `sampleData`, `sampleConfig`, `sampleMultiObjective5DimData`, `sampleMultiObjective5DimConfig` constants from `test_optimizer.py`. (The `sampleMultiObjectiveData`, `brownie_with_constraints`, and `brownie_without_constraints` constants can stay or move to conftest as you prefer.) + +- [ ] **Step 3: Run the full suite** + +```bash +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: same number of passes as before. + +- [ ] **Step 4: Commit** + +```bash +git add tests/conftest.py tests/test_optimizer.py +git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' +refactor(tests): move sample fixtures into conftest.py + +Each test now declares its dependencies explicitly via fixture +parameters. Mutating a fixture in one test no longer leaks into others. + +Co-Authored-By: Claude Opus 4.7 (1M context) +EOF +)" +``` + +--- + +### Task 7: Final lint + mypy + pytest + +The acceptance gate for the entire audit. + +**Files:** none. + +- [ ] **Step 1: flake8 clean** + +```bash +env/bin/flake8 optimizerapi tests --max-line-length=127 +``` +Expected: **no output**. Every pre-existing exception is now fixed. + +- [ ] **Step 2: mypy clean for `optimizerapi/`** + +```bash +env/bin/mypy optimizerapi +``` +Expected: 0 errors. (Phase 3 set up the config; Phase 4 cleaned up `optimizer.py`'s body; Phase 5 closes the long-tail gaps.) + +If errors remain, fix them inline. They should be small at this point — bad import paths or remaining `Any` leaks at module boundaries. + +- [ ] **Step 3: Full test suite** + +```bash +env/bin/python -m pytest 2>&1 | tail -3 +``` +Expected: 50+ passed (43 original + auth + types + plot_emitters + no-prints + minor). No failures, no errors. + +- [ ] **Step 4: No commit required.** + +--- + +## Acceptance criteria (overall audit) + +After Phase 5 completes, the following must hold: + +- `env/bin/flake8 optimizerapi tests --max-line-length=127` → no output. +- `env/bin/mypy optimizerapi` → 0 errors. +- `env/bin/python -m pytest` → all green. +- `grep -rn "print(" optimizerapi/` → no output (from Phase 2). +- `optimizerapi/optimizer.py:process_result` is under ~80 lines (from Phase 4). +- `optimizerapi/auth.py` uses `secrets.compare_digest` and never prints tokens (from Phase 1). +- `cryptography` is on a current major (from Phase 1). +- The OpenAPI surface is unchanged (response shape and field names preserved across the audit). +- The equivalence test from the pareto redesign still passes (this is the binding behaviour gate). + +## Out of scope + +- Flask 3 / Connexion 3 migration (separate project). +- Pinning `ProcessOptimizer` to a release tag (no tagged release of the required commit exists at the time of this plan). +- Switching to async / background-only request handling (Phase 6+ if it ever happens). +- Coverage tooling, mutation testing, CI workflow changes. +- Documentation rewrites — `docs/api-usage.md` is current. diff --git a/docs/superpowers/plans/2026-05-18-pareto-extras-redesign.md b/docs/superpowers/plans/2026-05-18-pareto-extras-redesign.md new file mode 100644 index 0000000..5bb4810 --- /dev/null +++ b/docs/superpowers/plans/2026-05-18-pareto-extras-redesign.md @@ -0,0 +1,993 @@ +# Pareto `extras` Redesign Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Lock in the contract for `extras.selectedPoint` / `extras.pickled` / `extras.includeModel` such that `pickled` is a verified cache hint (never authoritative) and a no-`pickled` round-trip produces equivalent output to a `pickled` round-trip. + +**Architecture:** Pull pickle pack/unpack logic into a new `optimizerapi/pickled_state.py` module that owns the request-fingerprint computation, payload structure (`{"fingerprint", "result", "next", "optimizer"}`), and the four-stage validation flow (decrypt → structural → fingerprint → fast-path). `optimizer.py:run` delegates to it. A `result.extras.pickledUsed` boolean is added so the UI can observe fast-path usage. PNG + `selectedPoint` and `includeModel=false` + `pickled` log warnings but otherwise do exactly what the request asked for. + +**Tech Stack:** Python 3.9+, pytest, Connexion/Flask, Fernet (already wired via `optimizerapi/securepickle`), ProcessOptimizer. + +**Spec:** [`docs/superpowers/specs/2026-05-18-pareto-extras-redesign-design.md`](../specs/2026-05-18-pareto-extras-redesign-design.md) + +--- + +## File Map + +- **Create:** `optimizerapi/pickled_state.py` — fingerprint + pack/unpack helpers; sole owner of payload structure and fall-through logging. +- **Create:** `tests/test_pickled_state.py` — unit tests for fingerprint determinism, round-trip, and each fall-through reason. +- **Modify:** `optimizerapi/optimizer.py` — replace the inline unpickle block in `run`, thread fingerprint + `pickled_used` through to `process_result`, add the two new warning logs. +- **Modify:** `optimizerapi/openapi/specification.yml` — tighten descriptions for `extras.pickled` and `extras.selectedPoint`; add `result.extras.pickledUsed`. +- **Modify:** `tests/test_optimizer.py` — add the binding equivalence test, fingerprint-mismatch test, PNG+selectedPoint warning test, includeModel+pickled warning test, and pickledUsed round-trip test; refresh `test_old_format_pickled_falls_back` and `test_pickled_response_is_dict_format` for the new payload shape. + +--- + +### Task 1: Fingerprint helper in new module + +**Files:** +- Create: `optimizerapi/pickled_state.py` +- Test: `tests/test_pickled_state.py` + +- [ ] **Step 1: Write the failing test** + +Create `tests/test_pickled_state.py` with the following content: + +```python +"""Tests for pickled_state: fingerprint + pack/unpack helpers.""" + +from optimizerapi.pickled_state import compute_fingerprint + + +SAMPLE_DATA = [ + {"xi": [651, 56, 722, "Ræv"], "yi": [1]}, + {"xi": [651, 42, 722, "Ræv"], "yi": [0.2]}, +] + +SAMPLE_CONFIG = { + "baseEstimator": "GP", + "acqFunc": "gp_hedge", + "initialPoints": 2, + "kappa": 1.96, + "xi": 0.012, + "space": [ + {"type": "discrete", "name": "Sukker", "from": 0, "to": 1000}, + {"type": "continuous", "name": "Peber", "from": 0, "to": 1000}, + {"type": "continuous", "name": "Hvedemel", "from": 0, "to": 1000}, + {"type": "category", "name": "Kunde", "categories": ["Mus", "Ræv"]}, + ], +} + + +def test_fingerprint_is_deterministic(): + fp1 = compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) + fp2 = compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) + assert fp1 == fp2 + assert isinstance(fp1, str) + assert len(fp1) == 64 # sha256 hex + + +def test_fingerprint_is_independent_of_dict_key_order(): + reordered_config = { + "space": SAMPLE_CONFIG["space"], + "xi": SAMPLE_CONFIG["xi"], + "kappa": SAMPLE_CONFIG["kappa"], + "initialPoints": SAMPLE_CONFIG["initialPoints"], + "acqFunc": SAMPLE_CONFIG["acqFunc"], + "baseEstimator": SAMPLE_CONFIG["baseEstimator"], + } + assert compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) == compute_fingerprint( + SAMPLE_DATA, reordered_config + ) + + +def test_fingerprint_changes_when_data_changes(): + other_data = SAMPLE_DATA + [{"xi": [100, 100, 100, "Mus"], "yi": [0.5]}] + assert compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) != compute_fingerprint( + other_data, SAMPLE_CONFIG + ) + + +def test_fingerprint_changes_when_config_changes(): + other_config = dict(SAMPLE_CONFIG, kappa=2.0) + assert compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) != compute_fingerprint( + SAMPLE_DATA, other_config + ) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m pytest tests/test_pickled_state.py -v` +Expected: FAIL with `ModuleNotFoundError: No module named 'optimizerapi.pickled_state'`. + +- [ ] **Step 3: Write minimal implementation** + +Create `optimizerapi/pickled_state.py` with: + +```python +"""Pickled-state cache: fingerprint a request, pack/unpack pickled payloads. + +The fingerprint binds a pickled blob to the request that produced it, so a +client that sends a stale pickled along with newer data gets a clean fall- +through to a full run instead of silently buggy reuse. +""" + +import hashlib +import json + + +def compute_fingerprint(data, optimizer_config): + """Return sha256 hex of the canonical-JSON of (data, optimizerConfig). + + Canonical JSON: sorted keys, no whitespace. Numbers are emitted as Python's + default JSON representation, which is stable for the integer / float values + that flow through the API. + """ + payload = {"data": data, "optimizerConfig": optimizer_config} + canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `python -m pytest tests/test_pickled_state.py -v` +Expected: PASS, 4 tests. + +- [ ] **Step 5: Commit** + +```bash +git add optimizerapi/pickled_state.py tests/test_pickled_state.py +git commit -m "feat: add request fingerprint helper for pickled cache" +``` + +--- + +### Task 2: Pack + unpack with fall-through logging + +**Files:** +- Modify: `optimizerapi/pickled_state.py` +- Modify: `tests/test_pickled_state.py` + +- [ ] **Step 1: Write the failing tests (extend `tests/test_pickled_state.py`)** + +Append to `tests/test_pickled_state.py`: + +```python +import logging + +from optimizerapi.pickled_state import pack, unpack_if_valid +from optimizerapi.securepickle import get_crypto, pickleToString + + +def _crypto(): + return get_crypto() + + +def test_pack_unpack_round_trip(): + crypto = _crypto() + fingerprint = compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) + blob = pack(result="result-sentinel", next_points=["next-sentinel"], + optimizer="opt-sentinel", fingerprint=fingerprint, crypto=crypto) + assert isinstance(blob, str) and len(blob) > 0 + + payload = unpack_if_valid(blob, expected_fingerprint=fingerprint, crypto=crypto) + assert payload is not None + assert payload["result"] == "result-sentinel" + assert payload["next"] == ["next-sentinel"] + assert payload["optimizer"] == "opt-sentinel" + assert payload["fingerprint"] == fingerprint + + +def test_unpack_returns_none_on_fingerprint_mismatch(caplog): + crypto = _crypto() + fingerprint = compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) + blob = pack(result="r", next_points=[], optimizer="o", + fingerprint=fingerprint, crypto=crypto) + + with caplog.at_level(logging.WARNING, logger="optimizerapi.pickled_state"): + result = unpack_if_valid(blob, expected_fingerprint="0" * 64, crypto=crypto) + assert result is None + assert any("fingerprint_mismatch" in record.message for record in caplog.records) + + +def test_unpack_returns_none_on_decrypt_failure(caplog): + crypto = _crypto() + with caplog.at_level(logging.WARNING, logger="optimizerapi.pickled_state"): + result = unpack_if_valid("not-a-valid-blob", expected_fingerprint="x" * 64, + crypto=crypto) + assert result is None + assert any("decrypt_failed" in record.message for record in caplog.records) + + +def test_unpack_returns_none_on_bad_structure(caplog): + crypto = _crypto() + # Encrypt a non-dict payload using the same machinery. + bogus_blob = pickleToString(["not", "a", "dict"], crypto) + + with caplog.at_level(logging.WARNING, logger="optimizerapi.pickled_state"): + result = unpack_if_valid(bogus_blob, expected_fingerprint="x" * 64, + crypto=crypto) + assert result is None + assert any("bad_structure" in record.message for record in caplog.records) +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `python -m pytest tests/test_pickled_state.py -v` +Expected: FAIL with `ImportError: cannot import name 'pack' from 'optimizerapi.pickled_state'`. + +- [ ] **Step 3: Implement `pack` and `unpack_if_valid`** + +Append to `optimizerapi/pickled_state.py`: + +```python +import logging + +from .securepickle import pickleToString, unpickleFromString + +_LOG = logging.getLogger(__name__) + +_REQUIRED_KEYS = ("fingerprint", "result", "next", "optimizer") + + +def pack(*, result, next_points, optimizer, fingerprint, crypto): + """Encrypt a pickled cache payload for the given fingerprint.""" + payload = { + "fingerprint": fingerprint, + "result": result, + "next": next_points, + "optimizer": optimizer, + } + return pickleToString(payload, crypto) + + +def unpack_if_valid(blob, *, expected_fingerprint, crypto): + """Decrypt and validate a pickled cache payload. + + Returns the payload dict on success, or None if anything is off. Any + failure is logged once at WARNING with a reason tag in the message: + decrypt_failed | bad_structure | fingerprint_mismatch. + """ + if not blob: + return None + try: + payload = unpickleFromString(blob, crypto) + except Exception: + _LOG.warning("pickled cache ignored: decrypt_failed") + return None + if not isinstance(payload, dict) or not all(k in payload for k in _REQUIRED_KEYS): + _LOG.warning("pickled cache ignored: bad_structure") + return None + if payload["fingerprint"] != expected_fingerprint: + _LOG.warning("pickled cache ignored: fingerprint_mismatch") + return None + return payload +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `python -m pytest tests/test_pickled_state.py -v` +Expected: PASS, 8 tests total (4 from Task 1 + 4 new). + +- [ ] **Step 5: Commit** + +```bash +git add optimizerapi/pickled_state.py tests/test_pickled_state.py +git commit -m "feat: add pack/unpack_if_valid for pickled cache with fall-through logging" +``` + +--- + +### Task 3: Wire `pickled_state` into `optimizer.py` + +This task replaces the inline unpickle block at `optimizer.py:105-129` with a single call to `unpack_if_valid`, threads a `pickled_used` flag through `process_result`, and switches the repack at `optimizer.py:395` to use `pack`. The `pickled_used` flag surfaces as `result.extras.pickledUsed`. + +**Files:** +- Modify: `optimizerapi/optimizer.py:32` (imports) +- Modify: `optimizerapi/optimizer.py:68-167` (`run` function) +- Modify: `optimizerapi/optimizer.py:177-415` (`process_result` function) +- Modify: `tests/test_optimizer.py` (add the round-trip assertion test described below) + +- [ ] **Step 1: Write the failing test (`pickledUsed` round-trip)** + +Add this test to `tests/test_optimizer.py`, after `test_pickled_round_trip`: + +```python +def test_pickled_used_flag_round_trip(): + """First run has pickledUsed=False; second run with returned pickled has pickledUsed=True.""" + first = optimizer.run(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"includeModel": "true"}, + }) + assert first["result"]["extras"]["pickledUsed"] is False + pickled_value = first["result"]["pickled"] + assert len(pickled_value) > 0 + + second = optimizer.run(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"includeModel": "true", "pickled": pickled_value}, + }) + assert second["result"]["extras"]["pickledUsed"] is True +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m pytest tests/test_optimizer.py::test_pickled_used_flag_round_trip -v` +Expected: FAIL — `pickledUsed` key not in `result["result"]["extras"]`. + +- [ ] **Step 3: Update the imports in `optimizer.py`** + +Replace the line: + +```python +from .securepickle import get_crypto, pickleToString, unpickleFromString +``` + +with: + +```python +from .securepickle import get_crypto +from .pickled_state import compute_fingerprint, pack, unpack_if_valid +``` + +- [ ] **Step 4: Replace the inline unpickle block in `run`** + +Inside `run`, replace the block currently at lines 105–129: + +```python + # Pickled consumption: attempt to skip training if extras.pickled is provided + pickled_input = extras.get("pickled", "") + if pickled_input: + try: + unpickled = unpickleFromString(pickled_input, get_crypto()) + if isinstance(unpickled, dict) and "result" in unpickled and "optimizer" in unpickled: + result = unpickled["result"] + optimizer = unpickled["optimizer"] + response = process_result(result, optimizer, dimensions, cfg, extras, data, space) + response["result"]["extras"]["parameters"] = { + "dimensions": dimensions, + "space": space, + "hyperparams": hyperparams, + "Xi": Xi, + "Yi": Yi, + "extras": extras, + } + return json.loads(json_tricks.dumps(response)) + else: + pickled_input = "" + except Exception: + logging.getLogger(__name__).warning( + "Failed to unpickle extras.pickled, falling back to full run" + ) + pickled_input = "" +``` + +with: + +```python + request_fingerprint = compute_fingerprint(body["data"], cfg) + pickled_input = extras.get("pickled", "") + cached = unpack_if_valid( + pickled_input, expected_fingerprint=request_fingerprint, crypto=get_crypto() + ) if pickled_input else None + + if cached is not None: + result = cached["result"] + optimizer = cached["optimizer"] + response = process_result( + result, optimizer, dimensions, cfg, extras, data, space, + request_fingerprint=request_fingerprint, pickled_used=True, + ) + response["result"]["extras"]["parameters"] = { + "dimensions": dimensions, + "space": space, + "hyperparams": hyperparams, + "Xi": Xi, + "Yi": Yi, + "extras": extras, + } + return json.loads(json_tricks.dumps(response)) +``` + +- [ ] **Step 5: Update the full-run path in `run` to pass the new kwargs** + +Find the existing call at `optimizer.py:154`: + +```python + response = process_result(result, optimizer, dimensions, cfg, extras, data, space) +``` + +Replace with: + +```python + response = process_result( + result, optimizer, dimensions, cfg, extras, data, space, + request_fingerprint=request_fingerprint, pickled_used=False, + ) +``` + +- [ ] **Step 6: Update `process_result` signature and body** + +Change the signature from: + +```python +def process_result(result, optimizer, dimensions, cfg, extras, data, space): +``` + +to: + +```python +def process_result(result, optimizer, dimensions, cfg, extras, data, space, + *, request_fingerprint, pickled_used): +``` + +Inside `process_result`, locate the existing line: + +```python + result_details = {"next": [], "models": [], "pickled": "", "extras": {}} +``` + +Immediately after the existing `add_version_info(result_details["extras"])` call near the end of `process_result`, set the flag: + +```python + result_details["extras"]["pickledUsed"] = pickled_used +``` + +Then replace the existing repack block: + +```python + if pickle_model: + result_details["pickled"] = pickleToString( + {"result": result, "next": result_details["next"], "optimizer": optimizer}, + get_crypto() + ) +``` + +with: + +```python + if pickle_model: + result_details["pickled"] = pack( + result=result, + next_points=result_details["next"], + optimizer=optimizer, + fingerprint=request_fingerprint, + crypto=get_crypto(), + ) +``` + +- [ ] **Step 7: Run the new test to verify it passes** + +Run: `python -m pytest tests/test_optimizer.py::test_pickled_used_flag_round_trip -v` +Expected: PASS. + +- [ ] **Step 8: Run the existing pickled tests to confirm no regression** + +Run: +```bash +python -m pytest tests/test_optimizer.py -k "pickled or selectedPoint" -v +``` +Expected: All targeted tests PASS. (One pre-existing unrelated failure `test_multi_objective_json_single_plots` is out of scope and not in this filter.) + +- [ ] **Step 9: Commit** + +```bash +git add optimizerapi/optimizer.py tests/test_optimizer.py +git commit -m "feat: delegate pickled cache to pickled_state and surface pickledUsed" +``` + +--- + +### Task 4: Binding equivalence test + +This is the test that enforces the north-star principle in §2 of the spec: the same request with and without `pickled` produces identical single-plot output. + +**Files:** +- Modify: `tests/test_optimizer.py` + +- [ ] **Step 1: Write the failing test** + +Add to `tests/test_optimizer.py` (placement: near the other multi-objective `selectedPoint` tests): + +```python +def test_equivalence_with_and_without_pickled_multi_objective(): + """Same selectedPoint, same data: pickled vs no-pickled produce identical single plots.""" + base_body = { + "data": sampleMultiObjective5DimData, + "optimizerConfig": sampleMultiObjective5DimConfig, + "extras": { + "graphs": ["single", "pareto"], + "graphFormat": "json", + "selectedPoint": [50, 833, 150, 60, "Whipped cream"], + "includeModel": "true", + }, + } + + # Seed: one full run to capture pickled. + seed = optimizer.run(body=copy.deepcopy({ + **base_body, + "extras": {**base_body["extras"], "selectedPoint": None}, + })) + pickled_value = seed["result"]["pickled"] + assert len(pickled_value) > 0 + + # Run A: no pickled (full retrain), with selectedPoint. + body_no_pickle = copy.deepcopy(base_body) + result_no_pickle = optimizer.run(body=body_no_pickle) + + # Run B: same request + pickled. + body_with_pickle = copy.deepcopy(base_body) + body_with_pickle["extras"]["pickled"] = pickled_value + result_with_pickle = optimizer.run(body=body_with_pickle) + + # Sanity: fast path actually engaged. + assert result_with_pickle["result"]["extras"]["pickledUsed"] is True + assert result_no_pickle["result"]["extras"]["pickledUsed"] is False + + # Compare every plot entry except the pickled string itself (which is + # not in plots) — identical plot ids and identical plot bodies. + plots_no_pickle = {p["id"]: p["plot"] for p in result_no_pickle["plots"]} + plots_with_pickle = {p["id"]: p["plot"] for p in result_with_pickle["plots"]} + assert set(plots_no_pickle) == set(plots_with_pickle) + for plot_id in plots_no_pickle: + assert plots_no_pickle[plot_id] == plots_with_pickle[plot_id], ( + f"divergence on plot {plot_id}" + ) +``` + +If `sampleMultiObjective5DimData` and `sampleMultiObjective5DimConfig` are not defined as module-level fixtures yet, define them near the top of the test file (mirroring the values from `scripts/sample-multi.curl`): + +```python +sampleMultiObjective5DimData = [ + {"xi": [16.7, 500, 250, 20, "None"], "yi": [-2, -17]}, + {"xi": [50, 833, 150, 60, "Whipped cream"], "yi": [-3, -6]}, + {"xi": [58.3, 22, 85, 6, "Frosting"], "yi": [-6, -25]}, +] + +sampleMultiObjective5DimConfig = { + "baseEstimator": "GP", + "acqFunc": "EI", + "initialPoints": 3, + "kappa": 1.96, + "xi": 2, + "space": [ + {"type": "continuous", "name": "Sugar", "from": 0, "to": 100}, + {"type": "continuous", "name": "Flour", "from": 0, "to": 1000}, + {"type": "discrete", "name": "Temperature", "from": 0, "to": 300}, + {"type": "discrete", "name": "Time", "from": 0, "to": 120}, + {"type": "category", "name": "Finish", "categories": ["None", "Frosting", "Whipped cream"]}, + ], + "constraints": [], +} +``` + +(Check first whether equivalents already exist further down `tests/test_optimizer.py` — search for `sampleMultiObjective5Dim`. If they do, reuse them and skip the redefinition.) + +- [ ] **Step 2: Run test to verify it passes** + +Run: `python -m pytest tests/test_optimizer.py::test_equivalence_with_and_without_pickled_multi_objective -v` +Expected: PASS. If it FAILS with a plot divergence, that is a real regression — investigate before continuing. Likely culprits: the fast path silently skipping `selectedPoint`, or `_get_brownie_bee_1d_plot_safe` being called with different state on the two paths. + +- [ ] **Step 3: Commit** + +```bash +git add tests/test_optimizer.py +git commit -m "test: assert equivalence of pickled and full-run plots" +``` + +--- + +### Task 5: Fingerprint-mismatch test + +**Files:** +- Modify: `tests/test_optimizer.py` + +- [ ] **Step 1: Write the failing test** + +Add to `tests/test_optimizer.py`: + +```python +def test_pickled_fingerprint_mismatch_falls_through(caplog): + """A pickled produced from one data set is ignored when data changes.""" + import logging as _logging + + seed = optimizer.run(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"includeModel": "true"}, + }) + pickled_value = seed["result"]["pickled"] + assert len(pickled_value) > 0 + + altered_data = sampleData + [{"xi": [100, 100, 100, "Mus"], "yi": [0.5]}] + + with caplog.at_level(_logging.WARNING, logger="optimizerapi.pickled_state"): + result = optimizer.run(body={ + "data": altered_data, + "optimizerConfig": sampleConfig, + "extras": {"includeModel": "true", "pickled": pickled_value}, + }) + + assert result["result"]["extras"]["pickledUsed"] is False + assert any("fingerprint_mismatch" in r.message for r in caplog.records) + # Response still valid — server fell through to a full run. + assert "next" in result["result"] + assert len(result["result"]["next"]) > 0 +``` + +- [ ] **Step 2: Run test to verify it passes** + +Run: `python -m pytest tests/test_optimizer.py::test_pickled_fingerprint_mismatch_falls_through -v` +Expected: PASS. + +- [ ] **Step 3: Commit** + +```bash +git add tests/test_optimizer.py +git commit -m "test: fingerprint mismatch falls through with warning" +``` + +--- + +### Task 6: Warn when PNG + `selectedPoint` are combined + +**Files:** +- Modify: `optimizerapi/optimizer.py` (`process_result`) +- Modify: `tests/test_optimizer.py` + +- [ ] **Step 1: Write the failing test** + +Add to `tests/test_optimizer.py`: + +```python +def test_selectedPoint_with_png_logs_warning_and_is_ignored(caplog): + import logging as _logging + + with caplog.at_level(_logging.WARNING, logger="optimizerapi.optimizer"): + result = optimizer.run(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": { + "graphFormat": "png", + "selectedPoint": [651, 56, 722, "Ræv"], + "includeModel": "false", + }, + }) + + assert any("selectedPoint ignored on png path" in r.message for r in caplog.records) + # No crash, response is valid. + assert "plots" in result + assert "next" in result["result"] +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m pytest tests/test_optimizer.py::test_selectedPoint_with_png_logs_warning_and_is_ignored -v` +Expected: FAIL — warning not emitted. + +- [ ] **Step 3: Add the warning in `process_result`** + +In `optimizerapi/optimizer.py`, inside `process_result`, right after the existing lines that read `graph_format` and `selected_point`: + +```python + graph_format = extras.get("graphFormat", "png") + # ... + pickle_model = json.loads(extras.get("includeModel", "true").lower()) + selected_point = extras.get("selectedPoint") +``` + +append: + +```python + if selected_point is not None and graph_format != "json": + logging.getLogger(__name__).warning( + "selectedPoint ignored on png path (graphFormat=%s)", graph_format + ) +``` + +(No other behavior change — the PNG branch already never reads `selected_point`.) + +- [ ] **Step 4: Run test to verify it passes** + +Run: `python -m pytest tests/test_optimizer.py::test_selectedPoint_with_png_logs_warning_and_is_ignored -v` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add optimizerapi/optimizer.py tests/test_optimizer.py +git commit -m "feat: warn when selectedPoint is sent with png graphFormat" +``` + +--- + +### Task 7: Warn when `includeModel=false` + `pickled` are combined + +**Files:** +- Modify: `optimizerapi/optimizer.py` (`run`) +- Modify: `tests/test_optimizer.py` + +- [ ] **Step 1: Write the failing test** + +Add to `tests/test_optimizer.py`: + +```python +def test_includeModel_false_with_pickled_logs_chain_break_warning(caplog): + import logging as _logging + + seed = optimizer.run(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"includeModel": "true"}, + }) + pickled_value = seed["result"]["pickled"] + + with caplog.at_level(_logging.WARNING, logger="optimizerapi.optimizer"): + second = optimizer.run(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"pickled": pickled_value, "includeModel": "false"}, + }) + + assert any("includeModel=false with extras.pickled" in r.message for r in caplog.records) + # Contract preserved: empty pickled returned. + assert second["result"]["pickled"] == "" +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `python -m pytest tests/test_optimizer.py::test_includeModel_false_with_pickled_logs_chain_break_warning -v` +Expected: FAIL. + +- [ ] **Step 3: Add the warning in `run`** + +In `optimizerapi/optimizer.py`, inside `run`, after the `extras` dict is read and after `pickled_input` is computed (Task 3 put `pickled_input = extras.get("pickled", "")` right above the `cached = ...` line), add: + +```python + if pickled_input: + include_model_str = str(extras.get("includeModel", "true")).lower() + if include_model_str == "false": + logging.getLogger(__name__).warning( + "includeModel=false with extras.pickled — next call will pay the full cost" + ) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `python -m pytest tests/test_optimizer.py::test_includeModel_false_with_pickled_logs_chain_break_warning -v` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add optimizerapi/optimizer.py tests/test_optimizer.py +git commit -m "feat: warn when includeModel=false drops a returnable pickled cache" +``` + +--- + +### Task 8: OpenAPI spec updates + +**Files:** +- Modify: `optimizerapi/openapi/specification.yml` + +- [ ] **Step 1: Tighten `extras.pickled` description** + +In `specification.yml`, find: + +```yaml + pickled: + description: Previously-pickled model state to skip expensive GP model retraining + type: string +``` + +Replace the description with: + +```yaml + pickled: + description: > + Opaque cache hint produced by a previous response. The server + validates it matches the current data / optimizerConfig; on + mismatch it is ignored and a full run is performed. + type: string +``` + +- [ ] **Step 2: Tighten `extras.selectedPoint` description** + +Find: + +```yaml + selectedPoint: + description: Override the highlight point in JSON single plots with explicit X-space coordinates + type: array +``` + +Replace the description with: + +```yaml + selectedPoint: + description: > + Override the highlight point in single plots with explicit + X-space coordinates. Honored only when graphFormat is "json"; + ignored on the PNG path. + type: array +``` + +- [ ] **Step 3: Add `pickledUsed` to the response result schema** + +Locate the `result` schema (around line 264) and find the inner `result:` block: + +```yaml + result: + type: object + properties: + expected_minimum: + ... + pickled: + type: string + next: + ... + models: + ... + extras: + type: object +``` + +Change the trailing `extras:` block to: + +```yaml + extras: + type: object + properties: + pickledUsed: + description: True iff the server reused the pickled cache hint for this response. + type: boolean +``` + +- [ ] **Step 4: Confirm OpenAPI still parses** + +Run: `python -m optimizerapi.server` and confirm it boots without YAML/OpenAPI errors. Hit `Ctrl-C` once the line `Serving on http://...` (or the Connexion startup line) appears. Then close. + +Alternative non-interactive check: + +```bash +python -c "import yaml; yaml.safe_load(open('optimizerapi/openapi/specification.yml'))" +``` + +Expected: no traceback. + +- [ ] **Step 5: Commit** + +```bash +git add optimizerapi/openapi/specification.yml +git commit -m "docs(openapi): tighten descriptions for pickled/selectedPoint, add pickledUsed" +``` + +--- + +### Task 9: Refresh legacy pickled tests + +The payload structure is now `{"fingerprint", "result", "next", "optimizer"}`. Two existing tests need their expectations refreshed. + +**Files:** +- Modify: `tests/test_optimizer.py` + +- [ ] **Step 1: Update `test_old_format_pickled_falls_back`** + +The current test (at the location shown by `grep -n "test_old_format_pickled_falls_back" tests/test_optimizer.py`) constructs an old-format list and expects fall-through. Under the new code path this now flows through `bad_structure`. The functional assertion still holds; add an explicit warning assertion and a `pickledUsed=False` assertion to make it precise. + +Replace the test body with: + +```python +def test_old_format_pickled_falls_back(caplog): + """A pickled payload that decrypts but isn't the new dict shape falls through.""" + import logging as _logging + + old_format_data = ["some", "old", "data"] + old_pickled = pickleToString(old_format_data, get_crypto()) + + with caplog.at_level(_logging.WARNING, logger="optimizerapi.pickled_state"): + result = optimizer.run(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"pickled": old_pickled, "includeModel": "false"}, + }) + + assert "result" in result + assert len(result["result"]["next"]) > 0 + assert result["result"]["extras"]["pickledUsed"] is False + assert any("bad_structure" in r.message for r in caplog.records) +``` + +- [ ] **Step 2: Update `test_pickled_response_is_dict_format`** + +The existing test asserts the response pickled is a dict with `result`, `next`, `optimizer`. Add `fingerprint` to the assertions so the new contract is locked. + +Locate the test and replace its tail-end assertions (after `unpickled = unpickleFromString(pickled_value, get_crypto())`) with: + +```python + assert isinstance(unpickled, dict), f"Expected dict, got {type(unpickled)}" + assert set(unpickled.keys()) >= {"fingerprint", "result", "next", "optimizer"} + assert isinstance(unpickled["fingerprint"], str) and len(unpickled["fingerprint"]) == 64 +``` + +- [ ] **Step 3: Update `test_invalid_pickled_falls_back`** + +This test sends raw garbage and expects fall-through. Add `pickledUsed` and warning assertions for symmetry: + +```python +def test_invalid_pickled_falls_back(caplog): + """An undecodable pickled string falls through to a full run.""" + import logging as _logging + + with caplog.at_level(_logging.WARNING, logger="optimizerapi.pickled_state"): + result = optimizer.run(body={ + "data": sampleData, + "optimizerConfig": sampleConfig, + "extras": {"pickled": "this_is_not_valid_pickled_data_at_all", "includeModel": "false"}, + }) + + assert "result" in result + assert len(result["result"]["next"]) > 0 + assert result["result"]["extras"]["pickledUsed"] is False + assert any("decrypt_failed" in r.message for r in caplog.records) +``` + +- [ ] **Step 4: Run the three updated tests** + +Run: +```bash +python -m pytest tests/test_optimizer.py::test_old_format_pickled_falls_back \ + tests/test_optimizer.py::test_pickled_response_is_dict_format \ + tests/test_optimizer.py::test_invalid_pickled_falls_back -v +``` +Expected: all PASS. + +- [ ] **Step 5: Commit** + +```bash +git add tests/test_optimizer.py +git commit -m "test: refresh legacy pickled tests for new payload shape and warnings" +``` + +--- + +### Task 10: Full suite + lint + +**Files:** none new. + +- [ ] **Step 1: Run the full test suite** + +Run: `python -m pytest` +Expected: All tests pass except the pre-existing `test_multi_objective_json_single_plots` (out of scope per the spec). If anything else fails, stop and investigate; do not paper over by adjusting tests. + +- [ ] **Step 2: Run flake8** + +Run: `flake8 . --max-line-length=127` +Expected: no warnings. + +- [ ] **Step 3: Smoke-check the server boots** + +Run: `python -m optimizerapi.server` in one shell; wait for the startup line; then `Ctrl-C`. Confirms the OpenAPI parses end-to-end and there are no import errors from the new module. + +- [ ] **Step 4: Final commit (if any cleanup landed during steps 1–3)** + +If steps 1–3 surfaced fixes: + +```bash +git add -A +git commit -m "chore: final cleanup after pareto extras redesign" +``` + +Otherwise skip. + +--- + +## Out of scope (do not implement here) + +- Server-side session cache with short-ID `extras.pickleHandle`. +- Migrating `includeModel` to a real boolean. +- A structured `pickledRejectedReason` in the response. +- Fixing `test_multi_objective_json_single_plots`. +- Modifications to ProcessOptimizer source or `_get_brownie_bee_1d_plot_safe`. diff --git a/docs/superpowers/specs/2026-05-18-pareto-extras-redesign-design.md b/docs/superpowers/specs/2026-05-18-pareto-extras-redesign-design.md new file mode 100644 index 0000000..2760b51 --- /dev/null +++ b/docs/superpowers/specs/2026-05-18-pareto-extras-redesign-design.md @@ -0,0 +1,130 @@ +# Pareto `extras` Redesign — Design + +**Date:** 2026-05-18 +**Branch:** `langdal/ai-handoff-pareto` +**Status:** Design (awaiting plan) +**Supersedes the `extras.pickled` / `extras.selectedPoint` decisions in:** `.sisyphus/plans/pareto-point-selection.md` + +## 1. Goal + +Lock in the contract for the three `extras` fields that drive the pareto-point UI flow — `selectedPoint`, `pickled`, `includeModel` — under a single north-star principle, and produce a stable, equivalence-checkable API before merging the `pareto` branch. + +## 2. North-star principle + +> **The pickled model is an *efficiency* feature only. The full round-trip (user clicks a pareto point → server re-renders single plots) MUST work without `pickled`, just slower.** + +Made formal: + +> **Equivalence guarantee.** For any request `R`, the response to `R` and to `R ∪ {extras.pickled: P}` are observably equivalent — ignoring `result.pickled` and timing — provided `P` was produced by an earlier response whose `data` and `optimizerConfig` matched `R`. + +Authoritative inputs are `data` and `optimizerConfig`. The server NEVER trusts state inside `pickled` to override them. The client MUST always send `data`. + +## 3. Contract + +### 3.1 `extras.selectedPoint` + +- **Shape:** unchanged. Raw X-space coordinates as a list mixing numbers and category strings, matching the `space` dimensions in order. Example: `[50, 833, 150, 60, "Whipped cream"]`. +- **Rationale for keeping raw coords (vs. an index into the prior `pareto_data.front_x_data`):** stateless, self-contained, no coupling to prior response shape, no dependency on deterministic re-derivation of the pareto. The UI already has the coords; sending them back is free. +- **Honored when:** `extras.graphFormat == "json"`. PNG path does not support `selectedPoint`. +- **PNG + `selectedPoint`:** a `warning` is logged; the field is ignored; the response is otherwise unchanged. No error. +- **OpenAPI:** description updated to state the `graphFormat: "json"` precondition explicitly. + +### 3.2 `extras.pickled` + +- **Shape:** opaque string blob, as today. Clients SHOULD treat it as opaque and round-trip it verbatim. +- **Server-side payload structure (Fernet-encrypted, pickled):** + ``` + { + "fingerprint": "", + "result": , + "next": , + "optimizer": , + } + ``` + No version field — the encryption-key rotation (see §5) makes prior payloads undecryptable, so no in-tree migration logic is required. +- **Fingerprint definition:** `sha256_hex(canonical_json({"data": data, "optimizerConfig": optimizerConfig}))`, where `data` and `optimizerConfig` are the raw top-level request fields. `canonical_json` means sorted keys, no whitespace, stable number representation. The exact serializer choice is an implementation detail and lives in `pickled_state.py`. Fingerprinting on the raw request fields (rather than on internally reconstructed `space` / `hyperparams` / `constraints`) keeps the contract independent of internal refactoring. +- **Validation flow on incoming `extras.pickled`:** + 1. Decrypt with current `PICKLE_KEY`. Failure → fall through. + 2. Unpickle to a dict. Wrong type / missing keys → fall through. + 3. Compare `payload["fingerprint"]` to the fingerprint recomputed from the current request. Mismatch → fall through. + 4. All pass → fast path: reuse `payload["result"]` and `payload["optimizer"]`; skip `optimizer.tell(...)`. +- **Fall-through behavior:** a single `warning` log line with a reason tag (`decrypt_failed`, `bad_structure`, `fingerprint_mismatch`), and a full run is performed. The response is otherwise equivalent to one without `extras.pickled`. + +### 3.3 `extras.includeModel` + +- **Behavior:** unchanged. Controls whether `result.pickled` is populated in the response. +- **Typing:** unchanged — stringly-typed `"true"`/`"false"` parsed via `json.loads(...lower())`. Tightening to a real boolean is a separate, out-of-scope ticket; existing clients depend on the string form. +- **`includeModel: "false"` + `extras.pickled` sent in:** accepted, documented. The server uses the fast path and returns an empty `result.pickled`. A `warning` is logged so the chain-break is debuggable. The client is asking for exactly what it asked for; no auto-override. + +### 3.4 New response field: `result.extras.pickledUsed` + +- **Type:** boolean. +- **Set to `True` iff** the fast path was taken (all four validation steps in §3.2 passed). +- **Purpose:** lets the UI tell whether its cache hint was honored without inferring from timing. Optional for callers to read. + +### 3.5 Response schema (recap of changes) + +- `result.extras.pickledUsed: boolean` added. +- No other response-shape changes. + +## 4. Code organization + +- New module: `optimizerapi/pickled_state.py`, exposing: + - `compute_fingerprint(data, optimizerConfig) -> str` + - `pack(result, next_points, optimizer, fingerprint, crypto) -> str` + - `unpack_if_valid(blob, expected_fingerprint, crypto) -> Optional[dict]` — returns the inner dict on success, `None` on any failure, and logs the reason on failure. +- `optimizerapi/optimizer.py`: + - The unpickle block at ~lines 105–129 collapses to a single call to `unpack_if_valid(...)`. The `if/else/except` chain that exists today is replaced. + - The pickle block at ~line 395 calls `pack(...)` with the request fingerprint computed earlier in `run`. + - In `process_result`, set `result_details["extras"]["pickledUsed"]` according to whether the fast path ran. + - Add `warning` logs for: + - PNG + `selectedPoint` (in `process_result`, where `selected_point` is read). + - `includeModel: "false"` + `extras.pickled` (in `run`, after the pickled path decision is made). + +The point of pulling `pickled_state.py` out is to keep `optimizer.py:run` focused on optimizer-lifecycle code, and to make the fingerprint logic and fall-through paths independently testable. + +## 5. Encryption-key rotation (operational) + +- Rotate `PICKLE_KEY` in any environment that may hold pre-redesign pickled blobs. Operationally this means: deploy with a new key. No in-tree migration code is required; old blobs simply fail to decrypt and the existing fall-through path takes over. +- There are no in-production users of the `pickled` feature at this point, so this is risk-free. +- Mention the rotation requirement in the PR description / release notes for downstream consumers. + +## 6. OpenAPI changes (`specification.yml`) + +- `extras.pickled` description tightened to: "Opaque cache hint produced by a previous response. The server validates it matches the current `data` / `optimizerConfig`; on mismatch it is ignored and a full run is performed." +- `extras.selectedPoint` description appended with: "Honored only when `graphFormat` is `\"json\"`. Ignored on the PNG path." +- `result.extras` gains an optional `pickledUsed: boolean`. + +## 7. Tests (new) + +- **Equivalence test (the binding test for §2).** Run a multi-objective request twice — once without `pickled`, once with the `pickled` from the first response — both with the same `selectedPoint`. Assert that all `single_*` / `objective_*` plot entries are byte-equal (or numerically equal within tolerance for any floating-point reductions). +- **Fingerprint mismatch.** Send `pickled` from a previous response together with a modified `data` array. Assert: response matches a no-`pickled` run for the new `data`; `result.extras.pickledUsed == False`; one `warning` logged with reason `fingerprint_mismatch`. +- **Decrypt-failure fall-through.** Send a garbage `pickled` string. Assert: full run executed; `pickledUsed == False`; warning logged with reason `decrypt_failed`. +- **Bad-structure fall-through.** Decrypt-able but missing keys. Assert fall-through with reason `bad_structure`. +- **`pickledUsed` round-trip.** Two-call sequence: first call sets `pickledUsed == False`; second call (with returned `pickled`) sets `pickledUsed == True`. +- **PNG + `selectedPoint`.** Send PNG + `selectedPoint`. Assert: no error; warning logged; PNG plots returned and unaffected by `selectedPoint` value. +- **`includeModel: "false"` + `pickled`.** Existing `test_pickled_consumption_skips_training` already exercises this. Extend it to also assert that a warning is logged. + +Existing tests for `selectedPoint` and old-format pickled should be reviewed: the old-format test (`test_old_format_pickled_falls_back`) likely still passes via the decrypt-failure or bad-structure path under the rotated key, but its assertions may need refreshing. + +## 8. Out of scope + +- Server-side session cache + short-ID `extras.pickleHandle` (deferred; the opaque-blob contract preserves a clean swap path). +- Migrating `includeModel` to a real boolean. +- Surfacing a structured `pickledRejectedReason` to the UI. Logs are sufficient until proven otherwise. +- Modifying `ProcessOptimizer` source. +- Changing `expected_minimum` computation. +- The `_get_brownie_bee_1d_plot_safe` workaround — it stays. +- The pre-existing `test_multi_objective_json_single_plots` failure. + +## 9. Acceptance criteria + +A reviewer can verify the work is done by: + +1. The equivalence test in §7 passes. +2. All fall-through paths log exactly one `warning` with the documented reason tag. +3. `result.extras.pickledUsed` correctly reflects fast-path vs. full-run on a two-call sequence. +4. `specification.yml` reflects the description and schema changes in §6. +5. `optimizer.py:run` no longer contains inline unpickle/repack logic — it delegates to `pickled_state.py`. +6. PNG path is unchanged in behavior; `selectedPoint` is logged-and-ignored on that path. +7. `python -m pytest` is green except for the documented pre-existing `test_multi_objective_json_single_plots` failure; `flake8 . --max-line-length=127` is clean. From 1bbeb68bfa92bcff7f197ce4e7970d96551c1882 Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Sat, 30 May 2026 08:18:32 +0000 Subject: [PATCH 08/14] docs: add ADR convention, initial records, and finish-branch flow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Establish ADRs under docs/adr/ as the only durable design doc kept on main, and add a vendor-neutral branch-finalize workflow. - docs/adr/: Nygard-style template, index/README, and two records — 0001 (pickled model as an advisory, fingerprint-validated cache hint, distilled from the pareto-extras redesign) and 0002 (the ADR / branch-only agent-docs convention itself, with the alternatives weighed). - AGENTS.md: vendor-neutral 'Decisions and finishing a branch' section — ADR policy plus the advisory branch-finalize flow (distil a decision into an ADR, keep plans/specs through review, remove them in a cleanup commit before merge). Surfaced by the agent, not mechanically enforced. - CLAUDE.md: points at the finish-branch skill and the new AGENTS section. - .claude/skills/finish-branch: encodes the flow, adapted to this repo's pytest/flake8/mypy checks; .gitignore narrowed to track skills while keeping local Claude settings ignored. Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude/skills/finish-branch/SKILL.md | 69 ++++++++++++++++++ .gitignore | 6 +- AGENTS.md | 20 ++++++ CLAUDE.md | 5 ++ docs/adr/0001-pickled-model-cache-contract.md | 71 +++++++++++++++++++ .../adr/0002-adrs-and-ephemeral-agent-docs.md | 50 +++++++++++++ docs/adr/README.md | 39 ++++++++++ docs/adr/template.md | 25 +++++++ 8 files changed, 284 insertions(+), 1 deletion(-) create mode 100644 .claude/skills/finish-branch/SKILL.md create mode 100644 docs/adr/0001-pickled-model-cache-contract.md create mode 100644 docs/adr/0002-adrs-and-ephemeral-agent-docs.md create mode 100644 docs/adr/README.md create mode 100644 docs/adr/template.md diff --git a/.claude/skills/finish-branch/SKILL.md b/.claude/skills/finish-branch/SKILL.md new file mode 100644 index 0000000..0b32a7f --- /dev/null +++ b/.claude/skills/finish-branch/SKILL.md @@ -0,0 +1,69 @@ +--- +name: finish-branch +description: Use when wrapping up, finalizing, or merging a feature branch in this repo. Distils the branch's agent-generated plans/specs into a durable Architecture Decision Record when warranted (with human approval), then reminds the user to remove the plans/specs in a cleanup commit before the usual tests/merge/cleanup steps. +--- + +# Finish branch + +Finalize a feature branch in this repo. The distinctive step is **distilling the +branch's agent-generated plans/specs into a durable ADR** when warranted, then +removing those plans/specs in a cleanup commit before merge — see `AGENTS.md` +and `docs/adr/README.md` for the policy. This is advisory: surface it to the +user, don't enforce it. + +Do not start until the feature work itself is complete and the user is wrapping +up the branch. + +## Checklist + +Create a TodoWrite item per step and work them in order. + +1. **Confirm scope.** Identify the branch and its diff against `main` + (`git log --oneline main..HEAD`, `git diff main...HEAD --stat`). Confirm with + the user that the feature is done and ready to finalize. + +2. **Gather the working artifacts.** Find the agent-generated plans/specs for + this branch (typically under `docs/superpowers/plans/` and + `docs/superpowers/specs/`; they may or may not be committed). Read them with + the commit messages and the diff. You now have the full "what and why." + +3. **Decide if an ADR is warranted.** Apply the bar in `docs/adr/README.md`: + write one only if a future maintainer would ask "why is it like this?" and + the code won't answer — an architectural choice, a reversal of a prior + decision, a cross-cutting convention, or a non-obvious trade-off. + **Most branches warrant none.** If none: say so, with a one-line reason, and + skip to step 6. + +4. **Draft the ADR.** Copy `docs/adr/template.md` to the next number + `docs/adr/NNNN-short-title.md`. Fill in Context / Decision / Consequences / + Alternatives considered. Keep it short; link to the key commit(s)/PR and to + any superseded ADR. Add a row to the index table in `docs/adr/README.md`. + +5. **Get human approval of the ADR.** Show the user the drafted ADR and ask them + to confirm or edit before proceeding. Judgment about what is architecturally + relevant stays with the human. Revise until approved. + +6. **Verify green.** Run the checks CI runs and the branch touched: + `python -m pytest`, `flake8 . --max-line-length=127`, and `mypy optimizerapi`. + Report results honestly; do not finalize over failures without the user's + say-so (a known, documented pre-existing failure is the user's call). + +7. **Clean up the plans/specs.** If they were committed on the branch, remove + them in a dedicated cleanup commit so they don't land on `main` — their + durable content (if any) is now in the ADR, the rest stays in git history. + Keep them through review; the natural time to remove is just before merge. + **Don't enforce this** — surface it: remind the user and confirm, rather than + blocking the merge if they'd rather keep them. + +8. **Finalize.** Commit the ADR (and code, if uncommitted) on the branch, then + open a PR or merge per the user's preference and delete the branch. If the + `superpowers:finishing-a-development-branch` skill is available, use it for + this git mechanics step. End git commit messages with the repo's required + `Co-Authored-By` trailer. + +## Notes + +- One ADR per branch is a smell — recording nothing is the common, healthy case. +- Don't invent a timeframe or follow-up the branch didn't actually create. +- Respect the repo's commit-signing setup; if it was temporarily disabled for a + history rewrite, restore it before the user signs. diff --git a/.gitignore b/.gitignore index 814de84..97d118f 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,8 @@ dist/ build/ *.egg .sisyphus -/.claude +# Ignore local Claude Code settings, but keep shared skills tracked. +/.claude/* +!/.claude/skills/ +/.claude/skills/* +!/.claude/skills/finish-branch/ diff --git a/AGENTS.md b/AGENTS.md index 07e47f3..cd23747 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,6 +27,26 @@ - When modifying the API, update `optimizerapi/openapi/specification.yml` and keep handler names consistent. - Write tests under `tests/` using `pytest` style; use `unittest.mock.patch` for external effects as in `tests/test_optimizer.py`. +## Decisions and finishing a branch + +- **ADRs are the only durable design doc on `main`.** Significant decisions live + as Architecture Decision Records under `docs/adr/` (Nygard format); see + `docs/adr/README.md` for the bar and the workflow. Write one only when a future + maintainer would ask "why is it like this?" and the code won't answer. Most + branches need none — one ADR per branch is a smell. +- **Agent-generated plans/specs are branch-only working artifacts.** The + spec/plan files produced while building (e.g. under `docs/superpowers/`) may + stay on the branch and in the PR for review, but are removed in a cleanup + commit before merge so they don't accumulate on `main`. Their durable content, + if any, is distilled into an ADR first. +- **Branch-finalize flow** (advisory — the agent surfaces it, nothing enforces + it): when wrapping up a branch, (1) decide whether the work warrants an ADR and + draft it with human approval, (2) run the CI checks (`python -m pytest`, + `flake8 . --max-line-length=127`, `mypy optimizerapi`), (3) remove the + branch's plans/specs in a cleanup commit, then (4) open the PR / merge. The + Claude Code `finish-branch` skill (`.claude/skills/finish-branch/`) encodes + this; this policy is vendor-neutral and applies under any agent framework. + ## Architecture This is an OpenAPI-first REST API wrapping [ProcessOptimizer](https://github.com/novonordisk-research/ProcessOptimizer) (Bayesian optimization). The API has a single main endpoint: `POST /optimizer`. diff --git a/CLAUDE.md b/CLAUDE.md index ea46917..cbd4978 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -3,3 +3,8 @@ This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. Source of truth for setup, workflow, style, and architecture: @AGENTS.md. + +When wrapping up or merging a feature branch, use the `finish-branch` skill +(`.claude/skills/finish-branch/`): distil any durable decision into an ADR under +`docs/adr/`, then clean up the branch's plans/specs before merge. See the +"Decisions and finishing a branch" section of @AGENTS.md. diff --git a/docs/adr/0001-pickled-model-cache-contract.md b/docs/adr/0001-pickled-model-cache-contract.md new file mode 100644 index 0000000..cd32565 --- /dev/null +++ b/docs/adr/0001-pickled-model-cache-contract.md @@ -0,0 +1,71 @@ +# 0001. Pickled model is an advisory, fingerprint-validated cache hint + +- **Status:** accepted +- **Date:** 2026-05-30 +- **Deciders:** Jakob Langdal + +## Context + +The pareto-point UI flow re-renders single plots whenever a user clicks a point +on the front. Re-running the Bayesian optimizer (GP `tell` / retraining) on +every such request is expensive, so the API lets a client round-trip an opaque +`extras.pickled` blob produced by a previous response to skip that work. + +The risk is that `pickled` carries optimizer state that could silently diverge +from the request's authoritative inputs (`data`, `optimizerConfig`) — a corrupt +blob, a stale one from a different dataset, or one encrypted under a rotated +`PICKLE_KEY`. If the server trusted that state, two requests with identical +inputs could return different results depending on whether a cache hint was +attached, which is impossible to debug from the client side. + +## Decision + +We will treat the pickled model as an **efficiency feature only**, never as +authoritative state. See `optimizerapi/pickled_state.py` and its use in +`optimizerapi/optimizer.py:run`. + +- **Equivalence guarantee.** For any request `R`, the responses to `R` and to + `R ∪ {extras.pickled: P}` are observably equivalent (ignoring `result.pickled` + and timing), provided `P` came from an earlier response whose `data` and + `optimizerConfig` matched `R`. The full round-trip MUST work without `pickled`, + just slower. The client always sends `data`; the server never reads inputs out + of the blob. +- **Fingerprint validation.** The blob stores + `sha256_hex(canonical_json({data, optimizerConfig}))`. On an incoming + `pickled`, the server decrypts → checks structure → compares the fingerprint + to the current request, and only then takes the fast path. Any failure falls + through to a full run with a single `warning` tagged `decrypt_failed`, + `bad_structure`, or `fingerprint_mismatch`. +- **Observability.** `result.extras.pickledUsed` (boolean) reports whether the + fast path was taken, so the UI need not infer it from timing. +- **No in-tree payload versioning.** Rotating `PICKLE_KEY` makes pre-existing + blobs undecryptable, and the decrypt-failure fall-through handles them — so no + migration code is carried. +- **Adjacent request semantics** fixed at the same time: `selectedPoint` is + honored only on the `graphFormat: "json"` path (logged-and-ignored on PNG), + and `includeModel: "false"` with `pickled` is honored verbatim (fast path, + empty `result.pickled`, warning logged) rather than auto-overridden. + +## Consequences + +- Identical inputs always produce identical results; `pickled` can only make a + response faster, never different. This is enforceable by an equivalence test. +- Fall-through is safe by construction, so a bad or stale blob degrades to a + correct (slower) answer instead of an error or wrong output. +- Fingerprinting on the raw request fields keeps the contract stable across + internal refactors of `space` / `hyperparams` / `constraints`. +- Cost: every fast path pays a decrypt + sha256 + structural check, and the blob + is opaque and bound to the current key — clients cannot reuse it across a key + rotation. Accepted as cheap relative to a GP refit. + +## Alternatives considered + +- **Trust optimizer state inside `pickled`.** Rejected: breaks the equivalence + guarantee and makes client-visible behavior depend on an opaque blob. +- **`selectedPoint` as an index into the prior response's `front_x_data`.** + Rejected in favor of raw X-space coordinates: stateless, no coupling to prior + response shape, no dependency on deterministically re-deriving the front. +- **A payload `version` field with in-tree migration.** Rejected: key rotation + plus fall-through already covers stale blobs without migration code. +- **Server-side session cache keyed by a short `pickleHandle`.** Deferred (out + of scope); the opaque-blob contract leaves a clean swap path if needed later. diff --git a/docs/adr/0002-adrs-and-ephemeral-agent-docs.md b/docs/adr/0002-adrs-and-ephemeral-agent-docs.md new file mode 100644 index 0000000..7f9bc97 --- /dev/null +++ b/docs/adr/0002-adrs-and-ephemeral-agent-docs.md @@ -0,0 +1,50 @@ +# 0002. Record decisions in ADRs; treat agent plans/specs as branch-only + +- **Status:** accepted +- **Date:** 2026-05-30 +- **Deciders:** Jakob Langdal + +## Context + +Agent-driven development (the superpowers brainstorming / writing-plans +workflow, and similar tools) produces a spec and a plan for essentially every +feature — this repo already carries several under `docs/superpowers/`. Committing +all of them to `main` accumulates ephemeral, quickly-stale documents that bury +the few decisions that actually matter. We wanted a durable record of _why_ the +code is shaped as it is, without the noise of _how_ each feature was built — and +the convention had to work across agent frameworks, not just one tool. + +## Decision + +We will keep **one** durable design document on `main`: Architecture Decision +Records under `docs/adr/` (Nygard format). + +- **Agent-generated plans/specs are working artifacts.** They may live on the + branch — and in the PR — during development and review, but are removed in a + cleanup commit before merge. They are not kept on `main`. +- **At branch finalization, distil any durable decision into an ADR**, with a + human approving it. Don't force one per branch; most branches record none. +- **This is advisory, not enforced.** Plans/specs are not git-ignored and there + is no CI gate; the agent surfaces the cleanup/distillation and the human + decides. +- Instructions live in `AGENTS.md` (vendor-neutral), imported by `CLAUDE.md`. + The Claude Code `finish-branch` skill encodes the procedure. + +## Consequences + +- `main` carries only high-signal decision records; plans/specs don't rot there. +- The convention is portable across agent frameworks via `AGENTS.md`. +- Reviewers still see the plans/specs in the PR, before the cleanup commit. +- Because it's advisory, a branch could merge with stale specs still committed, + or without an ADR that arguably should exist. Accepted: we preferred low + ceremony over enforcement for a small team. The `finish-branch` reminder is + the mitigation. + +## Alternatives considered + +- **Keep specs/plans committed permanently** (the original workflow default). + Rejected: accumulates noise and stale docs, and buries the "why". +- **Git-ignore specs/plans entirely** (never committed). Rejected: reviewers + wouldn't see them in the PR. We chose branch-visible + cleanup-before-merge. +- **Mechanically enforce the cleanup** (git-ignore, or a CI/hook gate). + Rejected: too much ceremony for a small team; an advisory reminder is enough. diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 0000000..a720dc0 --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,39 @@ +# Architecture Decision Records + +This folder is the durable record of significant decisions in this repo. Each +ADR captures one decision: the context that forced it, what was decided, the +consequences, and the alternatives that were rejected. + +ADRs are the **only** design document we keep on `main` long-term. The +agent-generated plans and specs produced while building (e.g. under +`docs/superpowers/`) may live on the branch for review, but are removed in a +cleanup commit before merge (see `AGENTS.md`). + +## When to write one + +Write an ADR when a future maintainer would ask _"why is it like this?"_ and the +code alone won't answer. Typical triggers: + +- An architectural choice with viable alternatives. +- Reversing or superseding an earlier decision. +- A cross-cutting convention or constraint. +- A trade-off that is non-obvious from the code. + +**Don't** write one for routine features, bug fixes, or refactors. Most branches +record no ADR — that's expected. One ADR per branch is a smell. + +## How + +1. Copy `template.md` to `NNNN-short-title.md` — `NNNN` is the next zero-padded + number. +2. Fill it in. Keep it short; link to code, PRs, or superseded ADRs. +3. Set `Status` (`accepted` once agreed). When a later ADR overrides this one, + set this one's status to `superseded by ADR-XXXX` and reference it. +4. Add a line to the index below. + +## Index + +| ADR | Title | Status | +| -------------------------------------------- | ---------------------------------------------------------------- | -------- | +| [0001](0001-pickled-model-cache-contract.md) | Pickled model is an advisory, fingerprint-validated cache hint | accepted | +| [0002](0002-adrs-and-ephemeral-agent-docs.md) | Record decisions in ADRs; treat agent plans/specs as branch-only | accepted | diff --git a/docs/adr/template.md b/docs/adr/template.md new file mode 100644 index 0000000..05d69b3 --- /dev/null +++ b/docs/adr/template.md @@ -0,0 +1,25 @@ +# NNNN. + +- **Status:** proposed | accepted | superseded by ADR-XXXX | deprecated +- **Date:** YYYY-MM-DD +- **Deciders:** + +## Context + +What forced this decision? The problem, constraints, and relevant facts. Enough +that a reader who wasn't there understands the pressure behind the choice. + +## Decision + +What we decided, stated plainly in the active voice ("We will …"). Link to the +key code, PR, or commit. + +## Consequences + +What becomes easier and what becomes harder as a result. Include the costs and +risks, not just the upside — breaking changes, follow-ups, things we accept. + +## Alternatives considered + +The options we rejected and, briefly, why. This is often the most valuable part +of the record. From ed4037d1e409d9191eacd84f6e93f6e2c7c76d97 Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Sat, 30 May 2026 08:53:21 +0000 Subject: [PATCH 09/14] ci: run the test workflow on the mise toolchain (Python 3.13 + uv) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The workflow pinned Python 3.9 and installed deps from requirements-freeze.txt, which the pyproject migration removed — so runtime deps (rq, cryptography, numpy, ProcessOptimizer) never installed and optimizerapi/types.py failed to import (NotRequired needs 3.11+). Replace setup-python with jdx/mise-action so CI provisions the same Python 3.13 + uv as local dev (mise.toml) and installs the project via 'uv pip install -e .[dev]'. Add a mypy step (now matches the AGENTS.md claim) and a .flake8 config excluding the local .venv/env so 'flake8 .' lints only project sources in CI and locally. Co-Authored-By: Claude Opus 4.8 (1M context) --- .flake8 | 5 +++++ .github/workflows/python-app.yml | 30 +++++++++++++++--------------- 2 files changed, 20 insertions(+), 15 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..bb82ebc --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +max-line-length = 127 +# Keep flake8's built-in excludes and skip the local virtualenv and build +# artifacts so `flake8 .` lints only project sources (matches CI). +extend-exclude = .venv,env,build,dist,*.egg-info diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index c75e6ee..80f522e 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -1,6 +1,5 @@ -# This workflow will install Python dependencies, run tests and lint with a single version of Python -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - +# Installs dependencies, lints, type-checks, and tests using the same +# toolchain as local development (mise-managed Python + uv; see mise.toml). name: Python application on: @@ -15,22 +14,23 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.9 - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - name: Set up toolchain with mise (Python + uv, matches dev env) + uses: jdx/mise-action@v2 with: - python-version: 3.9 + install: true + cache: true - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 pytest - if [ -f requirements-freeze.txt ]; then pip install -r requirements-freeze.txt; fi + # mise.toml provisions Python 3.13 + uv and creates .venv; install the + # project with its dev extras (pytest, flake8, mypy) into that venv. + run: mise exec -- uv pip install -e ".[dev]" - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + mise exec -- flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + mise exec -- flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Type-check with mypy + run: mise exec -- mypy optimizerapi - name: Test with pytest - run: | - pytest + run: mise exec -- python -m pytest From 6aa28f3ee2a1f09b27f8c64ceb15b6c65f523327 Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Sat, 30 May 2026 10:13:32 +0000 Subject: [PATCH 10/14] ci(docker): build on Python 3.13 with uv, drop arm64, cache layers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The image still targeted python:3.9-bullseye while pyproject now requires >=3.13, and the builder copied only pyproject.toml — so 'pip install .' could neither run on 3.9 nor build the project (setuptools needs README.md and the optimizerapi/ source). Rebuild the Dockerfile on python:3.13-slim-bookworm, install with uv (matching dev/CI), and copy the metadata + source the build actually needs, with a uv cache mount so the ProcessOptimizer git dependency isn't re-fetched each build. Workflow: build linux/amd64 only (arm64 was emulated under QEMU and dominated build time), add GHA layer caching (cache-from/to: type=gha), and bump the docker/* and checkout actions to current major versions. Add a .dockerignore so the local .venv (multi-GB) and other cruft stay out of the build context. Co-Authored-By: Claude Opus 4.8 (1M context) --- .dockerignore | 25 +++++++ .github/workflows/github-actions-docker.yaml | 22 +++--- Dockerfile | 79 ++++++++++++-------- 3 files changed, 83 insertions(+), 43 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..f265479 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,25 @@ +# Keep the build context small and reproducible: the image installs deps from +# pyproject.toml via uv, so local virtualenvs, VCS, tests, and docs are noise. +.git +.github +.venv +env +*.egg-info +__pycache__ +**/__pycache__ +*.pyc +.mypy_cache +.pytest_cache +.ruff_cache +tests +docs +scripts +.claude +.sisyphus +*.md +!README.md +mise.toml +.flake8 +pytest.ini +Dockerfile +.dockerignore diff --git a/.github/workflows/github-actions-docker.yaml b/.github/workflows/github-actions-docker.yaml index 88360db..076c1b2 100644 --- a/.github/workflows/github-actions-docker.yaml +++ b/.github/workflows/github-actions-docker.yaml @@ -20,11 +20,11 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Docker meta id: meta - uses: docker/metadata-action@v3 + uses: docker/metadata-action@v5 with: # list of Docker images to use as base name for tags images: | @@ -39,30 +39,30 @@ jobs: type=semver,pattern={{major}}.{{minor}} type=semver,pattern={{major}} type=sha - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - with: - platforms: all - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v6 with: context: . push: true - platforms: linux/amd64,linux/arm64 + # amd64 only — arm64 was emulated under QEMU and dominated build time. + platforms: linux/amd64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + # Reuse layers across runs so the heavy dependency layer is not rebuilt + # when only application code changes. + cache-from: type=gha + cache-to: type=gha,mode=max build-args: | GITHUB_REF_NAME GITHUB_SHA diff --git a/Dockerfile b/Dockerfile index 0ac7d5c..20cae2d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,42 +1,57 @@ +# syntax=docker/dockerfile:1 ARG GITHUB_REF_NAME=develop ARG GITHUB_SHA=local -# First stage -FROM python:3.9-bullseye AS builder -RUN python3 -m venv /opt/venv -ENV PATH="/opt/venv/bin:${PATH}" - -RUN pip install --upgrade pip - -COPY pyproject.toml . -RUN pip install . - -# Second stage - -FROM python:3.9-bullseye +# ---- Builder: resolve and install dependencies with uv (matches dev/CI) ---- +FROM python:3.13-slim-bookworm AS builder + +# git is required to install ProcessOptimizer from its git ref (see pyproject.toml). +RUN apt-get update \ + && apt-get install -y --no-install-recommends git \ + && rm -rf /var/lib/apt/lists/* + +# uv: fast, parallel resolver — the same tool used locally and in CI. +COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv + +ENV VIRTUAL_ENV=/opt/venv \ + UV_COMPILE_BYTECODE=1 \ + UV_LINK_MODE=copy +RUN uv venv "$VIRTUAL_ENV" +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +WORKDIR /src +# setuptools needs the readme (project metadata) and the package source +# (tool.setuptools.packages.find) to build the project; copy both before install. +COPY pyproject.toml README.md ./ +COPY optimizerapi/ ./optimizerapi/ +# Cache mount keeps uv's download/build cache across builds even when this layer +# is rebuilt, so the ProcessOptimizer git dependency isn't re-fetched every time. +RUN --mount=type=cache,target=/root/.cache/uv uv pip install . + +# ---- Runtime ---- +FROM python:3.13-slim-bookworm ARG GITHUB_REF_NAME ARG GITHUB_SHA -COPY --from=builder /opt/venv /opt/venv -WORKDIR /code -ENV VERSION=${GITHUB_REF_NAME} -ENV SHA=${GITHUB_SHA} -# add non-root user -RUN addgroup --system user && adduser --system --no-create-home --group user -RUN chown -R user:user /code && chmod -R 755 /code -RUN mkdir -p /code/mapplotlib - -USER user - -COPY --from=builder /pyproject.toml /code/pyproject.toml -#COPY version.txt /code -RUN echo "${VERSION}-${SHA}" > /code/version.txt +WORKDIR /code +ENV VERSION=${GITHUB_REF_NAME} \ + SHA=${GITHUB_SHA} \ + FLASK_ENV=production \ + MPLCONFIGDIR=/tmp/matplotlib \ + PATH=/opt/venv/bin:${PATH} + +# Dependencies come from the builder venv; the app itself runs from /code where +# the full source tree (incl. optimizerapi/openapi/specification.yml) is present. +COPY --from=builder /opt/venv /opt/venv +COPY pyproject.toml /code/pyproject.toml COPY optimizerapi/ /code/optimizerapi -ENV FLASK_ENV=production -ENV MPLCONFIGDIR=/tmp/mapplotlib +# Run as a non-root user with a version stamp baked in. +RUN addgroup --system user \ + && adduser --system --no-create-home --group user \ + && echo "${VERSION}-${SHA}" > /code/version.txt \ + && chown -R user:user /code -ENV PATH=/opt/venv/bin:${PATH} -VOLUME /code/matplotlib +USER user -CMD [ "python", "-m", "optimizerapi.server" ] \ No newline at end of file +CMD [ "python", "-m", "optimizerapi.server" ] From aa490b01c691d17de38d246d2dcf4e3f3165e69c Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Sat, 30 May 2026 10:52:55 +0000 Subject: [PATCH 11/14] build: install CPU-only torch to slim the image and speed builds torch arrives transitively via ProcessOptimizer and, from PyPI, drags in the full NVIDIA CUDA stack (~5 GB: cublas, cudnn, nccl, cusolver, ...). The API only does CPU inference, so this was pure build-time and image-size cost. Pin torch to PyTorch's CPU index via [tool.uv.index] + [tool.uv.sources]. uv source pins apply only to *direct* dependencies, so torch is also declared explicitly in [project.dependencies]. Resolution now yields torch==2.12.0+cpu with zero CUDA packages (88 -> 69 packages). Verified by building the image and booting it: GET /v1.0/health -> 200, /v1.0/ui/ -> 200, image 1.42 GB (was ~6 GB+), torch-2.12.0+cpu installed. Co-Authored-By: Claude Opus 4.8 (1M context) --- pyproject.toml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 9e2d2c7..bc2ecdb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,11 @@ dependencies = [ "cryptography>=44,<46", "waitress==2.1.2", "rq==2.0.0", + # torch arrives transitively via ProcessOptimizer, but uv source pins only + # apply to *direct* dependencies — so we declare it here to force the CPU-only + # build (see [tool.uv.sources] below). The API runs CPU inference; the default + # PyPI torch wheel drags in ~5 GB of CUDA libraries we never use. + "torch", ] [project.optional-dependencies] @@ -40,6 +45,17 @@ Homepage = "https://github.com/BoostV/process-optimizer-api" Repository = "https://github.com/BoostV/process-optimizer-api" Issues = "https://github.com/BoostV/process-optimizer-api/issues" +# Pull torch from PyTorch's CPU-only index so the install/image stays slim. +# `explicit = true` means only packages mapped to it in [tool.uv.sources] use +# this index; everything else still resolves from PyPI. +[[tool.uv.index]] +name = "pytorch-cpu" +url = "https://download.pytorch.org/whl/cpu" +explicit = true + +[tool.uv.sources] +torch = { index = "pytorch-cpu" } + [tool.setuptools.packages.find] include = ["optimizerapi*"] From 7284c5ca9f64c8d414ca8bf67d1b4f7eb42d7d79 Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Sat, 30 May 2026 18:14:57 +0000 Subject: [PATCH 12/14] docs: remove obsolete current-context handoff doc current-context.md was a temporary design-checkpoint handoff (2026-05-18) that self-documented 'Delete after the brainstorm produces an updated plan'. That brainstorm happened (docs/superpowers/.../pareto-extras-redesign*), the work landed, and the durable decision is now captured in ADR 0001. The doc also referenced things that no longer exist (.sisyphus/, result.json) and stale facts (29 tests, untracked CLAUDE.md). Co-Authored-By: Claude Opus 4.8 (1M context) --- .devcontainer-allowlist | 13 ++++++++ current-context.md | 74 ----------------------------------------- 2 files changed, 13 insertions(+), 74 deletions(-) create mode 100644 .devcontainer-allowlist delete mode 100644 current-context.md diff --git a/.devcontainer-allowlist b/.devcontainer-allowlist new file mode 100644 index 0000000..a3be5d8 --- /dev/null +++ b/.devcontainer-allowlist @@ -0,0 +1,13 @@ +registry-1.docker.io +auth.docker.io +production.cloudfront.docker.com +ghcr.io +pkg-containers.githubusercontent.com +deb.debian.org +security.debian.org +pypi.org +files.pythonhosted.org +download.pytorch.org +download-r2.pytorch.org +github.com +codeload.github.com diff --git a/current-context.md b/current-context.md deleted file mode 100644 index 234944f..0000000 --- a/current-context.md +++ /dev/null @@ -1,74 +0,0 @@ -# Current Context — pareto branch, design checkpoint - -> Temporary working doc. Captures where the `pareto` branch sits today (2026-05-18) and what the next brainstorm needs to settle. Delete after the brainstorm produces an updated plan. - -## TL;DR - -The `pareto-point-selection` plan was fully implemented and QA-approved on 2026-03-27, but the `pareto` branch has **not** been merged. A UI is being built in parallel and will inform changes to the API. Before merging, we need a design brainstorm focused on the **ergonomics of `selectedPoint` / `pickled` / `includeModel`** — driven by one north-star principle the user has now articulated: - -> **The pickled model is an *efficiency* feature only. The full round-trip (user clicks a pareto point → server re-renders single plots) must work without pickled, just slower.** - -This principle was not stated in the original plan and contradicts at least one of its design decisions (see "Tensions" below). - -## What's on the branch today - -- Two features shipped (commits `5e13f92` → `590b09e`): - - `extras.selectedPoint` — list of X-space coords; overrides `x_eval` in `_get_brownie_bee_1d_plot_safe` at 3 sites in `optimizer.py`. JSON graphFormat only. - - `extras.pickled` — round-trip string; payload format `{"result", "next", "optimizer"}`. Silent fallback to full run on bad/old-format input. -- OpenAPI schema updated with both fields under `extras`. -- `_get_brownie_bee_1d_plot_safe()` wraps a known ProcessOptimizer v1.1.1 bug where `get_Brownie_Bee_1d_plot(x_eval=...)` with categoricals creates a string-dtype numpy array. -- `includeModel` is unchanged from before: `extras.includeModel` is a **stringly-typed** boolean parsed via `json.loads(extras.get("includeModel", "true").lower())` at `optimizer.py:224`. Controls whether the response `pickled` field is populated. -- 29 tests pass. One pre-existing failure (`test_multi_objective_json_single_plots`) is unrelated and explicitly out of scope. - -Full trace lives under `.sisyphus/`: `plans/pareto-point-selection.md`, `notepads/pareto-point-selection/{decisions,issues,learnings}.md`, and `evidence/` (including `final-qa/qa-summary.txt`). - -## UI flow driving the redesign - -User clicks a point on the pareto plot → UI re-requests the optimizer endpoint with that point set as `selectedPoint` → wants single plots refreshed quickly. `pickled` is the speed lever, not a correctness requirement. - -## Loose ends in the working tree - -- `scripts/multi-blank-first-run.curl`, `scripts/multi-blank-second-run.curl` — untracked. 4-dim space (water/temperature/angle/color, categorical), first has empty `data`, second has 5 points × 2 objectives. Hand-rolled smoke tests, not bug repros. Different shape from the existing `sample-multi.curl` (5-dim) and `sample-multi-with-selection.curl`. -- `result.json` — untracked, 0 bytes. Placeholder. -- `CLAUDE.md` — untracked. -- Branch is unmerged; QA approval is six weeks old; behavior may need to shift based on UI findings. - -## Tensions to resolve in the brainstorm - -These are the concrete things to argue out — not yet decided. - -1. **"Pickled overrides data" vs "pickled is efficiency only"** - Original plan decision #23 said pickled overrides the request `data` field. The current implementation actually passes `data` through to `process_result` regardless, so the contract is unclear. Under the new principle, the same request *with* and *without* `extras.pickled` should produce equivalent output — just at different speeds. That means `data` must be authoritative and the client should always send it. Confirm and document; possibly enforce. - -2. **`includeModel=false` + `extras.pickled` sent in** - Today this consumes the incoming state but returns an empty `pickled` field — silently breaking the next iteration's fast path. Options: warn, auto-override to true, forbid, or accept as-is and document. - -3. **`includeModel` typing** - String `"true"`/`"false"` parsed via `json.loads`. Cheap to fix to a real boolean in the OpenAPI spec; would touch every existing client. Decide whether to bundle with the redesign or leave alone. - -4. **`selectedPoint` as raw coordinates vs index** - UI currently has to read `pareto_data.front_x_data`, pick an element, and round-trip the coordinate list. An index reference would be tighter but couples the request to the prior response shape. Open. - -5. **Pickled payload size and shape** - The pickled string is large enough that `sample-multi-with-selection.curl` documents it as "paste manually". Round-tripping it through the browser is workable but ugly. A server-side session cache + short ID is an option; not requested, but worth flagging since the UI will surface this pain. - -6. **PNG path has no `selectedPoint` support** - Deliberate exclusion in the original plan. If the UI ever wants PNG fallback, this becomes a gap. - -## Out of scope for the brainstorm (do not reopen) - -- Fixing the pre-existing `test_multi_objective_json_single_plots` failure. -- Modifying `ProcessOptimizer` source. -- Changing `expected_minimum` computation. -- The `_get_brownie_bee_1d_plot_safe` workaround — it works. - -## Open questions for the user (to bring into the brainstorm) - -- What does the UI need from a *failed* fast path? Silent fallback (current), explicit error, or a "stale" flag in the response? -- Is the long-term plan to keep optimizer state strictly client-managed (stateless server), or is a session/cache acceptable? -- Are there UI affordances already designed that lock us into a particular extras shape (e.g., is the UI already sending `selectedPoint` as coordinates)? -- What's the merge target / timeline once the brainstorm settles? - -## Suggested next move - -Run `superpowers:brainstorming` against this doc as the seed. Goal: produce a revised plan that replaces the relevant TODOs in `.sisyphus/plans/pareto-point-selection.md` (or a fresh plan file) and clarifies the principle above in writing before any more code lands. From 13ae0ae3152fee899f4dd2ae415f5cb0712d260c Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Sat, 30 May 2026 18:17:39 +0000 Subject: [PATCH 13/14] docs(adr): record CPU-only torch decision (ADR 0003) Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/adr/0003-cpu-only-torch.md | 51 +++++++++++++++++++++++++++++++++ docs/adr/README.md | 1 + 2 files changed, 52 insertions(+) create mode 100644 docs/adr/0003-cpu-only-torch.md diff --git a/docs/adr/0003-cpu-only-torch.md b/docs/adr/0003-cpu-only-torch.md new file mode 100644 index 0000000..521861f --- /dev/null +++ b/docs/adr/0003-cpu-only-torch.md @@ -0,0 +1,51 @@ +# 0003. Pin torch to the CPU-only wheel + +- **Status:** accepted +- **Date:** 2026-05-30 +- **Deciders:** Jakob Langdal + +## Context + +ProcessOptimizer (a core dependency, installed from a git ref) pulls `torch` in +transitively. On Linux, the default `torch` wheel on PyPI is the CUDA build, +which drags in the full NVIDIA CUDA stack (`nvidia-cublas`, `cudnn`, `nccl`, +`cusolver`, `cusparse`, ...) — roughly 5 GB. The API only ever runs CPU +inference, so none of that is used. It was the dominant cost behind slow Docker +builds and produced a ~6 GB image. + +## Decision + +We will pin `torch` to PyTorch's CPU-only index. See `pyproject.toml` +(`[[tool.uv.index]]` `pytorch-cpu` + `[tool.uv.sources]`) and commit `aa490b0`. + +A subtlety drove the shape of this: **uv's `[tool.uv.sources]` index pins apply +only to _direct_ dependencies, not transitive ones** (verified empirically). So +we also declare `torch` explicitly in `[project.dependencies]` (unconstrained, +so uv unifies it with ProcessOptimizer's requirement) and map it to the +`pytorch-cpu` index via `[tool.uv.sources]`. Resolution then yields +`torch==2.12.0+cpu` with zero CUDA packages. + +## Consequences + +- Image drops from ~6 GB to ~1.42 GB; builds and CI are markedly faster. The fix + applies uniformly to the Docker image, CI, and local dev because all three + install via uv from the same `pyproject.toml`. +- The build now depends on `download.pytorch.org` **and** `download-r2.pytorch.org` + (PyTorch redirects the actual wheel/metadata blobs to its R2 backend) being + reachable — relevant only where egress is filtered. +- `torch` is now a direct dependency we must keep loosely in step with what + ProcessOptimizer expects; leaving it unconstrained lets uv resolve the unified + version. +- If GPU inference is ever wanted, this pin must be revisited. + +## Alternatives considered + +- **Accept the default CUDA `torch`.** Rejected: ~5 GB of libraries we never use, + for a CPU-only workload. +- **`[tool.uv.sources]` pin without declaring `torch` directly.** Rejected: source + pins don't apply to transitive dependencies, so it had no effect on resolution + (confirmed by testing). +- **Install CPU `torch` only inside the Dockerfile** (e.g. `uv pip install torch + --index-url .../cpu` before the project). Rejected: it would slim the image but + leave dev and CI installs on the CUDA build. The `pyproject.toml` approach fixes + all three in one place. diff --git a/docs/adr/README.md b/docs/adr/README.md index a720dc0..264e26d 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -37,3 +37,4 @@ record no ADR — that's expected. One ADR per branch is a smell. | -------------------------------------------- | ---------------------------------------------------------------- | -------- | | [0001](0001-pickled-model-cache-contract.md) | Pickled model is an advisory, fingerprint-validated cache hint | accepted | | [0002](0002-adrs-and-ephemeral-agent-docs.md) | Record decisions in ADRs; treat agent plans/specs as branch-only | accepted | +| [0003](0003-cpu-only-torch.md) | Pin torch to the CPU-only wheel (slim image, faster builds) | accepted | From 840cac71f80c7f0bd7b62ae3e37faaef31cfa456 Mon Sep 17 00:00:00 2001 From: Jakob Langdal Date: Sat, 30 May 2026 18:21:01 +0000 Subject: [PATCH 14/14] docs: remove branch plans/specs before merge Per the ADR convention (ADR 0002 / AGENTS.md), agent-generated plans and specs are branch-only working artifacts kept through review and removed before merge. The durable decision from the pareto-extras redesign is captured in ADR 0001; the 5-phase audit plans' outcomes live in the code and commit history. Content remains recoverable from git history. Removed: - docs/superpowers/plans/2026-05-18-audit-{overview,phase-1..5}.md - docs/superpowers/plans/2026-05-18-pareto-extras-redesign.md - docs/superpowers/specs/2026-05-18-pareto-extras-redesign-design.md Co-Authored-By: Claude Opus 4.8 (1M context) --- .../plans/2026-05-18-audit-overview.md | 45 - .../2026-05-18-audit-phase-1-security.md | 698 ----------- .../plans/2026-05-18-audit-phase-2-logging.md | 417 ------- ...2026-05-18-audit-phase-3-boundary-types.md | 648 ---------- ...8-audit-phase-4-refactor-process-result.md | 1050 ----------------- .../plans/2026-05-18-audit-phase-5-cleanup.md | 646 ---------- .../2026-05-18-pareto-extras-redesign.md | 993 ---------------- ...026-05-18-pareto-extras-redesign-design.md | 130 -- 8 files changed, 4627 deletions(-) delete mode 100644 docs/superpowers/plans/2026-05-18-audit-overview.md delete mode 100644 docs/superpowers/plans/2026-05-18-audit-phase-1-security.md delete mode 100644 docs/superpowers/plans/2026-05-18-audit-phase-2-logging.md delete mode 100644 docs/superpowers/plans/2026-05-18-audit-phase-3-boundary-types.md delete mode 100644 docs/superpowers/plans/2026-05-18-audit-phase-4-refactor-process-result.md delete mode 100644 docs/superpowers/plans/2026-05-18-audit-phase-5-cleanup.md delete mode 100644 docs/superpowers/plans/2026-05-18-pareto-extras-redesign.md delete mode 100644 docs/superpowers/specs/2026-05-18-pareto-extras-redesign-design.md diff --git a/docs/superpowers/plans/2026-05-18-audit-overview.md b/docs/superpowers/plans/2026-05-18-audit-overview.md deleted file mode 100644 index 4f80b09..0000000 --- a/docs/superpowers/plans/2026-05-18-audit-overview.md +++ /dev/null @@ -1,45 +0,0 @@ -# Codebase Audit — 5-Phase Improvement Plan - -This is the index for the audit-driven improvements identified after the Pareto extras redesign landed. The audit covered structure, idioms, typing, security, logging, and tests. Findings are split into five phases that each ship independent, testable software. - -## Ordering rationale - -The phases are ordered by risk × dependency: - -1. **Security & dependency hygiene** — low blast radius, highest urgency. CVE bumps + fix auth issues. Lays no traps for later phases. -2. **Logging migration** — mechanical sweep; produces no behavior change but cleans up the surface that Phase 1 partly touched. -3. **Boundary types** — adds `TypedDict`s and a static type checker. Acts as a safety net for Phase 4. -4. **Refactor `process_result`** — the biggest behavior-preserving change. Splits ~250 lines into focused helpers. Types from Phase 3 catch mistakes that would otherwise need runtime to surface. -5. **Opportunistic cleanup** — small wins (fixtures, idiomatic Python, dead code) that benefit from the structural work above. - -Each phase has its own plan file in this directory. - -## Phase plans - -| # | Plan | Touches | -| - | --------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------ | -| 1 | [`2026-05-18-audit-phase-1-security.md`](./2026-05-18-audit-phase-1-security.md) | `auth.py`, `securepickle/`, `pyproject.toml` | -| 2 | [`2026-05-18-audit-phase-2-logging.md`](./2026-05-18-audit-phase-2-logging.md) | `server.py`, `auth.py`, `secure.py`, `optimizer_handler.py`, `optimizer.py` | -| 3 | [`2026-05-18-audit-phase-3-boundary-types.md`](./2026-05-18-audit-phase-3-boundary-types.md) | new `types.py`, all module signatures, mypy config | -| 4 | [`2026-05-18-audit-phase-4-refactor-process-result.md`](./2026-05-18-audit-phase-4-refactor-process-result.md) | `optimizer.py`, new `plot_emitters.py` | -| 5 | [`2026-05-18-audit-phase-5-cleanup.md`](./2026-05-18-audit-phase-5-cleanup.md) | tests, `optimizer_handler.py`, `auth.py` | - -## Out of scope across all phases - -- Flask 2 → 3 / Connexion 2 → 3 migration. Connexion 3 is a rewrite (async) — this is its own project, not a cleanup. Phase 1 limits the dependency bumps to libraries that don't ripple through the framework. -- ProcessOptimizer source changes. -- Switching auth providers; redesigning the API contract; multi-tenant features. -- Performance work (async/background queue model). Phase 5 notes the issue but does not fix it. - -## Acceptance for the audit as a whole - -When all five phases are done: - -- `flake8 optimizerapi tests --max-line-length=127` is fully clean (no pre-existing exceptions remain). -- `mypy optimizerapi` (or `pyright`) is clean. -- `python -m pytest` is 43+ passed (no regressions; new tests for security fixes). -- `optimizer.py` is under ~250 lines; `process_result` is under ~80 lines. -- `auth.py` uses `secrets.compare_digest` and never prints tokens. -- `cryptography` is on a current major. -- No `print()` calls in any module under `optimizerapi/`. -- The OpenAPI surface is unchanged (response shape and field names stay the same). diff --git a/docs/superpowers/plans/2026-05-18-audit-phase-1-security.md b/docs/superpowers/plans/2026-05-18-audit-phase-1-security.md deleted file mode 100644 index 282cf53..0000000 --- a/docs/superpowers/plans/2026-05-18-audit-phase-1-security.md +++ /dev/null @@ -1,698 +0,0 @@ -# Audit Phase 1 — Security & Dependency Hygiene - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Bump out-of-date crypto dependencies, eliminate auth-related anti-patterns (timing-attack-vulnerable equality, token-logging, default-API-key acceptance), and stop the `securepickle` module from mutating `os.environ` as a side effect. - -**Architecture:** Single-shot changes to three files: `pyproject.toml` (dependency bumps), `optimizerapi/auth.py` (constant-time comparison + fail-closed + drop token log), `optimizerapi/securepickle/secure.py` (`is None` + no env mutation + warning). One new test module added: `tests/test_auth.py`. No API contract changes. - -**Tech Stack:** Python 3.13 (just bumped), `cryptography`, Flask 2.2.x, connexion 2.14.x, pytest. `secrets.compare_digest` from stdlib for constant-time string comparison. - -**Audit reference:** §5 of the audit findings. Specifically `cryptography==3.4.7` (2021, has CVEs), `apikey_handler`'s `==` comparison, the `print(access_token)` line in `auth.py:31`, and the `os.environ` mutation in `securepickle/secure.py`. - ---- - -## File Map - -- **Modify:** `pyproject.toml` — bump `cryptography`, `jsonschema`. -- **Modify:** `optimizerapi/auth.py` — `secrets.compare_digest`, drop token print, fail-closed default behavior. -- **Modify:** `optimizerapi/securepickle/secure.py` — `is None`, drop `os.environ` mutation, use `logging` for the no-key-set message. -- **Create:** `tests/test_auth.py` — unit tests for `apikey_handler` covering correct key, wrong key, and fail-closed default. - -All work happens on branch `langdal/ai-restructure`. Test commands run via the existing `env/` venv (Python 3.13.13). - ---- - -### Task 1: Bump `cryptography` to a current major - -**Files:** -- Modify: `pyproject.toml:32` - -- [ ] **Step 1: Check the dependency tree for hidden cryptography constraints** - -Run: -```bash -env/bin/pip show python-keycloak python-jose | grep -i "Requires\|cryptography" -``` -Expected: lists what other packages need from `cryptography`. Note the highest minor version implied by any constraint. - -- [ ] **Step 2: Edit `pyproject.toml`** - -Find the line: -``` - "cryptography==3.4.7", -``` - -Replace with: -``` - "cryptography>=44,<46", -``` - -The lower bound is the current LTS series; the upper guards against major-version surprises while still letting patch updates land. - -- [ ] **Step 3: Reinstall** - -Run: -```bash -env/bin/pip install -e ".[dev]" 2>&1 | tail -5 -``` -Expected: `Successfully installed ... cryptography-44.x.x ...` and no error. - -- [ ] **Step 4: Run securepickle tests** - -Run: -```bash -env/bin/python -m pytest tests/test_securepickle.py -v -``` -Expected: PASS. - -- [ ] **Step 5: Run the full suite** - -Run: -```bash -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: 43 passed. - -- [ ] **Step 6: Commit** - -```bash -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -am "$(cat <<'EOF' -build: bump cryptography to current major (>=44) - -cryptography 3.4.7 is from 2021 and has known CVEs. Fernet API surface -has not changed across 3.x→44.x so this is a drop-in bump. - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - -If Step 3 surfaces an irreconcilable conflict (some dep pins `cryptography=4.21,<5", -``` - -- [ ] **Step 2: Reinstall** - -```bash -env/bin/pip install -e ".[dev]" 2>&1 | tail -5 -``` -Expected: jsonschema upgraded, no errors. - -- [ ] **Step 3: Run the full suite** - -```bash -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: 43 passed. - -If pytest fails, revert the bump (`git checkout pyproject.toml`) and pin to the highest minor that works. Connexion 2.14.2 uses jsonschema for OpenAPI validation — a bad bump shows up immediately as a validation error on every request. - -- [ ] **Step 4: Commit** - -```bash -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -am "$(cat <<'EOF' -build: bump jsonschema to 4.21+ - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -### Task 3: Constant-time API-key comparison - -**Files:** -- Modify: `optimizerapi/auth.py:44-55` -- Create: `tests/test_auth.py` - -- [ ] **Step 1: Write the failing test** - -Create `tests/test_auth.py`: - -```python -"""Tests for the API-key handler.""" - -from unittest.mock import patch - - -def test_correct_apikey_returns_scope_dict(): - """A request with the configured API key returns a scope dict.""" - with patch("optimizerapi.auth.AUTH_API_KEY", "secret-key"), \ - patch("optimizerapi.auth.AUTH_SERVER", None): - from optimizerapi import auth - result = auth.apikey_handler("secret-key") - assert result == {"scope": []} - - -def test_wrong_apikey_returns_none(): - """An incorrect API key is rejected.""" - with patch("optimizerapi.auth.AUTH_API_KEY", "secret-key"), \ - patch("optimizerapi.auth.AUTH_SERVER", None): - from optimizerapi import auth - result = auth.apikey_handler("not-the-key") - assert result is None - - -def test_apikey_handler_uses_constant_time_comparison(): - """The handler must compare keys via secrets.compare_digest.""" - from optimizerapi import auth - import secrets - - with patch("optimizerapi.auth.AUTH_API_KEY", "secret-key"), \ - patch("optimizerapi.auth.AUTH_SERVER", None), \ - patch.object(secrets, "compare_digest", wraps=secrets.compare_digest) as mock_cmp: - auth.apikey_handler("secret-key") - assert mock_cmp.called, "apikey_handler must use secrets.compare_digest" -``` - -- [ ] **Step 2: Run test to verify some fail** - -Run: -```bash -env/bin/python -m pytest tests/test_auth.py -v -``` -Expected: the first two PASS (current `==` logic works for those), the third FAILS with `AssertionError: apikey_handler must use secrets.compare_digest`. - -- [ ] **Step 3: Edit `optimizerapi/auth.py`** - -At the top of the file, the existing imports are: - -```python -import os -from keycloak import KeycloakOpenID -``` - -Add `secrets` and `logging`: - -```python -import logging -import os -import secrets - -from keycloak import KeycloakOpenID -``` - -Then replace the existing `apikey_handler`: - -```python -def apikey_handler(access_token) -> dict: - """Verify API key based on environment variable - - Returns - ------- - dict - a dictionary containing sub and scope - None in case of invalid token - """ - if not AUTH_SERVER and AUTH_API_KEY == access_token: - return {"scope": []} - return None -``` - -with: - -```python -def apikey_handler(access_token: str) -> dict | None: - """Verify the API key passed by the client. - - Returns - ------- - dict - ``{"scope": []}`` if the supplied key matches the configured - ``AUTH_API_KEY``. - None - If the key is wrong, or if no static-key auth is configured. - """ - if AUTH_SERVER: - # OIDC is configured; static-key path is disabled. - return None - expected = AUTH_API_KEY.encode("utf-8") - provided = (access_token or "").encode("utf-8") - if secrets.compare_digest(expected, provided): - return {"scope": []} - return None -``` - -- [ ] **Step 4: Run tests** - -Run: -```bash -env/bin/python -m pytest tests/test_auth.py -v -``` -Expected: all 3 PASS. - -Run the full suite: -```bash -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: 46 passed (43 + 3 new). - -- [ ] **Step 5: Commit** - -```bash -git add optimizerapi/auth.py tests/test_auth.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -fix(auth): use secrets.compare_digest for API-key comparison - -Constant-time comparison protects against timing-side-channel attacks -when the configured AUTH_API_KEY is non-trivial. - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -### Task 4: Remove access-token logging in `token_info` - -**Files:** -- Modify: `optimizerapi/auth.py:22-41` - -- [ ] **Step 1: Edit `optimizerapi/auth.py`** - -Replace the existing `token_info`: - -```python -def token_info(access_token) -> dict: - """Verify token with authentication server - - Returns - ------- - dict - a dictionary containing sub and scope - None in case of invalid token - """ - print(access_token) - if not AUTH_SERVER: - return {"scope": []} - token = access_token - token_data = keycloak_openid.introspect(token) - if "active" in token_data and token_data["active"]: - print("OK") - return token_data - print("NOT OK") - print(token_data) - return None -``` - -with: - -```python -_LOG = logging.getLogger(__name__) - - -def token_info(access_token: str) -> dict | None: - """Verify a bearer token against the configured Keycloak server. - - Returns - ------- - dict - Token data from Keycloak introspection when the token is active. - If no OIDC server is configured, returns ``{"scope": []}``. - None - If the server reports the token as inactive. - """ - if not AUTH_SERVER: - return {"scope": []} - token_data = keycloak_openid.introspect(access_token) - if token_data.get("active"): - _LOG.debug("token accepted") - return token_data - _LOG.warning("token rejected by Keycloak") - return None -``` - -The four `print` lines (including the one that leaked the raw access token) are gone. `_LOG.debug` / `_LOG.warning` use the standard logger; the token itself is never logged. - -- [ ] **Step 2: Run the full suite** - -```bash -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: 46 passed. - -- [ ] **Step 3: Verify no `print` remains in `auth.py`** - -Run: -```bash -grep -n "print(" optimizerapi/auth.py -``` -Expected: no output. - -- [ ] **Step 4: Commit** - -```bash -git add optimizerapi/auth.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -fix(auth): stop logging raw access tokens to stdout - -token_info previously printed the access_token on every request, which -leaks bearer credentials to whatever scrapes the API server's stdout. -Replaced with structured debug/warning logs that don't reference the -token value. - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -### Task 5: Fail closed when neither auth mechanism is configured - -The current default (`AUTH_API_KEY = "none"`, `AUTH_SERVER = None`) means a request with `?apikey=none` succeeds. Convenient in dev, dangerous in prod if either env var is dropped on deploy. - -**Files:** -- Modify: `optimizerapi/auth.py` -- Modify: `tests/test_auth.py` - -- [ ] **Step 1: Add a failing test** - -Append to `tests/test_auth.py`: - -```python -def test_default_apikey_value_is_rejected_in_production_mode(): - """When AUTH_API_KEY is the default 'none' and FLASK_ENV is 'production', - even matching the default value must be rejected.""" - with patch("optimizerapi.auth.AUTH_API_KEY", "none"), \ - patch("optimizerapi.auth.AUTH_SERVER", None), \ - patch("optimizerapi.auth.FLASK_ENV", "production"): - from optimizerapi import auth - result = auth.apikey_handler("none") - assert result is None - - -def test_default_apikey_value_is_accepted_in_development_mode(): - """In development mode (FLASK_ENV != 'production'), the legacy - behaviour of accepting AUTH_API_KEY='none' is preserved for ergonomics.""" - with patch("optimizerapi.auth.AUTH_API_KEY", "none"), \ - patch("optimizerapi.auth.AUTH_SERVER", None), \ - patch("optimizerapi.auth.FLASK_ENV", "development"): - from optimizerapi import auth - result = auth.apikey_handler("none") - assert result == {"scope": []} -``` - -- [ ] **Step 2: Run tests to confirm one fails** - -Run: -```bash -env/bin/python -m pytest tests/test_auth.py -v -``` -Expected: `test_default_apikey_value_is_rejected_in_production_mode` FAILS (current code accepts it). The other new test PASSES. - -- [ ] **Step 3: Edit `optimizerapi/auth.py`** - -In the module-level constants block, add `FLASK_ENV`: - -Find: -```python -AUTH_API_KEY = os.getenv("AUTH_API_KEY", "none") -AUTH_SERVER = os.getenv("AUTH_SERVER", None) -``` - -Replace with: -```python -AUTH_API_KEY = os.getenv("AUTH_API_KEY", "none") -AUTH_SERVER = os.getenv("AUTH_SERVER", None) -FLASK_ENV = os.getenv("FLASK_ENV", "development") - -_DEFAULT_APIKEY = "none" # sentinel: matches AUTH_API_KEY default; never accept in production -``` - -Update `apikey_handler`: - -```python -def apikey_handler(access_token: str) -> dict | None: - """Verify the API key passed by the client. - - Returns - ------- - dict - ``{"scope": []}`` if the supplied key matches the configured - ``AUTH_API_KEY``. - None - If the key is wrong, OIDC is configured (OIDC handles this path), - or the operator has not configured a real key in production. - """ - if AUTH_SERVER: - return None - # In production we refuse the unconfigured default value, even if it - # technically matches the request — operators should set a real key. - if FLASK_ENV == "production" and AUTH_API_KEY == _DEFAULT_APIKEY: - return None - expected = AUTH_API_KEY.encode("utf-8") - provided = (access_token or "").encode("utf-8") - if secrets.compare_digest(expected, provided): - return {"scope": []} - return None -``` - -- [ ] **Step 4: Run tests** - -```bash -env/bin/python -m pytest tests/test_auth.py -v -``` -Expected: all 5 PASS. - -```bash -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: 48 passed (46 + 2 new). - -- [ ] **Step 5: Commit** - -```bash -git add optimizerapi/auth.py tests/test_auth.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -fix(auth): refuse default AUTH_API_KEY='none' in production - -Previously, a server deployed with FLASK_ENV=production but no explicit -AUTH_API_KEY would still accept ?apikey=none. Now that's rejected; -operators must set a real key to enable the static-key path in prod. -Development mode keeps the legacy behaviour for ergonomics. - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -### Task 6: Fix `== None` in `securepickle/secure.py` - -Pre-existing flake8 E711 violations. Trivial fix while we're in this file anyway. - -**Files:** -- Modify: `optimizerapi/securepickle/secure.py:6,8` - -- [ ] **Step 1: Edit** - -Replace: -```python -def get_crypto(key=None): - if key == None: - key = os.getenv("PICKLE_KEY", None) - if key == None: - print("No key found, generating new key") - key = Fernet.generate_key() - os.environ["PICKLE_KEY"] = key.decode("utf-8") - print("To reuse key for future server runs, set environment variable PICKLE_KEY=" + - os.environ["PICKLE_KEY"]) - return Fernet(key) -``` - -with (this also lands Task 7's changes — see step note): - -```python -def get_crypto(key=None): - if key is None: - key = os.getenv("PICKLE_KEY", None) - if key is None: - print("No key found, generating new key") - key = Fernet.generate_key() - os.environ["PICKLE_KEY"] = key.decode("utf-8") - print("To reuse key for future server runs, set environment variable PICKLE_KEY=" + - os.environ["PICKLE_KEY"]) - return Fernet(key) -``` - -**Only the two `== None` → `is None` changes here.** Task 7 finishes the rest of the cleanup. - -- [ ] **Step 2: Verify lint** - -```bash -env/bin/flake8 optimizerapi/securepickle/secure.py --max-line-length=127 -``` -Expected: only the warning-free output (no E711). May still show other style issues; those are Task 7's job. - -- [ ] **Step 3: Commit** - -```bash -git add optimizerapi/securepickle/secure.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -style(securepickle): use 'is None' instead of '== None' - -Resolves long-standing flake8 E711 violations. - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -### Task 7: `get_crypto` no longer mutates `os.environ`, logs instead of prints - -**Files:** -- Modify: `optimizerapi/securepickle/secure.py` - -- [ ] **Step 1: Edit** - -Replace the whole file: - -```python -import logging -import os - -from cryptography.fernet import Fernet - -_LOG = logging.getLogger(__name__) - - -def get_crypto(key=None): - """Return a Fernet crypto handle for the given key. - - Resolution order: - 1. ``key`` argument (highest priority). - 2. ``PICKLE_KEY`` environment variable. - 3. Generate a new ephemeral key and warn the operator. - - The ephemeral path is intended for local development only. In any - environment where pickled state must survive a restart, ``PICKLE_KEY`` - must be set explicitly. This function no longer mutates ``os.environ`` - so caller environments are unchanged. - """ - if key is None: - key = os.getenv("PICKLE_KEY") - if key is None: - key = Fernet.generate_key() - _LOG.warning( - "No PICKLE_KEY set; generated an ephemeral key. Set " - "PICKLE_KEY=%s in the environment to persist pickled state " - "across restarts.", - key.decode("utf-8"), - ) - return Fernet(key) -``` - -Changes from the previous version: - -- `os.environ["PICKLE_KEY"] = ...` removed — `get_crypto` is now side-effect-free except for logging. -- `print` → `_LOG.warning` — visible in production logs, can be filtered/routed. -- The warning text is unified into a single message; before it printed two lines. -- Imports reordered (stdlib, then third-party). - -- [ ] **Step 2: Run the securepickle tests** - -```bash -env/bin/python -m pytest tests/test_securepickle.py -v -``` -Expected: PASS. - -- [ ] **Step 3: Run the full suite** - -```bash -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: 48 passed. - -- [ ] **Step 4: Verify behaviour by smoke-testing the server** - -```bash -timeout 5 env/bin/python -m optimizerapi.server 2>&1 | head -15 || true -``` -Expected: the `_LOG.warning` line about generating a key appears (this used to be two `print` lines). The Flask startup line appears below it. - -- [ ] **Step 5: Commit** - -```bash -git add optimizerapi/securepickle/secure.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -refactor(securepickle): use logging, drop os.environ mutation - -get_crypto previously printed to stdout and wrote PICKLE_KEY back into -os.environ as a side effect. Both are surprising for a "getter": -- print bypasses log routing/filtering -- env mutation makes the caller's environment opaque to itself - -Now it logs a single WARNING and leaves os.environ alone. The behaviour -contract (return a Fernet handle, optionally backed by a freshly -generated key) is unchanged. - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -### Task 8: Final lint + test pass - -**Files:** none. - -- [ ] **Step 1: flake8** - -```bash -env/bin/flake8 optimizerapi tests --max-line-length=127 -``` -Expected: fewer violations than before. `optimizerapi/securepickle/secure.py` should be fully clean (no E711). `optimizerapi/auth.py` should be clean. `optimizerapi/securepickle/__init__.py` F401s (re-export warnings) and `tests/context.py` F401s remain (those are deferred to Phase 5). `tests/test_optimizer.py:12,15` E501s remain (Phase 5). - -- [ ] **Step 2: pytest** - -```bash -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: 48 passed. - -- [ ] **Step 3: No additional commit required.** - ---- - -## Out of scope (for Phase 1) - -- Flask 2 → 3 migration (requires Connexion 3 rewrite; see overview). -- ProcessOptimizer pinning to a release tag (deferred; need a release to exist first). -- `print` cleanup outside `auth.py` and `secure.py` — that's Phase 2. -- `typing` annotations beyond what fits naturally into the auth signatures (`dict | None`) — Phase 3. -- Pre-existing flake8 violations outside the touched files — Phase 5. - -## Acceptance criteria - -- `cryptography>=44` installed; `tests/test_securepickle.py` passes. -- `apikey_handler` uses `secrets.compare_digest`; a unit test asserts the call. -- `apikey_handler` refuses `AUTH_API_KEY="none"` in production. -- `token_info` no longer prints the access token or any other stdout output. -- `get_crypto` no longer mutates `os.environ` and no longer prints; emits one WARNING when generating an ephemeral key. -- `flake8 optimizerapi/auth.py optimizerapi/securepickle/secure.py` is clean. -- Full suite: 48 passed (43 prior + 5 new auth tests). diff --git a/docs/superpowers/plans/2026-05-18-audit-phase-2-logging.md b/docs/superpowers/plans/2026-05-18-audit-phase-2-logging.md deleted file mode 100644 index 31d9488..0000000 --- a/docs/superpowers/plans/2026-05-18-audit-phase-2-logging.md +++ /dev/null @@ -1,417 +0,0 @@ -# Audit Phase 2 — Logging Migration - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Replace every `print()` call in `optimizerapi/` with a properly-scoped `logging` call routed through a single `basicConfig` set up at server startup. Remove dead `# print(...)` comments. - -**Architecture:** Centralize log configuration in `optimizerapi/server.py` (run before any handlers fire). Each module uses `logging.getLogger(__name__)`, so loggers fall into the natural module hierarchy (`optimizerapi.server`, `optimizerapi.optimizer_handler`, etc.) and can be filtered/routed individually in production. - -**Tech Stack:** Standard library `logging`. - -**Audit reference:** §4 of the audit. `print` calls remain in `server.py`, `optimizer_handler.py`, and dead-code comments in `optimizer.py`. Phase 1 already removed prints from `auth.py` and `secure.py`. - -**Prereq:** Phase 1 complete (`auth.py` and `secure.py` are already on `logging`). - ---- - -## File Map - -- **Modify:** `optimizerapi/server.py` — add `logging.basicConfig`, convert 3 prints. -- **Modify:** `optimizerapi/optimizer_handler.py` — convert ~4 prints. -- **Modify:** `optimizerapi/optimizer.py` — delete 2 dead `# print(...)` comments. -- **Modify:** `tests/test_no_prints.py` (new) — enforce the no-print rule under `optimizerapi/`. - ---- - -### Task 1: Centralize log config in `server.py` - -**Files:** -- Modify: `optimizerapi/server.py` - -- [ ] **Step 1: Edit `optimizerapi/server.py`** - -The existing file starts with: - -```python -""" -Main server -""" -import os -import re -import connexion -from waitress import serve -from flask_cors import CORS -from .securepickle import get_crypto -``` - -Replace with: - -```python -""" -Main server -""" -import logging -import os -import re - -import connexion -from flask_cors import CORS -from waitress import serve - -from .securepickle import get_crypto - -_LOG = logging.getLogger(__name__) - - -def _configure_logging() -> None: - """Configure root logging once, before any handler is dispatched. - - Level defaults to INFO. Override with the LOG_LEVEL environment - variable (e.g. LOG_LEVEL=DEBUG for local debugging). - """ - level_name = os.getenv("LOG_LEVEL", "INFO").upper() - level = getattr(logging, level_name, logging.INFO) - logging.basicConfig( - level=level, - format="%(asctime)s %(levelname)s %(name)s: %(message)s", - ) -``` - -Then, inside the existing `if __name__ == "__main__":` block, add a call to `_configure_logging()` as the very first statement: - -Find: -```python -if __name__ == "__main__": - # Initialize crypto - get_crypto() -``` - -Replace with: -```python -if __name__ == "__main__": - _configure_logging() - # Initialize crypto - get_crypto() -``` - -Then replace the three CORS-related `print` lines. - -Find: -```python - print("CORS: " + cors_origin) - except re.error: - print("CORS: failed - the regex might be malformed.") - else: - print("CORS: disabled") -``` - -Replace with: -```python - _LOG.info("CORS: %s", cors_origin) - except re.error: - _LOG.warning("CORS: failed - the regex might be malformed.") - else: - _LOG.info("CORS: disabled") -``` - -- [ ] **Step 2: Smoke-test the server boots and emits log lines** - -```bash -LOG_LEVEL=INFO timeout 5 env/bin/python -m optimizerapi.server 2>&1 | head -15 || true -``` - -Expected: At least one line of the form `2026-... INFO optimizerapi.server: CORS: ...` appears. The Phase-1 `_LOG.warning("No PICKLE_KEY set...")` line from `securepickle.secure` is also routed through this config and appears with the right format. - -- [ ] **Step 3: Run the full suite** - -```bash -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: 48 passed (unchanged from end of Phase 1). - -- [ ] **Step 4: Commit** - -```bash -git add optimizerapi/server.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -refactor(server): central logging.basicConfig + CORS prints to logger - -LOG_LEVEL env var (default INFO) controls verbosity. All module loggers -inherit this single basicConfig, so per-module routing is now possible -in production. - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -### Task 2: Convert prints in `optimizer_handler.py` - -**Files:** -- Modify: `optimizerapi/optimizer_handler.py` - -- [ ] **Step 1: Audit the existing prints** - -Run: -```bash -grep -n "print(" optimizerapi/optimizer_handler.py -``` - -Expected output (lines may shift slightly): -``` -25:print("Connecting to" + REDIS_URL) -67: print("Found existing job") -69: print(f"Creating new job (WORKER_TIMEOUT={WORKER_TIMEOUT})") -80: print(f"Client disconnected, cancelling job {job.id}") -``` - -- [ ] **Step 2: Edit the file** - -At the top, the existing import block looks like: - -```python -import os -import time -import json -import traceback -import hashlib -from rq import Queue -from rq.job import Job -``` - -Add `import logging` (alphabetical, with the other stdlib imports): - -```python -import hashlib -import json -import logging -import os -import time -import traceback - -from rq import Queue -from rq.job import Job -``` - -Right after the imports, add the module logger and replace the module-level connection print. - -Find: -```python -if "REDIS_URL" in os.environ: - REDIS_URL = os.environ["REDIS_URL"] -else: - REDIS_URL = "redis://localhost:6379" -print("Connecting to" + REDIS_URL) -redis = Redis.from_url(REDIS_URL) -``` - -Replace with: -```python -_LOG = logging.getLogger(__name__) - -REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379") -_LOG.info("Connecting to %s", REDIS_URL) -redis = Redis.from_url(REDIS_URL) -``` - -(The `os.environ.get(..., default)` collapse and the formatted-arg log are idiomatic; the latter also fixes the missing space between "to" and the URL in the previous line.) - -Then inside `run(body)`, replace: - -```python - try: - job = Job.fetch(job_id, connection=redis) - - print("Found existing job") - except NoSuchJobError: - print(f"Creating new job (WORKER_TIMEOUT={WORKER_TIMEOUT})") -``` - -with: - -```python - try: - job = Job.fetch(job_id, connection=redis) - _LOG.info("Found existing job %s", job_id) - except NoSuchJobError: - _LOG.info("Creating new job (WORKER_TIMEOUT=%s)", WORKER_TIMEOUT) -``` - -And the disconnect print: - -```python - print(f"Client disconnected, cancelling job {job.id}") -``` - -with: - -```python - _LOG.warning("Client disconnected, cancelling job %s", job.id) -``` - -- [ ] **Step 3: Verify no print remains** - -```bash -grep -n "print(" optimizerapi/optimizer_handler.py -``` -Expected: no output. - -- [ ] **Step 4: Run the full suite** - -```bash -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: 48 passed. - -- [ ] **Step 5: Commit** - -```bash -git add optimizerapi/optimizer_handler.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -refactor(handler): use logging in place of print - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -### Task 3: Remove dead `# print(...)` comments in `optimizer.py` - -**Files:** -- Modify: `optimizerapi/optimizer.py` - -- [ ] **Step 1: Find the dead comments** - -```bash -grep -n "# print(" optimizerapi/optimizer.py -``` -Expected output: -``` -: # print(str(response)) -: # print("IMAGE: " + str(pic_hash, "utf-8")) -``` - -- [ ] **Step 2: Delete those two lines.** - -Edit the file and remove exactly those two `# print(...)` lines. Do not touch surrounding code. - -- [ ] **Step 3: Confirm clean** - -```bash -grep -n "print(" optimizerapi/optimizer.py -``` -Expected: no output. - -- [ ] **Step 4: Run the full suite** - -```bash -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: 48 passed. - -- [ ] **Step 5: Commit** - -```bash -git add optimizerapi/optimizer.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -chore(optimizer): drop two dead # print debug comments - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -### Task 4: Enforce no-print rule with a unit test - -This locks in the migration so future regressions are caught. - -**Files:** -- Create: `tests/test_no_prints.py` - -- [ ] **Step 1: Write the test** - -Create `tests/test_no_prints.py`: - -```python -"""Lint test: no stray print() in production code. - -Tests in this repo configure their own logging; production code MUST -route everything through ``logging`` so operators can filter, level- -gate, and route messages in production. ``print`` calls bypass that. -""" - -import ast -import pathlib - -import pytest - -OPTIMIZERAPI_ROOT = pathlib.Path(__file__).parent.parent / "optimizerapi" - - -def _all_python_files() -> list[pathlib.Path]: - return [p for p in OPTIMIZERAPI_ROOT.rglob("*.py") if p.is_file()] - - -@pytest.mark.parametrize("path", _all_python_files(), ids=lambda p: str(p.relative_to(OPTIMIZERAPI_ROOT.parent))) -def test_no_print_calls(path): - """No `print()` call is allowed under optimizerapi/.""" - source = path.read_text(encoding="utf-8") - tree = ast.parse(source, filename=str(path)) - offenders = [ - node.lineno - for node in ast.walk(tree) - if isinstance(node, ast.Call) - and isinstance(node.func, ast.Name) - and node.func.id == "print" - ] - assert not offenders, ( - f"{path}: found print() calls at line(s) {offenders}. " - "Use logging.getLogger(__name__) instead." - ) -``` - -- [ ] **Step 2: Run the test** - -```bash -env/bin/python -m pytest tests/test_no_prints.py -v -``` -Expected: PASS for every file under `optimizerapi/`. - -If anything fails, find the missed `print` and convert it. The error message will say which file and line. - -- [ ] **Step 3: Run the full suite** - -```bash -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: 48+N passed where N is the number of `.py` files under `optimizerapi/` (the parametrize generates one test per file). - -- [ ] **Step 4: Commit** - -```bash -git add tests/test_no_prints.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -test: enforce no print() under optimizerapi/ - -Locks in the logging migration with an AST-based lint test so future -regressions fail loudly. - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -## Acceptance criteria - -- `grep -rn "print(" optimizerapi/` returns nothing. -- `tests/test_no_prints.py` passes for every module. -- Server still boots; log lines appear under the new `2026-... LEVEL name: message` format. -- Full pytest run is green. diff --git a/docs/superpowers/plans/2026-05-18-audit-phase-3-boundary-types.md b/docs/superpowers/plans/2026-05-18-audit-phase-3-boundary-types.md deleted file mode 100644 index dee9670..0000000 --- a/docs/superpowers/plans/2026-05-18-audit-phase-3-boundary-types.md +++ /dev/null @@ -1,648 +0,0 @@ -# Audit Phase 3 — Boundary Types & Static Checking - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `TypedDict` definitions for the request/response/cache shapes, annotate every public function signature in `optimizerapi/`, and add `mypy` to the dev workflow as a CI-grade type check. Acts as a safety net before the Phase 4 refactor. - -**Architecture:** One new module — `optimizerapi/types.py` — owns every boundary `TypedDict`. Modules import the relevant types and use them in signatures only (the runtime structures stay plain dicts, since the request validation lives in Connexion + OpenAPI, not Python). `mypy` is configured in `pyproject.toml` and runs against `optimizerapi/` only (tests are not type-checked yet). - -**Tech Stack:** Python 3.13 stdlib `typing` (`TypedDict`, `Literal`, `NotRequired`, `Self`), `mypy`. - -**Audit reference:** §2 of the audit. `body: dict` is the only thing keeping the contract honest today; adding types catches mistakes at edit time and makes Phase 4 safer. - -**Prereq:** Phase 1 & 2 complete (clean baseline of 48+ tests, no `print` in `optimizerapi/`). - ---- - -## File Map - -- **Create:** `optimizerapi/types.py` — all `TypedDict` definitions for request, response, cache. -- **Modify:** `pyproject.toml` — add `mypy>=1.13` to dev deps; add `[tool.mypy]` section. -- **Modify:** `optimizerapi/pickled_state.py` — annotate signatures using the new types. -- **Modify:** `optimizerapi/optimizer_handler.py` — annotate signatures. -- **Modify:** `optimizerapi/optimizer.py` — annotate `run` and `process_result` signatures. -- **Modify:** `AGENTS.md`/`CLAUDE.md` — document `mypy` in the dev loop. - ---- - -### Task 1: Create `optimizerapi/types.py` - -**Files:** -- Create: `optimizerapi/types.py` -- Create: `tests/test_types.py` — sanity check the module imports and the types are usable. - -- [ ] **Step 1: Write the failing test** - -Create `tests/test_types.py`: - -```python -"""Sanity checks for the boundary type definitions.""" - -from optimizerapi.types import ( - CachePayload, - Constraint, - DataPoint, - Dimension, - Extras, - OptimizerConfig, - Plot, - RequestBody, - ResponseEnvelope, - ResponseResult, - ResponseResultExtras, -) - - -def test_request_body_accepts_realistic_request(): - """A typical request body matches the RequestBody shape at runtime. - - TypedDicts don't enforce at runtime; we use isinstance(d, dict) as a - proxy and let mypy catch shape mismatches statically. - """ - body: RequestBody = { - "data": [{"xi": [50, 833, 150, 60, "Whipped cream"], "yi": [-3, -6]}], - "optimizerConfig": { - "baseEstimator": "GP", - "acqFunc": "EI", - "initialPoints": 3, - "kappa": 1.96, - "xi": 2, - "space": [ - {"type": "continuous", "name": "Sugar", "from": 0, "to": 100}, - {"type": "category", "name": "Finish", - "categories": ["None", "Whipped cream"]}, - ], - "constraints": [], - }, - "extras": {"graphFormat": "json", "includeModel": "true"}, - } - assert isinstance(body, dict) - assert body["data"][0]["xi"][-1] == "Whipped cream" - - -def test_response_envelope_shape(): - response: ResponseEnvelope = { - "plots": [{"id": "single_0_0", "plot": "{}"}], - "result": { - "next": [[1, 2, 3]], - "models": [], - "pickled": "", - "extras": {"pickledUsed": False}, - }, - } - assert response["result"]["extras"]["pickledUsed"] is False - - -def test_cache_payload_shape(): - payload: CachePayload = { - "fingerprint": "a" * 64, - "result": "opaque", - "next": [[1, 2]], - "optimizer": "opaque", - } - assert payload["fingerprint"] == "a" * 64 -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -env/bin/python -m pytest tests/test_types.py -v -``` -Expected: FAIL with `ModuleNotFoundError: No module named 'optimizerapi.types'`. - -- [ ] **Step 3: Create `optimizerapi/types.py`** - -```python -"""Boundary types for the optimizer API. - -These ``TypedDict``s describe the shape of the request body, response -envelope, and internal cache payload. They are *static* contracts — -they exist to be checked by ``mypy`` and to document the API to -readers. Runtime validation of incoming requests lives in Connexion + -OpenAPI; we do not duplicate that here. - -Note on the ``Dimension`` field name ``from``: - ``from`` is a Python keyword, so the class-based ``TypedDict`` - syntax cannot declare it. We use the alternative call-based - syntax to define ``Dimension``. -""" - -from typing import Any, Literal, NotRequired, TypedDict - - -# --- Request types --------------------------------------------------------- - -Dimension = TypedDict( - "Dimension", - { - "type": Literal["continuous", "discrete", "category"], - "name": str, - "from": NotRequired[float], - "to": NotRequired[float], - "categories": NotRequired[list[str]], - }, -) - - -class Constraint(TypedDict): - type: Literal["sum"] - dimensions: list[int] - value: float - - -class OptimizerConfig(TypedDict): - baseEstimator: str - acqFunc: str - initialPoints: int - kappa: float - xi: float - space: list[Dimension] - constraints: NotRequired[list[Constraint]] - - -class DataPoint(TypedDict): - xi: list[str | float] - yi: list[float] - - -GraphFormat = Literal["png", "json", "none"] -GraphName = Literal["objective", "convergence", "pareto", "single"] -ObjectivePars = Literal["result", "expected_minimum"] - - -class Extras(TypedDict, total=False): - objectivePars: ObjectivePars - graphFormat: GraphFormat - experimentSuggestionCount: int - maxQuality: int - graphs: list[GraphName] - selectedPoint: list[str | float] - pickled: str - # NB: stringly-typed; see audit §3 for the rationale for not migrating yet. - includeModel: Literal["true", "false"] - useActualMeasurementHistogram: Literal["true", "false"] - - -class RequestBody(TypedDict): - data: list[DataPoint] - optimizerConfig: OptimizerConfig - extras: NotRequired[Extras] - - -# --- Response types -------------------------------------------------------- - -class Plot(TypedDict): - id: str - plot: str # base64 PNG OR JSON-serialised plot data - - -class ResponseResultExtras(TypedDict, total=False): - pickledUsed: bool - parameters: dict[str, Any] - libraries: list[str] - pythonVersion: str - apiVersion: str - timeOfExecution: str - - -class ResponseModel(TypedDict, total=False): - expected_minimum: list[list[str | float]] - extras: dict[str, Any] - - -class ResponseResult(TypedDict, total=False): - next: list[list[str | float]] - models: list[ResponseModel] - pickled: str - extras: ResponseResultExtras - expected_minimum: list[list[str | float] | float] - - -class ResponseEnvelope(TypedDict): - plots: list[Plot] - result: ResponseResult - - -# --- Cache payload (pickled_state) ----------------------------------------- - -class CachePayload(TypedDict): - fingerprint: str - # ProcessOptimizer's OptimizerResult or list thereof — kept opaque - # so types.py does not depend on the optimizer library. - result: Any - next: list[list[str | float]] - # ProcessOptimizer.Optimizer — also opaque. - optimizer: Any -``` - -- [ ] **Step 4: Run tests** - -```bash -env/bin/python -m pytest tests/test_types.py -v -``` -Expected: PASS, 3 tests. - -- [ ] **Step 5: Run the full suite** - -```bash -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: 51+N passed (48 + 3 new). (`N` is the no-print parametrize tests from Phase 2.) - -- [ ] **Step 6: Commit** - -```bash -git add optimizerapi/types.py tests/test_types.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -feat(types): add TypedDicts for request, response, and cache payload - -Static-only contracts for mypy; runtime validation continues to live in -Connexion + OpenAPI. The new types.py module is the single source of -truth for what the boundary shapes look like in Python. - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -### Task 2: Install and configure `mypy` - -**Files:** -- Modify: `pyproject.toml` - -- [ ] **Step 1: Add `mypy` to dev deps** - -Find: -``` -dev = ["pytest==8.3.3", "pytest-watch==4.2.0", "flake8>=7"] -``` - -Replace with: -``` -dev = ["pytest==8.3.3", "pytest-watch==4.2.0", "flake8>=7", "mypy>=1.13"] -``` - -- [ ] **Step 2: Add `[tool.mypy]` configuration** - -Append to the end of `pyproject.toml`: - -```toml -[tool.mypy] -files = ["optimizerapi"] -python_version = "3.13" -strict = false -# Start permissive; we tighten as Phase 4 lands. -warn_unused_ignores = true -warn_redundant_casts = true -warn_return_any = true -disallow_any_unimported = false -check_untyped_defs = true -# Third-party libraries that don't ship type stubs. -ignore_missing_imports = true -``` - -- [ ] **Step 3: Install** - -```bash -env/bin/pip install -e ".[dev]" 2>&1 | tail -3 -``` -Expected: `Successfully installed ... mypy-1.x.x ...`. - -- [ ] **Step 4: Capture the baseline** - -```bash -env/bin/mypy optimizerapi 2>&1 | tail -20 -``` - -Don't try to fix things yet — this is the baseline that subsequent tasks will whittle down. The expected number of errors is "moderate" (most signatures are untyped today). Take note of the count for reference. - -- [ ] **Step 5: Commit** - -```bash -git add pyproject.toml -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -build: add mypy to dev deps with permissive baseline config - -Subsequent tasks tighten signatures module-by-module. Permissive mode -keeps the gate green during the migration. - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -### Task 3: Annotate `pickled_state.py` - -**Files:** -- Modify: `optimizerapi/pickled_state.py` - -- [ ] **Step 1: Edit** - -The current functions are untyped. Add precise signatures using the new types. - -Find: -```python -def compute_fingerprint(data, optimizer_config): - """Return sha256 hex of the canonical-JSON of (data, optimizerConfig). -``` - -Replace with: -```python -def compute_fingerprint( - data: list["DataPoint"], - optimizer_config: "OptimizerConfig", -) -> str: - """Return sha256 hex of the canonical-JSON of (data, optimizerConfig). -``` - -Find: -```python -def pack(*, result, next_points, optimizer, fingerprint, crypto): - """Encrypt a pickled cache payload for the given fingerprint.""" -``` - -Replace with: -```python -def pack( - *, - result: object, - next_points: list[list[str | float]], - optimizer: object, - fingerprint: str, - crypto: "Fernet", -) -> str: - """Encrypt a pickled cache payload for the given fingerprint.""" -``` - -Find: -```python -def unpack_if_valid(blob, *, expected_fingerprint, crypto): - """Decrypt and validate a pickled cache payload. -``` - -Replace with: -```python -def unpack_if_valid( - blob: str, - *, - expected_fingerprint: str, - crypto: "Fernet", -) -> "CachePayload | None": - """Decrypt and validate a pickled cache payload. -``` - -Add the imports at the top of the file (after the existing `import logging`): - -```python -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from cryptography.fernet import Fernet - - from .types import CachePayload, DataPoint, OptimizerConfig -``` - -Using `TYPE_CHECKING` keeps import-time light and avoids circular-import risk during the Phase 4 refactor when `optimizer.py` will also start importing from `types.py`. - -- [ ] **Step 2: Run mypy on the module** - -```bash -env/bin/mypy optimizerapi/pickled_state.py 2>&1 | tail -10 -``` -Expected: 0 errors (or the same module-local errors mypy reported in the baseline minus the ones the new annotations resolved). - -- [ ] **Step 3: Run the full suite** - -```bash -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: unchanged (tests don't change behavior). - -- [ ] **Step 4: Commit** - -```bash -git add optimizerapi/pickled_state.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -types: annotate pickled_state public surface - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -### Task 4: Annotate `optimizer_handler.py` - -**Files:** -- Modify: `optimizerapi/optimizer_handler.py` - -- [ ] **Step 1: Edit signatures** - -Find: -```python -def run(body) -> dict: -``` - -Replace with: -```python -def run(body: "RequestBody") -> "ResponseEnvelope | tuple[dict[str, str], int]": -``` - -The return-shape is the existing `dict | (dict, status_code)` mix that `do_run_work` produces. (Phase 5 normalizes this.) - -Find: -```python -def do_run_work(body) -> dict: - """ "Handle the run request""" -``` - -Replace with: -```python -def do_run_work(body: "RequestBody") -> "ResponseEnvelope | tuple[dict[str, str], int]": - """Handle the run request.""" -``` - -Add at the top of the file: -```python -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from .types import RequestBody, ResponseEnvelope -``` - -- [ ] **Step 2: Run mypy on the module** - -```bash -env/bin/mypy optimizerapi/optimizer_handler.py 2>&1 | tail -10 -``` -Expected: 0 new errors. - -- [ ] **Step 3: Run full suite** - -```bash -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: unchanged. - -- [ ] **Step 4: Commit** - -```bash -git add optimizerapi/optimizer_handler.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -types: annotate optimizer_handler.run and do_run_work - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -### Task 5: Annotate `optimizer.py` public entry points - -We only annotate `run` and `process_result` here. Helpers stay untyped for now — Phase 4 splits them into smaller pieces and annotates them as it goes. - -**Files:** -- Modify: `optimizerapi/optimizer.py` - -- [ ] **Step 1: Edit `run` signature** - -Find: -```python -def run(body) -> dict: - """ "Handle the run request""" -``` - -Replace with: -```python -def run(body: "RequestBody") -> dict: - """Handle the run request. - - Returns the response envelope as a plain ``dict`` — the json_tricks - round-trip at the bottom of the function dropping NumPy types is - why we cannot return ``ResponseEnvelope`` directly without an - explicit cast. - """ -``` - -- [ ] **Step 2: Edit `process_result` signature** - -Find: -```python -def process_result(result, optimizer, dimensions, cfg, extras, data, space, - *, request_fingerprint, pickled_used): -``` - -Replace with: -```python -def process_result( - result: object, - optimizer: object, - dimensions: list[str], - cfg: "OptimizerConfig", - extras: "Extras", - data: list[tuple[list[str | float], list[float]]], - space: list, - *, - request_fingerprint: str, - pickled_used: bool, -) -> dict: -``` - -- [ ] **Step 3: Add the imports** - -At the top of the file, near the existing imports, add: - -```python -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from .types import Extras, OptimizerConfig, RequestBody -``` - -- [ ] **Step 4: Run mypy and full suite** - -```bash -env/bin/mypy optimizerapi/optimizer.py 2>&1 | tail -10 -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: mypy may surface in-body issues; ignore them for now (they belong to Phase 4). The `def`-line types are what matter here. Pytest is unchanged. - -- [ ] **Step 5: Commit** - -```bash -git add optimizerapi/optimizer.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -types: annotate optimizer.run and process_result signatures - -Body-level annotations and helper functions land in Phase 4 along with -the structural refactor that splits process_result into focused units. - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -### Task 6: Document the type-check command - -**Files:** -- Modify: `AGENTS.md` -- Modify: `CLAUDE.md` - -- [ ] **Step 1: Add a line to `AGENTS.md`** - -Open `AGENTS.md`. After the line: -``` -- Lint locally with `flake8 . --max-line-length=127` (matches CI workflow). -``` - -Insert (alphabetical placement): -``` -- Type-check locally with `mypy optimizerapi` (matches CI workflow); see `[tool.mypy]` in `pyproject.toml`. -``` - -- [ ] **Step 2: Add the same to `CLAUDE.md`** - -In the `## Commands` block of `CLAUDE.md`, after the flake8 line, add: - -```bash -# Type-check (matches CI) -mypy optimizerapi -``` - -- [ ] **Step 3: Commit** - -```bash -git add AGENTS.md CLAUDE.md -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -docs: document mypy in dev-loop instructions - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -## Acceptance criteria - -- `optimizerapi/types.py` exists and exports the documented TypedDicts. -- `tests/test_types.py` passes (3 sanity-check assertions). -- `mypy optimizerapi` runs (it does not need to be error-free yet; Phase 4 cleans up the body-level issues). -- Public function signatures in `pickled_state.py`, `optimizer_handler.py`, and `optimizer.py` have types referencing the new module. -- Full pytest run is green. -- `AGENTS.md` and `CLAUDE.md` mention `mypy optimizerapi`. - -## Out of scope - -- Annotating every helper inside `optimizer.py` (Phase 4). -- Making `mypy strict = true` (deferred until after Phase 4). -- Type-checking the test suite (low ROI; tests are dynamic by nature). -- Switching to `pyright` (mypy chosen because it's the most familiar tooling). diff --git a/docs/superpowers/plans/2026-05-18-audit-phase-4-refactor-process-result.md b/docs/superpowers/plans/2026-05-18-audit-phase-4-refactor-process-result.md deleted file mode 100644 index d381bce..0000000 --- a/docs/superpowers/plans/2026-05-18-audit-phase-4-refactor-process-result.md +++ /dev/null @@ -1,1050 +0,0 @@ -# Audit Phase 4 — Refactor `process_result` - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Split `optimizer.py:process_result` (~250 lines, 4-way branching) into focused helpers. Pull plot-rendering code into a new `plot_emitters.py` module. Eliminate the three near-duplicate JSON single-plot loops via a single helper that takes a `prefix` argument. Behavior is unchanged — the equivalence test from the pareto redesign is the binding correctness gate. - -**Architecture:** Two refactors stacked: - -1. **Internal extraction in `optimizer.py`:** small helpers (`_parse_extras`, `_compute_next_experiments`, `_flatten_expected_minima`, `_set_expected_minimum`) live in the same file as private functions. -2. **External extraction:** the plot-emission code moves to `optimizerapi/plot_emitters.py`. The three duplicated single-plot loops collapse into `emit_json_single_plots(plots, model, *, prefix, selected_point)`. PNG emission moves to `emit_png_plots`. - -After both steps, `process_result` becomes a ~60-line dispatcher. - -**Tech Stack:** Python 3.13, mypy from Phase 3 catches refactor mistakes, the equivalence test (`test_equivalence_with_and_without_pickled_multi_objective`) catches behavior drift. - -**Audit reference:** §1 of the audit. The `process_result` function is the single largest source of structural debt; the refactor pays back the typing investment from Phase 3. - -**Prereq:** Phases 1, 2, 3 complete. `mypy optimizerapi/optimizer.py` runs (errors permitted; this phase reduces them). - ---- - -## File Map - -- **Modify:** `optimizerapi/optimizer.py` — extract helpers, shrink `process_result`. -- **Create:** `optimizerapi/plot_emitters.py` — `emit_png_plots`, `emit_json_single_plots`, `emit_pareto_data`. -- **Create:** `tests/test_plot_emitters.py` — focused unit tests for the new module. - -### Behavioral safety net - -Every task ends with `python -m pytest 2>&1 | tail -3`. The CRITICAL invariant is that `test_equivalence_with_and_without_pickled_multi_objective` passes after every commit — that's the binding contract from the pareto redesign. If it fails at any point, STOP and report BLOCKED with the divergence details. Do NOT modify the test to make it pass. - ---- - -### Task 1: Extract `_parse_extras` and the PNG-warning move - -The top of `process_result` reads extras keys and emits the PNG + selectedPoint warning. Pull both into a private helper that returns a small dataclass. - -**Files:** -- Modify: `optimizerapi/optimizer.py` - -- [ ] **Step 1: Add the dataclass and helper near the top of `optimizer.py`** - -Add at the top of the file (after the existing imports + version-info helpers, but before `process_result`): - -```python -from dataclasses import dataclass - - -@dataclass(frozen=True) -class _ParsedExtras: - graph_format: str - max_quality: int - graphs_to_return: list[str] - objective_pars: str - include_model: bool - selected_point: list[str | float] | None - experiment_suggestion_count: int - - -def _parse_bool(value: object, default: bool = True) -> bool: - """Coerce extras' stringly-typed boolean fields. - - Accepts the literals "true" / "false" (case-insensitive), real bools, - and JSON-style ``true``/``false``. Anything else falls back to *default*. - """ - if isinstance(value, bool): - return value - if value is None: - return default - text = str(value).strip().lower() - if text in ("true", "1", "yes"): - return True - if text in ("false", "0", "no"): - return False - return default - - -def _parse_extras(extras: "Extras", logger: "logging.Logger") -> _ParsedExtras: - """Read the request ``extras`` block into a typed view. - - Emits the PNG + selectedPoint warning here so callers don't need to - duplicate the check. - """ - graph_format = extras.get("graphFormat", "png") - selected_point = extras.get("selectedPoint") - if selected_point is not None and graph_format != "json": - logger.warning( - "selectedPoint ignored on png path (graphFormat=%s)", graph_format - ) - return _ParsedExtras( - graph_format=graph_format, - max_quality=int(extras.get("maxQuality", 5)), - graphs_to_return=extras.get( - "graphs", ["objective", "convergence", "pareto", "single"] - ), - objective_pars=extras.get("objectivePars", "result"), - include_model=_parse_bool(extras.get("includeModel", "true")), - selected_point=selected_point, - experiment_suggestion_count=int(extras.get("experimentSuggestionCount", 1)), - ) -``` - -- [ ] **Step 2: Use it inside `process_result`** - -Inside `process_result`, replace the block that currently reads the extras keys: - -```python - graph_format = extras.get("graphFormat", "png") - max_quality = int(extras.get("maxQuality", "5")) - graphs_to_return = extras.get( - "graphs", ["objective", "convergence", "pareto", "single"] - ) - - objective_pars = extras.get("objectivePars", "result") - - pickle_model = json.loads(extras.get("includeModel", "true").lower()) - selected_point = extras.get("selectedPoint") - if selected_point is not None and graph_format != "json": - logging.getLogger(__name__).warning( - "selectedPoint ignored on png path (graphFormat=%s)", graph_format - ) - - experiment_suggestion_count = 1 - if "experimentSuggestionCount" in extras: - experiment_suggestion_count = extras["experimentSuggestionCount"] -``` - -with: - -```python - parsed = _parse_extras(extras, logging.getLogger(__name__)) - graph_format = parsed.graph_format - max_quality = parsed.max_quality - graphs_to_return = parsed.graphs_to_return - objective_pars = parsed.objective_pars - pickle_model = parsed.include_model - selected_point = parsed.selected_point - experiment_suggestion_count = parsed.experiment_suggestion_count -``` - -The reassignment to local names preserves the existing body's references — minimising the diff and keeping the refactor incremental. Future tasks can drop the local aliases as the body shrinks. - -- [ ] **Step 3: Run the equivalence test** - -```bash -env/bin/python -m pytest tests/test_optimizer.py::test_equivalence_with_and_without_pickled_multi_objective -v -``` -Expected: PASS. - -- [ ] **Step 4: Run the full suite** - -```bash -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: unchanged from end of Phase 3. - -- [ ] **Step 5: Commit** - -```bash -git add optimizerapi/optimizer.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -refactor(optimizer): extract _parse_extras + _parse_bool helper - -Single source of truth for extras parsing (including the PNG+selectedPoint -warning). The local aliases preserve the rest of the function body for now; -subsequent tasks drop them as the body shrinks. - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -### Task 2: Extract `_compute_next_experiments` and `_flatten_expected_minima` - -These two are small but ugly. Pulling them out reduces noise in `process_result`. - -**Files:** -- Modify: `optimizerapi/optimizer.py` - -- [ ] **Step 1: Add the helpers** - -Below `_parse_extras`, add: - -```python -def _compute_next_experiments( - optimizer: object, - cfg: "OptimizerConfig", - n_points: int, -) -> list[list[str | float]]: - """Ask the optimizer for the next N experiments, normalising the shape. - - ``optimizer.ask`` can return either a single experiment (flat list) or - a list of experiments. We always return a list of lists. - """ - constraints = cfg.get("constraints", []) - if constraints: - next_exp = optimizer.ask(n_points=n_points, strategy="cl_min") - else: - next_exp = optimizer.ask(n_points=n_points) - if next_exp and not any(isinstance(x, list) for x in next_exp): - next_exp = [next_exp] - return round_to_length_scales(next_exp, optimizer.space) - - -def _flatten_expected_minima(models: list[dict]) -> None: - """In-place flatten of nested ``expected_minimum`` entries on each model. - - The pre-flatten shape from ``process_model`` can be a list of mixed - scalars and lists; the response contract is a single flat list inside - a one-element outer list. This function normalises that. - """ - for model in models: - flat = [] - for x in model["expected_minimum"]: - if isinstance(x, list): - flat.extend(x) - else: - flat.append(x) - model["expected_minimum"] = [flat] -``` - -- [ ] **Step 2: Use them inside `process_result`** - -Replace the existing next-experiment block: - -```python - if "constraints" in cfg and len(cfg["constraints"]) > 0: - next_exp = optimizer.ask( - n_points=experiment_suggestion_count, strategy="cl_min" - ) - else: - next_exp = optimizer.ask(n_points=experiment_suggestion_count) - if len(next_exp) > 0 and not any(isinstance(x, list) for x in next_exp): - next_exp = [next_exp] - result_details["next"] = round_to_length_scales(next_exp, optimizer.space) -``` - -with: - -```python - result_details["next"] = _compute_next_experiments( - optimizer, cfg, experiment_suggestion_count - ) -``` - -Replace the trailing flatten block (currently 3 levels of nested list comprehension): - -```python - org_models = response["result"]["models"] - for model in org_models: - # Flatten expected minimum entries - model["expected_minimum"] = [ - [ - item - for sublist in [ - x if isinstance(x, list) else [x] for x in model["expected_minimum"] - ] - for item in sublist - ] - ] - return response -``` - -with: - -```python - _flatten_expected_minima(response["result"]["models"]) - return response -``` - -- [ ] **Step 3: Run the equivalence test + full suite** - -```bash -env/bin/python -m pytest tests/test_optimizer.py::test_equivalence_with_and_without_pickled_multi_objective -v -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: equivalence PASS; full suite unchanged. - -- [ ] **Step 4: Commit** - -```bash -git add optimizerapi/optimizer.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -refactor(optimizer): extract _compute_next_experiments, _flatten_expected_minima - -Both were small but inline-ugly: the next-experiment shape normalisation -and the 3-deep list comprehension for flattening models. Each now reads -as a single function call at the call site. - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -### Task 3: Create `plot_emitters.py` with `emit_json_single_plots` - -The biggest DRY win. The same single-plot emission code is currently inlined three times in `process_result` (single-obj JSON, multi-obj objective_1, multi-obj objective_2), each with a different prefix. We replace all three with one helper. - -**Files:** -- Create: `optimizerapi/plot_emitters.py` -- Create: `tests/test_plot_emitters.py` - -- [ ] **Step 1: Write the failing test** - -Create `tests/test_plot_emitters.py`: - -```python -"""Tests for the plot-emitters module.""" - -import json - -import numpy -import pytest - -from optimizerapi.plot_emitters import emit_json_single_plots - - -class _FakePlotData: - """Mimics ProcessOptimizer's get_Brownie_Bee_1d_plot return value.""" - - @staticmethod - def make(n_dims: int): - # Each dimension yields a 4-element series; the last entry is a - # histogram (mean, std) pair. - dims = [[[1.0, 2.0, 3.0, 4.0] for _ in range(4)] for _ in range(n_dims)] - histogram = (numpy.array([42.0]), numpy.array([1.5])) - return dims + [histogram] - - -@pytest.fixture -def fake_brownie_1d(monkeypatch): - """Patch _get_brownie_bee_1d_plot_safe so this test doesn't need ProcessOptimizer.""" - def _stub(result, x_eval=None): - return _FakePlotData.make(n_dims=3) - monkeypatch.setattr( - "optimizerapi.plot_emitters._get_brownie_bee_1d_plot_safe", _stub - ) - - -def test_emit_json_single_plots_writes_one_entry_per_dim_plus_histogram(fake_brownie_1d): - plots: list = [] - emit_json_single_plots(plots, result=object(), prefix="single_0", selected_point=None) - ids = [p["id"] for p in plots] - assert ids == ["single_0_0", "single_0_1", "single_0_2", "single_0_3"] - # Last entry is the histogram payload. - histogram_payload = json.loads(plots[-1]["plot"]) - assert "histogram" in histogram_payload - assert histogram_payload["histogram"]["mean"] == 42.0 - assert histogram_payload["histogram"]["std"] == 1.5 - - -def test_emit_json_single_plots_supports_different_prefixes(fake_brownie_1d): - plots: list = [] - emit_json_single_plots(plots, result=object(), prefix="objective_2", selected_point=None) - ids = [p["id"] for p in plots] - assert ids[0] == "objective_2_0" - assert ids[-1] == "objective_2_3" - - -def test_emit_json_single_plots_passes_selected_point_through(monkeypatch): - captured = {} - - def _stub(result, x_eval=None): - captured["x_eval"] = x_eval - return _FakePlotData.make(n_dims=2) - - monkeypatch.setattr( - "optimizerapi.plot_emitters._get_brownie_bee_1d_plot_safe", _stub - ) - - plots: list = [] - sp = [50, 833, "Whipped cream"] - emit_json_single_plots(plots, result=object(), prefix="single_0", selected_point=sp) - assert captured["x_eval"] == sp -``` - -- [ ] **Step 2: Run test to verify it fails** - -```bash -env/bin/python -m pytest tests/test_plot_emitters.py -v -``` -Expected: FAIL — `ModuleNotFoundError: No module named 'optimizerapi.plot_emitters'`. - -- [ ] **Step 3: Create the module** - -Create `optimizerapi/plot_emitters.py`: - -```python -"""Plot emission for the optimizer API response. - -Two output formats are supported: -- ``emit_png_plots`` — base64 PNG plots, used by the legacy UI. -- ``emit_json_single_plots`` / ``emit_pareto_data`` — structured JSON - plot data, used by the React-based UI. -""" - -from typing import TYPE_CHECKING, Any - -import json_tricks -import numpy - -# Imported from optimizer.py — the private helper that wraps a ProcessOptimizer -# bug with categorical x_eval values. Phase 5 considers moving it here too. -from .optimizer import _get_brownie_bee_1d_plot_safe - -if TYPE_CHECKING: - from .types import Plot - - -def emit_json_single_plots( - plots: list["Plot"], - *, - result: Any, - prefix: str, - selected_point: list[str | float] | None, -) -> None: - """Append one plot entry per dimension plus a final histogram entry. - - Parameters - ---------- - plots: - The mutable list of plot entries to append to. - result: - A ProcessOptimizer ``OptimizerResult``-like object. - prefix: - Plot-id prefix. The emitted ids are ``{prefix}_0``, ``{prefix}_1``, - ..., with the final id being ``{prefix}_`` for the histogram. - selected_point: - X-space coordinates to highlight, or ``None`` to use the default. - """ - one_d_data = _get_brownie_bee_1d_plot_safe(result, x_eval=selected_point) - histogram_entry = one_d_data[-1] - for i, dim_data in enumerate(one_d_data[:-1]): - plots.append( - {"id": f"{prefix}_{i}", "plot": json_tricks.dumps({"data": dim_data})} - ) - plots.append( - { - "id": f"{prefix}_{len(one_d_data) - 1}", - "plot": json_tricks.dumps( - { - "histogram": { - "mean": float(numpy.ravel(histogram_entry[0])[0]), - "std": float(numpy.ravel(histogram_entry[1])[0]), - } - } - ), - } - ) -``` - -- [ ] **Step 4: Run tests** - -```bash -env/bin/python -m pytest tests/test_plot_emitters.py -v -``` -Expected: 3 PASS. - -- [ ] **Step 5: Replace the three duplicated loops in `process_result`** - -In `optimizer.py:process_result`, find the single-objective JSON loop: - -```python - elif graph_format == "json": - for idx, model in enumerate(result): - if "single" in graphs_to_return and optimizer.n_objectives != 2: - obj1_1D_data = _get_brownie_bee_1d_plot_safe(result[idx], x_eval=selected_point) - histogram_entry = obj1_1D_data[-1] - for i, dim_data in enumerate(obj1_1D_data[:-1]): - plots.append( - { - "id": f"single_{idx}_{i}", - "plot": json_tricks.dumps({"data": dim_data}), - } - ) - plots.append( - { - "id": f"single_{idx}_{len(obj1_1D_data) - 1}", - "plot": json_tricks.dumps( - { - "histogram": { - "mean": float( - numpy.ravel(histogram_entry[0])[0] - ), - "std": float( - numpy.ravel(histogram_entry[1])[0] - ), - } - } - ), - } - ) - if "convergence" in graphs_to_return: - pass - # skip plotting convergence data in json format - - if "objective" in graphs_to_return: - pass - # skip plotting objective data in json format -``` - -Replace with: - -```python - elif graph_format == "json": - for idx, model in enumerate(result): - if "single" in graphs_to_return and optimizer.n_objectives != 2: - emit_json_single_plots( - plots, - result=result[idx], - prefix=f"single_{idx}", - selected_point=selected_point, - ) - # convergence and objective plots are PNG-only; nothing to emit here. -``` - -Then find the multi-objective objective_1 loop (a near-copy): - -```python - if optimizer.n_objectives == 2 and "single" in graphs_to_return: - obj1_1D_data = _get_brownie_bee_1d_plot_safe(result[0], x_eval=selected_point) - histogram_entry = obj1_1D_data[-1] - for i, dim_data in enumerate(obj1_1D_data[:-1]): - plots.append( - { - "id": f"objective_1_{i}", - "plot": json_tricks.dumps({"data": dim_data}), - } - ) - plots.append( - { - "id": f"objective_1_{len(obj1_1D_data) - 1}", - "plot": json_tricks.dumps( - { - "histogram": { - "mean": float(numpy.ravel(histogram_entry[0])[0]), - "std": float(numpy.ravel(histogram_entry[1])[0]), - } - } - ), - } - ) - - obj2_1D_data = _get_brownie_bee_1d_plot_safe(result[1], x_eval=selected_point) - histogram_entry2 = obj2_1D_data[-1] - for i, dim_data in enumerate(obj2_1D_data[:-1]): - plots.append( - { - "id": f"objective_2_{i}", - "plot": json_tricks.dumps({"data": dim_data}), - } - ) - plots.append( - { - "id": f"objective_2_{len(obj2_1D_data) - 1}", - "plot": json_tricks.dumps( - { - "histogram": { - "mean": float(numpy.ravel(histogram_entry2[0])[0]), - "std": float(numpy.ravel(histogram_entry2[1])[0]), - } - } - ), - } - ) -``` - -Replace with: - -```python - if optimizer.n_objectives == 2 and "single" in graphs_to_return: - emit_json_single_plots( - plots, - result=result[0], - prefix="objective_1", - selected_point=selected_point, - ) - emit_json_single_plots( - plots, - result=result[1], - prefix="objective_2", - selected_point=selected_point, - ) -``` - -Add the import at the top of `optimizer.py` (after the existing local imports): - -```python -from .plot_emitters import emit_json_single_plots -``` - -- [ ] **Step 6: Run the equivalence test** - -```bash -env/bin/python -m pytest tests/test_optimizer.py::test_equivalence_with_and_without_pickled_multi_objective -v -``` -Expected: PASS. This test is the BINDING gate — if it fails, the prefix or argument order is wrong somewhere. - -- [ ] **Step 7: Run the full suite** - -```bash -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: all green, ~3 more tests than before (the new plot_emitters tests). - -- [ ] **Step 8: Commit** - -```bash -git add optimizerapi/plot_emitters.py optimizerapi/optimizer.py tests/test_plot_emitters.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -refactor(optimizer): unify three JSON single-plot loops into emit_json_single_plots - -Same body was inlined three times in process_result with different -prefix strings: single_{idx}, objective_1, objective_2. They are now -one function call each, in a new optimizerapi/plot_emitters.py module. - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -### Task 4: Move PNG plot emission to `plot_emitters.py` - -The PNG path inside `process_result` is its own ~25-line block. Move it out so the dispatcher has fewer branches to read. - -**Files:** -- Modify: `optimizerapi/plot_emitters.py` -- Modify: `optimizerapi/optimizer.py` - -- [ ] **Step 1: Add `emit_png_plots` to `plot_emitters.py`** - -Append to `optimizerapi/plot_emitters.py`: - -```python -import base64 -import io - -import matplotlib.pyplot as plt -from ProcessOptimizer.plots import ( - plot_brownie_bee_frontend, - plot_convergence, - plot_objective, -) - - -def emit_png_plots( - plots: list["Plot"], - *, - result: list, - dimensions: list[str], - graphs: list[str], - max_quality: int, - objective_pars: str, -) -> None: - """Append base64-encoded PNG plots for each model in ``result``.""" - for idx, model in enumerate(result): - if "single" in graphs: - bb_plots = plot_brownie_bee_frontend(model, max_quality=max_quality) - for i, plot in enumerate(bb_plots): - plots.append( - {"id": f"single_{idx}_{i}", "plot": _figure_to_b64(plot)} - ) - plt.close(plot) - if "convergence" in graphs: - plot_convergence(model) - _emit_current_figure(plots, f"convergence_{idx}") - if "objective" in graphs: - plot_objective( - model, - dimensions=dimensions, - usepartialdependence=False, - show_confidence=True, - pars=objective_pars, - ) - _emit_current_figure(plots, f"single_{idx}") - - -def _figure_to_b64(figure) -> str: - buf = io.BytesIO() - figure.savefig(buf, format="png") - buf.seek(0) - return base64.b64encode(buf.read()).decode("utf-8") - - -def _emit_current_figure(plots: list["Plot"], plot_id: str) -> None: - """Snapshot the current matplotlib figure into ``plots`` and clear it.""" - buf = io.BytesIO() - plt.savefig(buf, format="png", bbox_inches="tight") - buf.seek(0) - plots.append({"id": plot_id, "plot": base64.b64encode(buf.read()).decode("utf-8")}) - plt.clf() -``` - -`_emit_current_figure` is the new home for the logic that was in the old `add_plot` helper in `optimizer.py`. The `debug=True` branch of `add_plot` is dropped — it was an unused breakpoint helper. - -- [ ] **Step 2: Use `emit_png_plots` from `process_result`** - -In `optimizer.py:process_result`, find the PNG branch: - -```python - if graph_format == "png": - for idx, model in enumerate(result): - if "single" in graphs_to_return: - bb_plots = plot_brownie_bee_frontend(model, max_quality=max_quality) - for i, plot in enumerate(bb_plots): - pic_io_bytes = io.BytesIO() - plot.savefig(pic_io_bytes, format="png") - pic_io_bytes.seek(0) - pic_hash = base64.b64encode(pic_io_bytes.read()) - plots.append( - {"id": f"single_{idx}_{i}", "plot": str(pic_hash, "utf-8")} - ) - plt.close(plot) - if "convergence" in graphs_to_return: - plot_convergence(model) - add_plot(plots, f"convergence_{idx}") - - if "objective" in graphs_to_return: - plot_objective( - model, - dimensions=dimensions, - usepartialdependence=False, - show_confidence=True, - pars=objective_pars, - ) - add_plot(plots, f"single_{idx}") - - if optimizer.n_objectives == 1: - minimum = expected_minimum(result[0], return_std=True) - result_details["expected_minimum"] = [ - round_to_length_scales(minimum[0], optimizer.space), - minimum[1], - ] -``` - -Replace with: - -```python - if graph_format == "png": - emit_png_plots( - plots, - result=result, - dimensions=dimensions, - graphs=graphs_to_return, - max_quality=max_quality, - objective_pars=objective_pars, - ) - if optimizer.n_objectives == 1: - _set_expected_minimum(result_details, result[0], optimizer.space) -``` - -The `_set_expected_minimum` helper is new — add it near the other private helpers in `optimizer.py`: - -```python -def _set_expected_minimum( - result_details: dict, - single_result: object, - space: object, -) -> None: - """Compute and store the expected minimum (single-objective only).""" - minimum = expected_minimum(single_result, return_std=True) - result_details["expected_minimum"] = [ - round_to_length_scales(minimum[0], space), - minimum[1], - ] -``` - -Then find the duplicated `expected_minimum` block in the JSON single-objective branch: - -```python - if optimizer.n_objectives == 1: - minimum = expected_minimum(result[0], return_std=True) - result_details["expected_minimum"] = [ - round_to_length_scales(minimum[0], optimizer.space), - minimum[1], - ] -``` - -Replace with: - -```python - if optimizer.n_objectives == 1: - _set_expected_minimum(result_details, result[0], optimizer.space) -``` - -Add the import: - -```python -from .plot_emitters import emit_json_single_plots, emit_png_plots -``` - -(Replace the previous import line with the combined import.) - -- [ ] **Step 3: Delete the now-unused `add_plot` helper** - -The old `add_plot` function in `optimizer.py` (the one that writes to `result` and conditionally calls `plt.savefig` with `debug`) is no longer referenced. Delete it. - -```bash -grep -n "^def add_plot\|add_plot(" optimizerapi/optimizer.py -``` -Expected after deletion: no output for `^def add_plot` and no remaining call sites. - -- [ ] **Step 4: Run the equivalence test + full suite** - -```bash -env/bin/python -m pytest tests/test_optimizer.py::test_equivalence_with_and_without_pickled_multi_objective -v -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: equivalence PASS; full suite unchanged. - -- [ ] **Step 5: Commit** - -```bash -git add optimizerapi/plot_emitters.py optimizerapi/optimizer.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -refactor(optimizer): move PNG plot emission to plot_emitters, drop add_plot - -PNG plot rendering, the unused debug=True branch of add_plot, and the -duplicate expected_minimum block are folded into emit_png_plots / -_set_expected_minimum. - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -### Task 5: Move pareto-data emission to `plot_emitters.py` - -**Files:** -- Modify: `optimizerapi/plot_emitters.py` -- Modify: `optimizerapi/optimizer.py` - -- [ ] **Step 1: Add `emit_pareto_data` to `plot_emitters.py`** - -```python -from ProcessOptimizer.plots import get_Brownie_Bee_Pareto -from ProcessOptimizer.utils.utils import get_Pareto_front_compromise - - -def emit_pareto_data(plots: list["Plot"], optimizer: object) -> None: - """Append the pareto-front payload (multi-objective only).""" - front_x_data, front_y_data, obj1_error, obj2_error = get_Brownie_Bee_Pareto( - optimizer, n_points=200 - ) - best_idx = get_Pareto_front_compromise(front_y_data) - pareto_data = { - "front_x_data": front_x_data.tolist(), - "front_y_data": front_y_data.tolist(), - "obj1_error": obj1_error.tolist(), - "obj2_error": obj2_error.tolist(), - "best_idx": best_idx, - } - plots.append({"id": "pareto_data", "plot": json_tricks.dumps(pareto_data)}) -``` - -Add the `import` to the top of `plot_emitters.py` if not already there. - -- [ ] **Step 2: Use it in `process_result`** - -Find the pareto-computation block: - -```python - if optimizer.n_objectives == 2 and ( - "pareto" in graphs_to_return or "single" in graphs_to_return - ): - front_x_data, front_y_data, obj1_error, obj2_error = ( - get_Brownie_Bee_Pareto(optimizer, n_points=200) - ) - best_idx = get_Pareto_front_compromise(front_y_data) - if "pareto" in graphs_to_return: - pareto_data = { - "front_x_data": front_x_data.tolist(), - "front_y_data": front_y_data.tolist(), - "obj1_error": obj1_error.tolist(), - "obj2_error": obj2_error.tolist(), - "best_idx": best_idx, - } - plots.append( - { - "id": "pareto_data", - "plot": json_tricks.dumps(pareto_data), - } - ) -``` - -Replace with: - -```python - if optimizer.n_objectives == 2 and "pareto" in graphs_to_return: - emit_pareto_data(plots, optimizer) -``` - -**Notice the simplification:** the original `if` was `pareto OR single` but only emitted when `pareto` was actually in the list. The `OR single` was dead — the original outer condition gated a block where only the inner `if "pareto"` actually mattered. We drop the misleading OR. - -(If a future need arises to compute the pareto front for plotting reasons beyond `"pareto"`, this can come back. Right now it doesn't.) - -Update the import line: - -```python -from .plot_emitters import emit_json_single_plots, emit_pareto_data, emit_png_plots -``` - -Remove now-unused imports from `optimizer.py`: - -```bash -grep -n "from ProcessOptimizer.plots import\|get_Pareto_front_compromise" optimizerapi/optimizer.py -``` - -If `get_Brownie_Bee_Pareto`, `plot_brownie_bee_frontend`, `plot_convergence`, `plot_objective`, or `get_Pareto_front_compromise` are no longer used in `optimizer.py`, remove them from the imports. (`_get_brownie_bee_1d_plot_safe` still lives in `optimizer.py` and is re-imported by `plot_emitters.py` — leave that alone for now.) - -- [ ] **Step 3: Run the equivalence test + full suite** - -```bash -env/bin/python -m pytest tests/test_optimizer.py::test_equivalence_with_and_without_pickled_multi_objective -v -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: equivalence PASS; full suite unchanged. - -- [ ] **Step 4: Commit** - -```bash -git add optimizerapi/plot_emitters.py optimizerapi/optimizer.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -refactor(optimizer): move emit_pareto_data to plot_emitters - -Also drops the misleading 'pareto OR single' outer condition — only -'pareto' was ever load-bearing for the actual emission. - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -### Task 6: Final shrink + mypy + lint - -By this point `process_result` should be close to 60 lines. The body still has the local-alias reassignments from Task 1; drop them now that the function is small enough to use `parsed.X` directly throughout. - -**Files:** -- Modify: `optimizerapi/optimizer.py` - -- [ ] **Step 1: Drop the local-alias block in `process_result`** - -Find: - -```python - parsed = _parse_extras(extras, logging.getLogger(__name__)) - graph_format = parsed.graph_format - max_quality = parsed.max_quality - graphs_to_return = parsed.graphs_to_return - objective_pars = parsed.objective_pars - pickle_model = parsed.include_model - selected_point = parsed.selected_point - experiment_suggestion_count = parsed.experiment_suggestion_count -``` - -Replace with: - -```python - parsed = _parse_extras(extras, logging.getLogger(__name__)) -``` - -Then update every reference in the body from the bare name to `parsed.X`. For example: - -- `graph_format == "png"` → `parsed.graph_format == "png"` -- `experiment_suggestion_count` → `parsed.experiment_suggestion_count` -- `selected_point` → `parsed.selected_point` -- `graphs_to_return` → `parsed.graphs_to_return` -- `max_quality` → `parsed.max_quality` -- `objective_pars` → `parsed.objective_pars` -- `pickle_model` → `parsed.include_model` - -The body should now be small enough to inspect that no references are missed. Run pytest after the substitution. - -- [ ] **Step 2: Run mypy on `optimizer.py`** - -```bash -env/bin/mypy optimizerapi/optimizer.py 2>&1 | tail -20 -``` -Expected: significantly fewer errors than the Phase 3 baseline. The new helpers are typed; the dispatcher uses typed inputs. Fix anything that mypy now flags as a real issue (typically forgotten-import, attribute-access on a wider type than intended). - -- [ ] **Step 3: Run flake8** - -```bash -env/bin/flake8 optimizerapi/optimizer.py optimizerapi/plot_emitters.py --max-line-length=127 -``` -Expected: clean. - -- [ ] **Step 4: Run the equivalence test and full suite one final time** - -```bash -env/bin/python -m pytest tests/test_optimizer.py::test_equivalence_with_and_without_pickled_multi_objective -v -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: equivalence PASS; full suite green. - -- [ ] **Step 5: Commit** - -```bash -git add optimizerapi/optimizer.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -refactor(optimizer): drop local aliases; use parsed.X directly - -Final shrink of process_result after the extractions land. The -dispatcher is now under ~80 lines and walks each path in order: -extras → next → models → graph branches → pickled → extras tail. - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - -- [ ] **Step 6: Check the LOC delta** - -```bash -wc -l optimizerapi/optimizer.py optimizerapi/plot_emitters.py -``` - -Reasonable target: `optimizer.py` under 350 lines (was 565), `plot_emitters.py` under 200 lines. The total may be similar or slightly larger — the win is in distribution and readability, not raw line count. - ---- - -## Acceptance criteria - -- `optimizerapi/optimizer.py:process_result` is under ~80 lines. -- `optimizerapi/plot_emitters.py` exists and owns all plot emission. -- `emit_json_single_plots` is called three times from `process_result` (single-obj single, multi-obj objective_1, multi-obj objective_2) — i.e. the duplication is gone. -- The PNG path in `process_result` is a single call to `emit_png_plots`. -- `add_plot` and its `debug=True` branch are deleted (no callers remain). -- `_get_brownie_bee_1d_plot_safe`, `round_to_length_scales`, `process_model`, `add_version_info`, `convert_number_type` continue to live in `optimizer.py` — they don't fit the plot-emitters split. -- `test_equivalence_with_and_without_pickled_multi_objective` passes after every commit in this phase. -- Full pytest run is green. -- `mypy optimizerapi/optimizer.py optimizerapi/plot_emitters.py` is clean. -- `flake8 optimizerapi --max-line-length=127` introduces no new violations. - -## Out of scope - -- Annotating every internal helper (most are now small enough to be obvious). -- Removing `_get_brownie_bee_1d_plot_safe` from `optimizer.py` (it's the workaround for the upstream bug; moving it would cost more clarity than it gains). -- Async / streaming responses (a Phase 6+ idea, if it ever happens). -- Changing the OpenAPI response shape (this whole phase is behavior-preserving). diff --git a/docs/superpowers/plans/2026-05-18-audit-phase-5-cleanup.md b/docs/superpowers/plans/2026-05-18-audit-phase-5-cleanup.md deleted file mode 100644 index 29b43eb..0000000 --- a/docs/superpowers/plans/2026-05-18-audit-phase-5-cleanup.md +++ /dev/null @@ -1,646 +0,0 @@ -# Audit Phase 5 — Opportunistic Cleanup - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Finish the long-tail of code-quality items surfaced by the audit: pre-existing flake8 violations, the `optimizer_handler.py` rough edges, the auth typo, and idiomatic Python tidies that the earlier phases left for last. Bring `flake8 optimizerapi tests --max-line-length=127` fully clean. - -**Architecture:** Pure cleanup. No new modules, no new public surface, no behavior changes. - -**Tech Stack:** Standard library `secrets`, `pytest`, mypy from Phase 3. - -**Audit reference:** §3, §6, §7 of the audit, plus the "pre-existing E501s" deferred through Phases 1–4. - -**Prereq:** Phases 1–4 complete. Full pytest is green. - ---- - -## File Map - -- **Modify:** `tests/test_optimizer.py` — delete dead comment lines. -- **Modify:** `optimizerapi/securepickle/__init__.py` — add `__all__` to resolve F401. -- **Modify:** `tests/context.py` — remove unused imports or convert to `__all__`. -- **Modify:** `optimizerapi/optimizer_handler.py` — `USE_WORKER` bool parsing; consolidate `disconnect_check`; normalize return shape. -- **Modify:** `optimizerapi/auth.py` — fix `"Authenitcation"` typo. -- **Modify:** `optimizerapi/optimizer.py` — `enumerate` + `.get()` idiomatic wins. - ---- - -### Task 1: Delete dead 400-character comment lines - -**Files:** -- Modify: `tests/test_optimizer.py:12-15` - -- [ ] **Step 1: Inspect** - -```bash -sed -n '11,16p' tests/test_optimizer.py -``` - -Expected: three comment lines (`# {'data': ...}`, `# 'data': ...`, `# 'space': ...`, `# '...'}}`) totaling ~880 characters. These were sample-payload notes from early development. - -- [ ] **Step 2: Delete the comment block** - -Remove lines 12-15 of `tests/test_optimizer.py`. The block looks like: - -```python -# {'data': [{'xi': [651, 56, 722, 'Ræv'], 'yi': 1}, ... -# 'data': [{'xi': [0, 5, 'Rød'], 'yi': 10}, ... -# 'optimizerConfig': {'baseEstimator': 'GP', ... -# 'space': [{'type': 'discrete', 'name': 'Alkohol', ... -``` - -The information they recorded is now captured properly in `docs/api-usage.md` and `scripts/sample*.curl`. - -- [ ] **Step 3: Verify flake8** - -```bash -env/bin/flake8 tests/test_optimizer.py --max-line-length=127 -``` -Expected: no output. The two E501 violations that have been pre-existing since the redesign are gone. - -- [ ] **Step 4: Run the full suite** - -```bash -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: unchanged. - -- [ ] **Step 5: Commit** - -```bash -git add tests/test_optimizer.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -chore(tests): drop dead 400-char sample-payload comments - -The sample payloads are now documented properly in docs/api-usage.md -and scripts/sample*.curl. The comments at the top of test_optimizer.py -were one-off dev notes that have been silently failing flake8 E501 -for ages. - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -### Task 2: Fix F401 re-exports in `securepickle/__init__.py` - -The package `__init__.py` imports symbols only to re-export them. flake8 reports F401 because there's no `__all__` declaring intent. - -**Files:** -- Modify: `optimizerapi/securepickle/__init__.py` - -- [ ] **Step 1: Read current state** - -```bash -cat optimizerapi/securepickle/__init__.py -``` - -Expected (current contents): - -```python -""" -Secure pickle module -""" -from .pickler import pickleToString, unpickleFromString -from .secure import get_crypto -``` - -- [ ] **Step 2: Add `__all__`** - -Replace with: - -```python -"""Secure pickle module — Fernet-encrypted pickle round-trip.""" - -from .pickler import pickleToString, unpickleFromString -from .secure import get_crypto - -__all__ = ["get_crypto", "pickleToString", "unpickleFromString"] -``` - -- [ ] **Step 3: Verify flake8** - -```bash -env/bin/flake8 optimizerapi/securepickle/__init__.py --max-line-length=127 -``` -Expected: no output. - -- [ ] **Step 4: Commit** - -```bash -git add optimizerapi/securepickle/__init__.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -chore(securepickle): declare __all__ to resolve F401 on re-exports - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -### Task 3: Clean up `tests/context.py` - -**Files:** -- Modify: `tests/context.py` - -- [ ] **Step 1: Read current state** - -```bash -cat tests/context.py -``` - -Decide based on what you find: - -- If `context.py` is an unused / abandoned test helper: delete the file and remove any imports of it. (You'll need to `grep -rn "from .context\|from tests.context\|from context" tests/ optimizerapi/` first.) -- If it is used somewhere: convert its imports into a `__all__` block similar to Task 2. - -- [ ] **Step 2: Apply the fix** - -In practice, this file imports `optimizer` and `securepickle` for tests to use without going through the full package path. If nothing imports `tests.context`, delete the file. Otherwise add: - -```python -"""Test-only convenience re-exports.""" - -from optimizerapi import optimizer, securepickle - -__all__ = ["optimizer", "securepickle"] -``` - -- [ ] **Step 3: Verify** - -```bash -env/bin/flake8 tests/context.py --max-line-length=127 -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: no flake8 output; pytest unchanged. - -- [ ] **Step 4: Commit** - -```bash -git add tests/context.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -chore(tests): tidy tests/context.py (or delete if unused) - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - -(If the file was deleted, mention that in the commit body.) - ---- - -### Task 4: Sweep `optimizer_handler.py` - -Three small wins, one commit. - -**Files:** -- Modify: `optimizerapi/optimizer_handler.py` - -- [ ] **Step 1: Fix `USE_WORKER` boolean parsing** - -Find: - -```python - if "USE_WORKER" in os.environ and os.environ["USE_WORKER"]: -``` - -The bug: `os.environ["USE_WORKER"] = "false"` is truthy. Any non-empty string passes this gate. - -Replace with: - -```python - if _parse_env_bool("USE_WORKER"): -``` - -Add this helper at module scope (after the imports, before `_LOG`): - -```python -def _parse_env_bool(name: str, default: bool = False) -> bool: - """Parse a boolean-ish env var. - - Accepts "true", "1", "yes" (case-insensitive) as True. - Anything else — including unset or "false" — is False. - """ - raw = os.environ.get(name, "").strip().lower() - if not raw: - return default - return raw in ("true", "1", "yes") -``` - -- [ ] **Step 2: Consolidate `disconnect_check`** - -The current function defines `disconnect_check` three separate ways depending on which Connexion / Flask state is reachable. Replace: - -```python - try: - if "waitress.client_disconnected" in connexion.request.environ: - disconnect_check = connexion.request.environ["waitress.client_disconnected"] - else: - - def disconnect_check(): - return False - - except RuntimeError: - - def disconnect_check(): - return False -``` - -with: - -```python - disconnect_check = _resolve_disconnect_check() -``` - -And add the helper at module scope: - -```python -def _resolve_disconnect_check(): - """Return a callable that reports whether the client has disconnected. - - Outside of a request context (e.g. unit tests) or when running under - a server that doesn't expose ``waitress.client_disconnected``, this - returns a no-op that always says "still connected". - """ - try: - env = connexion.request.environ - except RuntimeError: - return lambda: False - return env.get("waitress.client_disconnected", lambda: False) -``` - -- [ ] **Step 3: Normalize `do_run_work` return shape** - -Currently the success path returns a `dict` while the error paths return a `(dict, status_code)` tuple. Make all paths return the tuple shape so callers don't branch on it. - -Find: - -```python -def do_run_work(body) -> dict: - """ "Handle the run request""" - try: - return handle_run(body) - except IOError as err: - return ({"message": "I/O error", "error": str(err)}, 400) - except TypeError as err: - return ({"message": "Type error", "error": str(err)}, 400) - except ValueError as err: - return ({"message": "Validation error", "error": str(err)}, 400) - except Exception as err: - # Log unknown exceptions to support debugging - traceback.print_exc() - return ({"message": "Unknown error", "error": str(err)}, 500) -``` - -Replace with: - -```python -def do_run_work(body: "RequestBody") -> "ResponseEnvelope": - """Handle the run request. - - On error we let Connexion translate the exception into the OpenAPI- - declared 400 / 500 response. The handler stops being a tuple-returning - special case. - """ - try: - return handle_run(body) - except (IOError, TypeError, ValueError) as err: - _LOG.warning("client error: %s", err) - raise connexion.problem(400, "Bad request", str(err)) - except Exception as err: - _LOG.exception("unexpected error during optimizer run") - raise connexion.problem(500, "Internal server error", str(err)) -``` - -If the project's Connexion version does not support `connexion.problem` (`2.14.2` does, as `from connexion.exceptions import ProblemException` or similar), substitute the explicit `flask.abort(...)` / raise-with-status path. Test on the dev server before committing. - -If `connexion.problem` returns a `(body, status)` tuple rather than an exception, adapt: - -```python - try: - return handle_run(body) - except (IOError, TypeError, ValueError) as err: - _LOG.warning("client error: %s", err) - return connexion.problem(400, "Bad request", str(err)) - except Exception as err: - _LOG.exception("unexpected error during optimizer run") - return connexion.problem(500, "Internal server error", str(err)) -``` - -Either way, the result is that `traceback.print_exc()` (a print equivalent) is gone — the same diagnostics flow via `_LOG.exception`, which includes the traceback automatically. - -- [ ] **Step 4: Add the imports if missing** - -At the top, ensure `connexion` is imported (it already is — used elsewhere in the file). No new imports needed. - -- [ ] **Step 5: Run the full suite** - -```bash -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: full suite still green. If a test expected the old tuple-return shape, fix it (the test was testing the bug, not the contract). - -- [ ] **Step 6: Smoke-test the server with the worker path off and on** - -```bash -USE_WORKER=false timeout 5 env/bin/python -m optimizerapi.server 2>&1 | head -10 || true -``` - -Expected: no log line saying "USE_WORKER detected" — the explicit `false` is correctly parsed. - -- [ ] **Step 7: Commit** - -```bash -git add optimizerapi/optimizer_handler.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -refactor(handler): _parse_env_bool, single disconnect_check resolver, normalized error path - -Fixes the USE_WORKER='false' truthiness bug, collapses three near- -identical disconnect_check definitions, and routes errors via -connexion.problem so do_run_work returns a single declared shape -instead of a mixed dict/tuple. - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -### Task 5: Fix `Authenitcation` typo and an idiomatic-Python sweep - -**Files:** -- Modify: `optimizerapi/auth.py` -- Modify: `optimizerapi/optimizer.py` - -- [ ] **Step 1: Fix the auth.py docstring typo** - -Find: - -```python -"""Authenitcation module - -This module will verify tokens provided bt a Keycloak OpenID server -""" -``` - -Replace with: - -```python -"""Authentication module. - -Verifies bearer tokens issued by a Keycloak OpenID server, and the -static API key provided via ``AUTH_API_KEY``. -""" -``` - -(Also fixes "bt" → "by".) - -- [ ] **Step 2: `enumerate` for `round_to_length_scales` in `optimizer.py`** - -Find: - -```python - for dim, i in zip(space.dimensions, range(len(space.dimensions))): -``` - -Replace with: - -```python - for i, dim in enumerate(space.dimensions): -``` - -- [ ] **Step 3: `in (...)` for the dimension-type check in `optimizer.py`** - -Find: - -```python - if (x["type"] == "discrete" or x["type"] == "continuous") -``` - -Replace with: - -```python - if x["type"] in ("discrete", "continuous") -``` - -- [ ] **Step 4: `.get()` for the `extras` and `constraints` parses in `optimizer.run`** - -Find: - -```python - cfg = body["optimizerConfig"] - constraints = cfg["constraints"] if "constraints" in cfg else [] - extras = body["extras"] if "extras" in body else {} -``` - -Replace with: - -```python - cfg = body["optimizerConfig"] - constraints = cfg.get("constraints", []) - extras = body.get("extras", {}) -``` - -- [ ] **Step 5: Verify lint + suite** - -```bash -env/bin/flake8 optimizerapi --max-line-length=127 -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: no new flake8 output; pytest green. - -- [ ] **Step 6: Commit** - -```bash -git add optimizerapi/auth.py optimizerapi/optimizer.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -chore: typo fix + small idiomatic Python sweep - -- auth.py: 'Authenitcation' → 'Authentication', 'bt' → 'by' -- optimizer.py: enumerate; tuple membership for dim type; - .get() defaults for cfg.constraints and body.extras - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -### Task 6: Optional — convert sample fixtures to `@pytest.fixture` - -This is offered as a discrete task because it touches many test signatures. Skip if appetite is low; the module-level constants work today and tests mostly `copy.deepcopy` defensively. - -**Files:** -- Modify: `tests/test_optimizer.py` -- Create: `tests/conftest.py` - -- [ ] **Step 1: Create `tests/conftest.py`** - -Pull the read-only fixtures out of `test_optimizer.py`: - -```python -"""Shared fixtures for the optimizer test suite.""" - -import copy - -import pytest - - -@pytest.fixture -def sample_data(): - return copy.deepcopy([ - {"xi": [651, 56, 722, "Ræv"], "yi": [1]}, - {"xi": [651, 42, 722, "Ræv"], "yi": [0.2]}, - ]) - - -@pytest.fixture -def sample_config(): - return copy.deepcopy({ - "baseEstimator": "GP", - "acqFunc": "gp_hedge", - "initialPoints": 2, - "kappa": 1.96, - "xi": 0.012, - "space": [ - {"type": "discrete", "name": "Sukker", "from": 0, "to": 1000}, - {"type": "continuous", "name": "Peber", "from": 0, "to": 1000}, - {"type": "continuous", "name": "Hvedemel", "from": 0, "to": 1000}, - {"type": "category", "name": "Kunde", "categories": ["Mus", "Ræv"]}, - ], - }) - - -@pytest.fixture -def sample_multi_objective_5dim_data(): - return copy.deepcopy([ - {"xi": [16.7, 500, 250, 20, "None"], "yi": [-2, -17]}, - {"xi": [50, 833, 150, 60, "Whipped cream"], "yi": [-3, -6]}, - {"xi": [58.3, 22, 85, 6, "Frosting"], "yi": [-6, -25]}, - ]) - - -@pytest.fixture -def sample_multi_objective_5dim_config(): - return copy.deepcopy({ - "baseEstimator": "GP", - "acqFunc": "EI", - "initialPoints": 3, - "kappa": 1.96, - "xi": 2, - "space": [ - {"type": "continuous", "name": "Sugar", "from": 0, "to": 100}, - {"type": "continuous", "name": "Flour", "from": 0, "to": 1000}, - {"type": "discrete", "name": "Temperature", "from": 0, "to": 300}, - {"type": "discrete", "name": "Time", "from": 0, "to": 120}, - {"type": "category", "name": "Finish", - "categories": ["None", "Frosting", "Whipped cream"]}, - ], - "constraints": [], - }) -``` - -- [ ] **Step 2: Migrate `tests/test_optimizer.py` to use the fixtures** - -For each test that referenced a module-level constant, add the relevant fixture as a parameter: - -```python -# Before -def test_can_be_run_without_data(): - result = optimizer.run(body={"data": [], "optimizerConfig": sampleConfig}) - ... - -# After -def test_can_be_run_without_data(sample_config): - result = optimizer.run(body={"data": [], "optimizerConfig": sample_config}) - ... -``` - -Repeat for every test in the file. This is mechanical but mass — be patient and re-run pytest after each batch of ~5 tests to catch errors early. - -Once every test uses fixtures, delete the module-level `sampleData`, `sampleConfig`, `sampleMultiObjective5DimData`, `sampleMultiObjective5DimConfig` constants from `test_optimizer.py`. (The `sampleMultiObjectiveData`, `brownie_with_constraints`, and `brownie_without_constraints` constants can stay or move to conftest as you prefer.) - -- [ ] **Step 3: Run the full suite** - -```bash -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: same number of passes as before. - -- [ ] **Step 4: Commit** - -```bash -git add tests/conftest.py tests/test_optimizer.py -git -c user.email="jakob.langdal@alexandra.dk" -c user.name="Jakob Langdal" commit -m "$(cat <<'EOF' -refactor(tests): move sample fixtures into conftest.py - -Each test now declares its dependencies explicitly via fixture -parameters. Mutating a fixture in one test no longer leaks into others. - -Co-Authored-By: Claude Opus 4.7 (1M context) -EOF -)" -``` - ---- - -### Task 7: Final lint + mypy + pytest - -The acceptance gate for the entire audit. - -**Files:** none. - -- [ ] **Step 1: flake8 clean** - -```bash -env/bin/flake8 optimizerapi tests --max-line-length=127 -``` -Expected: **no output**. Every pre-existing exception is now fixed. - -- [ ] **Step 2: mypy clean for `optimizerapi/`** - -```bash -env/bin/mypy optimizerapi -``` -Expected: 0 errors. (Phase 3 set up the config; Phase 4 cleaned up `optimizer.py`'s body; Phase 5 closes the long-tail gaps.) - -If errors remain, fix them inline. They should be small at this point — bad import paths or remaining `Any` leaks at module boundaries. - -- [ ] **Step 3: Full test suite** - -```bash -env/bin/python -m pytest 2>&1 | tail -3 -``` -Expected: 50+ passed (43 original + auth + types + plot_emitters + no-prints + minor). No failures, no errors. - -- [ ] **Step 4: No commit required.** - ---- - -## Acceptance criteria (overall audit) - -After Phase 5 completes, the following must hold: - -- `env/bin/flake8 optimizerapi tests --max-line-length=127` → no output. -- `env/bin/mypy optimizerapi` → 0 errors. -- `env/bin/python -m pytest` → all green. -- `grep -rn "print(" optimizerapi/` → no output (from Phase 2). -- `optimizerapi/optimizer.py:process_result` is under ~80 lines (from Phase 4). -- `optimizerapi/auth.py` uses `secrets.compare_digest` and never prints tokens (from Phase 1). -- `cryptography` is on a current major (from Phase 1). -- The OpenAPI surface is unchanged (response shape and field names preserved across the audit). -- The equivalence test from the pareto redesign still passes (this is the binding behaviour gate). - -## Out of scope - -- Flask 3 / Connexion 3 migration (separate project). -- Pinning `ProcessOptimizer` to a release tag (no tagged release of the required commit exists at the time of this plan). -- Switching to async / background-only request handling (Phase 6+ if it ever happens). -- Coverage tooling, mutation testing, CI workflow changes. -- Documentation rewrites — `docs/api-usage.md` is current. diff --git a/docs/superpowers/plans/2026-05-18-pareto-extras-redesign.md b/docs/superpowers/plans/2026-05-18-pareto-extras-redesign.md deleted file mode 100644 index 5bb4810..0000000 --- a/docs/superpowers/plans/2026-05-18-pareto-extras-redesign.md +++ /dev/null @@ -1,993 +0,0 @@ -# Pareto `extras` Redesign Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Lock in the contract for `extras.selectedPoint` / `extras.pickled` / `extras.includeModel` such that `pickled` is a verified cache hint (never authoritative) and a no-`pickled` round-trip produces equivalent output to a `pickled` round-trip. - -**Architecture:** Pull pickle pack/unpack logic into a new `optimizerapi/pickled_state.py` module that owns the request-fingerprint computation, payload structure (`{"fingerprint", "result", "next", "optimizer"}`), and the four-stage validation flow (decrypt → structural → fingerprint → fast-path). `optimizer.py:run` delegates to it. A `result.extras.pickledUsed` boolean is added so the UI can observe fast-path usage. PNG + `selectedPoint` and `includeModel=false` + `pickled` log warnings but otherwise do exactly what the request asked for. - -**Tech Stack:** Python 3.9+, pytest, Connexion/Flask, Fernet (already wired via `optimizerapi/securepickle`), ProcessOptimizer. - -**Spec:** [`docs/superpowers/specs/2026-05-18-pareto-extras-redesign-design.md`](../specs/2026-05-18-pareto-extras-redesign-design.md) - ---- - -## File Map - -- **Create:** `optimizerapi/pickled_state.py` — fingerprint + pack/unpack helpers; sole owner of payload structure and fall-through logging. -- **Create:** `tests/test_pickled_state.py` — unit tests for fingerprint determinism, round-trip, and each fall-through reason. -- **Modify:** `optimizerapi/optimizer.py` — replace the inline unpickle block in `run`, thread fingerprint + `pickled_used` through to `process_result`, add the two new warning logs. -- **Modify:** `optimizerapi/openapi/specification.yml` — tighten descriptions for `extras.pickled` and `extras.selectedPoint`; add `result.extras.pickledUsed`. -- **Modify:** `tests/test_optimizer.py` — add the binding equivalence test, fingerprint-mismatch test, PNG+selectedPoint warning test, includeModel+pickled warning test, and pickledUsed round-trip test; refresh `test_old_format_pickled_falls_back` and `test_pickled_response_is_dict_format` for the new payload shape. - ---- - -### Task 1: Fingerprint helper in new module - -**Files:** -- Create: `optimizerapi/pickled_state.py` -- Test: `tests/test_pickled_state.py` - -- [ ] **Step 1: Write the failing test** - -Create `tests/test_pickled_state.py` with the following content: - -```python -"""Tests for pickled_state: fingerprint + pack/unpack helpers.""" - -from optimizerapi.pickled_state import compute_fingerprint - - -SAMPLE_DATA = [ - {"xi": [651, 56, 722, "Ræv"], "yi": [1]}, - {"xi": [651, 42, 722, "Ræv"], "yi": [0.2]}, -] - -SAMPLE_CONFIG = { - "baseEstimator": "GP", - "acqFunc": "gp_hedge", - "initialPoints": 2, - "kappa": 1.96, - "xi": 0.012, - "space": [ - {"type": "discrete", "name": "Sukker", "from": 0, "to": 1000}, - {"type": "continuous", "name": "Peber", "from": 0, "to": 1000}, - {"type": "continuous", "name": "Hvedemel", "from": 0, "to": 1000}, - {"type": "category", "name": "Kunde", "categories": ["Mus", "Ræv"]}, - ], -} - - -def test_fingerprint_is_deterministic(): - fp1 = compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) - fp2 = compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) - assert fp1 == fp2 - assert isinstance(fp1, str) - assert len(fp1) == 64 # sha256 hex - - -def test_fingerprint_is_independent_of_dict_key_order(): - reordered_config = { - "space": SAMPLE_CONFIG["space"], - "xi": SAMPLE_CONFIG["xi"], - "kappa": SAMPLE_CONFIG["kappa"], - "initialPoints": SAMPLE_CONFIG["initialPoints"], - "acqFunc": SAMPLE_CONFIG["acqFunc"], - "baseEstimator": SAMPLE_CONFIG["baseEstimator"], - } - assert compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) == compute_fingerprint( - SAMPLE_DATA, reordered_config - ) - - -def test_fingerprint_changes_when_data_changes(): - other_data = SAMPLE_DATA + [{"xi": [100, 100, 100, "Mus"], "yi": [0.5]}] - assert compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) != compute_fingerprint( - other_data, SAMPLE_CONFIG - ) - - -def test_fingerprint_changes_when_config_changes(): - other_config = dict(SAMPLE_CONFIG, kappa=2.0) - assert compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) != compute_fingerprint( - SAMPLE_DATA, other_config - ) -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `python -m pytest tests/test_pickled_state.py -v` -Expected: FAIL with `ModuleNotFoundError: No module named 'optimizerapi.pickled_state'`. - -- [ ] **Step 3: Write minimal implementation** - -Create `optimizerapi/pickled_state.py` with: - -```python -"""Pickled-state cache: fingerprint a request, pack/unpack pickled payloads. - -The fingerprint binds a pickled blob to the request that produced it, so a -client that sends a stale pickled along with newer data gets a clean fall- -through to a full run instead of silently buggy reuse. -""" - -import hashlib -import json - - -def compute_fingerprint(data, optimizer_config): - """Return sha256 hex of the canonical-JSON of (data, optimizerConfig). - - Canonical JSON: sorted keys, no whitespace. Numbers are emitted as Python's - default JSON representation, which is stable for the integer / float values - that flow through the API. - """ - payload = {"data": data, "optimizerConfig": optimizer_config} - canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")) - return hashlib.sha256(canonical.encode("utf-8")).hexdigest() -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `python -m pytest tests/test_pickled_state.py -v` -Expected: PASS, 4 tests. - -- [ ] **Step 5: Commit** - -```bash -git add optimizerapi/pickled_state.py tests/test_pickled_state.py -git commit -m "feat: add request fingerprint helper for pickled cache" -``` - ---- - -### Task 2: Pack + unpack with fall-through logging - -**Files:** -- Modify: `optimizerapi/pickled_state.py` -- Modify: `tests/test_pickled_state.py` - -- [ ] **Step 1: Write the failing tests (extend `tests/test_pickled_state.py`)** - -Append to `tests/test_pickled_state.py`: - -```python -import logging - -from optimizerapi.pickled_state import pack, unpack_if_valid -from optimizerapi.securepickle import get_crypto, pickleToString - - -def _crypto(): - return get_crypto() - - -def test_pack_unpack_round_trip(): - crypto = _crypto() - fingerprint = compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) - blob = pack(result="result-sentinel", next_points=["next-sentinel"], - optimizer="opt-sentinel", fingerprint=fingerprint, crypto=crypto) - assert isinstance(blob, str) and len(blob) > 0 - - payload = unpack_if_valid(blob, expected_fingerprint=fingerprint, crypto=crypto) - assert payload is not None - assert payload["result"] == "result-sentinel" - assert payload["next"] == ["next-sentinel"] - assert payload["optimizer"] == "opt-sentinel" - assert payload["fingerprint"] == fingerprint - - -def test_unpack_returns_none_on_fingerprint_mismatch(caplog): - crypto = _crypto() - fingerprint = compute_fingerprint(SAMPLE_DATA, SAMPLE_CONFIG) - blob = pack(result="r", next_points=[], optimizer="o", - fingerprint=fingerprint, crypto=crypto) - - with caplog.at_level(logging.WARNING, logger="optimizerapi.pickled_state"): - result = unpack_if_valid(blob, expected_fingerprint="0" * 64, crypto=crypto) - assert result is None - assert any("fingerprint_mismatch" in record.message for record in caplog.records) - - -def test_unpack_returns_none_on_decrypt_failure(caplog): - crypto = _crypto() - with caplog.at_level(logging.WARNING, logger="optimizerapi.pickled_state"): - result = unpack_if_valid("not-a-valid-blob", expected_fingerprint="x" * 64, - crypto=crypto) - assert result is None - assert any("decrypt_failed" in record.message for record in caplog.records) - - -def test_unpack_returns_none_on_bad_structure(caplog): - crypto = _crypto() - # Encrypt a non-dict payload using the same machinery. - bogus_blob = pickleToString(["not", "a", "dict"], crypto) - - with caplog.at_level(logging.WARNING, logger="optimizerapi.pickled_state"): - result = unpack_if_valid(bogus_blob, expected_fingerprint="x" * 64, - crypto=crypto) - assert result is None - assert any("bad_structure" in record.message for record in caplog.records) -``` - -- [ ] **Step 2: Run tests to verify they fail** - -Run: `python -m pytest tests/test_pickled_state.py -v` -Expected: FAIL with `ImportError: cannot import name 'pack' from 'optimizerapi.pickled_state'`. - -- [ ] **Step 3: Implement `pack` and `unpack_if_valid`** - -Append to `optimizerapi/pickled_state.py`: - -```python -import logging - -from .securepickle import pickleToString, unpickleFromString - -_LOG = logging.getLogger(__name__) - -_REQUIRED_KEYS = ("fingerprint", "result", "next", "optimizer") - - -def pack(*, result, next_points, optimizer, fingerprint, crypto): - """Encrypt a pickled cache payload for the given fingerprint.""" - payload = { - "fingerprint": fingerprint, - "result": result, - "next": next_points, - "optimizer": optimizer, - } - return pickleToString(payload, crypto) - - -def unpack_if_valid(blob, *, expected_fingerprint, crypto): - """Decrypt and validate a pickled cache payload. - - Returns the payload dict on success, or None if anything is off. Any - failure is logged once at WARNING with a reason tag in the message: - decrypt_failed | bad_structure | fingerprint_mismatch. - """ - if not blob: - return None - try: - payload = unpickleFromString(blob, crypto) - except Exception: - _LOG.warning("pickled cache ignored: decrypt_failed") - return None - if not isinstance(payload, dict) or not all(k in payload for k in _REQUIRED_KEYS): - _LOG.warning("pickled cache ignored: bad_structure") - return None - if payload["fingerprint"] != expected_fingerprint: - _LOG.warning("pickled cache ignored: fingerprint_mismatch") - return None - return payload -``` - -- [ ] **Step 4: Run tests to verify they pass** - -Run: `python -m pytest tests/test_pickled_state.py -v` -Expected: PASS, 8 tests total (4 from Task 1 + 4 new). - -- [ ] **Step 5: Commit** - -```bash -git add optimizerapi/pickled_state.py tests/test_pickled_state.py -git commit -m "feat: add pack/unpack_if_valid for pickled cache with fall-through logging" -``` - ---- - -### Task 3: Wire `pickled_state` into `optimizer.py` - -This task replaces the inline unpickle block at `optimizer.py:105-129` with a single call to `unpack_if_valid`, threads a `pickled_used` flag through `process_result`, and switches the repack at `optimizer.py:395` to use `pack`. The `pickled_used` flag surfaces as `result.extras.pickledUsed`. - -**Files:** -- Modify: `optimizerapi/optimizer.py:32` (imports) -- Modify: `optimizerapi/optimizer.py:68-167` (`run` function) -- Modify: `optimizerapi/optimizer.py:177-415` (`process_result` function) -- Modify: `tests/test_optimizer.py` (add the round-trip assertion test described below) - -- [ ] **Step 1: Write the failing test (`pickledUsed` round-trip)** - -Add this test to `tests/test_optimizer.py`, after `test_pickled_round_trip`: - -```python -def test_pickled_used_flag_round_trip(): - """First run has pickledUsed=False; second run with returned pickled has pickledUsed=True.""" - first = optimizer.run(body={ - "data": sampleData, - "optimizerConfig": sampleConfig, - "extras": {"includeModel": "true"}, - }) - assert first["result"]["extras"]["pickledUsed"] is False - pickled_value = first["result"]["pickled"] - assert len(pickled_value) > 0 - - second = optimizer.run(body={ - "data": sampleData, - "optimizerConfig": sampleConfig, - "extras": {"includeModel": "true", "pickled": pickled_value}, - }) - assert second["result"]["extras"]["pickledUsed"] is True -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `python -m pytest tests/test_optimizer.py::test_pickled_used_flag_round_trip -v` -Expected: FAIL — `pickledUsed` key not in `result["result"]["extras"]`. - -- [ ] **Step 3: Update the imports in `optimizer.py`** - -Replace the line: - -```python -from .securepickle import get_crypto, pickleToString, unpickleFromString -``` - -with: - -```python -from .securepickle import get_crypto -from .pickled_state import compute_fingerprint, pack, unpack_if_valid -``` - -- [ ] **Step 4: Replace the inline unpickle block in `run`** - -Inside `run`, replace the block currently at lines 105–129: - -```python - # Pickled consumption: attempt to skip training if extras.pickled is provided - pickled_input = extras.get("pickled", "") - if pickled_input: - try: - unpickled = unpickleFromString(pickled_input, get_crypto()) - if isinstance(unpickled, dict) and "result" in unpickled and "optimizer" in unpickled: - result = unpickled["result"] - optimizer = unpickled["optimizer"] - response = process_result(result, optimizer, dimensions, cfg, extras, data, space) - response["result"]["extras"]["parameters"] = { - "dimensions": dimensions, - "space": space, - "hyperparams": hyperparams, - "Xi": Xi, - "Yi": Yi, - "extras": extras, - } - return json.loads(json_tricks.dumps(response)) - else: - pickled_input = "" - except Exception: - logging.getLogger(__name__).warning( - "Failed to unpickle extras.pickled, falling back to full run" - ) - pickled_input = "" -``` - -with: - -```python - request_fingerprint = compute_fingerprint(body["data"], cfg) - pickled_input = extras.get("pickled", "") - cached = unpack_if_valid( - pickled_input, expected_fingerprint=request_fingerprint, crypto=get_crypto() - ) if pickled_input else None - - if cached is not None: - result = cached["result"] - optimizer = cached["optimizer"] - response = process_result( - result, optimizer, dimensions, cfg, extras, data, space, - request_fingerprint=request_fingerprint, pickled_used=True, - ) - response["result"]["extras"]["parameters"] = { - "dimensions": dimensions, - "space": space, - "hyperparams": hyperparams, - "Xi": Xi, - "Yi": Yi, - "extras": extras, - } - return json.loads(json_tricks.dumps(response)) -``` - -- [ ] **Step 5: Update the full-run path in `run` to pass the new kwargs** - -Find the existing call at `optimizer.py:154`: - -```python - response = process_result(result, optimizer, dimensions, cfg, extras, data, space) -``` - -Replace with: - -```python - response = process_result( - result, optimizer, dimensions, cfg, extras, data, space, - request_fingerprint=request_fingerprint, pickled_used=False, - ) -``` - -- [ ] **Step 6: Update `process_result` signature and body** - -Change the signature from: - -```python -def process_result(result, optimizer, dimensions, cfg, extras, data, space): -``` - -to: - -```python -def process_result(result, optimizer, dimensions, cfg, extras, data, space, - *, request_fingerprint, pickled_used): -``` - -Inside `process_result`, locate the existing line: - -```python - result_details = {"next": [], "models": [], "pickled": "", "extras": {}} -``` - -Immediately after the existing `add_version_info(result_details["extras"])` call near the end of `process_result`, set the flag: - -```python - result_details["extras"]["pickledUsed"] = pickled_used -``` - -Then replace the existing repack block: - -```python - if pickle_model: - result_details["pickled"] = pickleToString( - {"result": result, "next": result_details["next"], "optimizer": optimizer}, - get_crypto() - ) -``` - -with: - -```python - if pickle_model: - result_details["pickled"] = pack( - result=result, - next_points=result_details["next"], - optimizer=optimizer, - fingerprint=request_fingerprint, - crypto=get_crypto(), - ) -``` - -- [ ] **Step 7: Run the new test to verify it passes** - -Run: `python -m pytest tests/test_optimizer.py::test_pickled_used_flag_round_trip -v` -Expected: PASS. - -- [ ] **Step 8: Run the existing pickled tests to confirm no regression** - -Run: -```bash -python -m pytest tests/test_optimizer.py -k "pickled or selectedPoint" -v -``` -Expected: All targeted tests PASS. (One pre-existing unrelated failure `test_multi_objective_json_single_plots` is out of scope and not in this filter.) - -- [ ] **Step 9: Commit** - -```bash -git add optimizerapi/optimizer.py tests/test_optimizer.py -git commit -m "feat: delegate pickled cache to pickled_state and surface pickledUsed" -``` - ---- - -### Task 4: Binding equivalence test - -This is the test that enforces the north-star principle in §2 of the spec: the same request with and without `pickled` produces identical single-plot output. - -**Files:** -- Modify: `tests/test_optimizer.py` - -- [ ] **Step 1: Write the failing test** - -Add to `tests/test_optimizer.py` (placement: near the other multi-objective `selectedPoint` tests): - -```python -def test_equivalence_with_and_without_pickled_multi_objective(): - """Same selectedPoint, same data: pickled vs no-pickled produce identical single plots.""" - base_body = { - "data": sampleMultiObjective5DimData, - "optimizerConfig": sampleMultiObjective5DimConfig, - "extras": { - "graphs": ["single", "pareto"], - "graphFormat": "json", - "selectedPoint": [50, 833, 150, 60, "Whipped cream"], - "includeModel": "true", - }, - } - - # Seed: one full run to capture pickled. - seed = optimizer.run(body=copy.deepcopy({ - **base_body, - "extras": {**base_body["extras"], "selectedPoint": None}, - })) - pickled_value = seed["result"]["pickled"] - assert len(pickled_value) > 0 - - # Run A: no pickled (full retrain), with selectedPoint. - body_no_pickle = copy.deepcopy(base_body) - result_no_pickle = optimizer.run(body=body_no_pickle) - - # Run B: same request + pickled. - body_with_pickle = copy.deepcopy(base_body) - body_with_pickle["extras"]["pickled"] = pickled_value - result_with_pickle = optimizer.run(body=body_with_pickle) - - # Sanity: fast path actually engaged. - assert result_with_pickle["result"]["extras"]["pickledUsed"] is True - assert result_no_pickle["result"]["extras"]["pickledUsed"] is False - - # Compare every plot entry except the pickled string itself (which is - # not in plots) — identical plot ids and identical plot bodies. - plots_no_pickle = {p["id"]: p["plot"] for p in result_no_pickle["plots"]} - plots_with_pickle = {p["id"]: p["plot"] for p in result_with_pickle["plots"]} - assert set(plots_no_pickle) == set(plots_with_pickle) - for plot_id in plots_no_pickle: - assert plots_no_pickle[plot_id] == plots_with_pickle[plot_id], ( - f"divergence on plot {plot_id}" - ) -``` - -If `sampleMultiObjective5DimData` and `sampleMultiObjective5DimConfig` are not defined as module-level fixtures yet, define them near the top of the test file (mirroring the values from `scripts/sample-multi.curl`): - -```python -sampleMultiObjective5DimData = [ - {"xi": [16.7, 500, 250, 20, "None"], "yi": [-2, -17]}, - {"xi": [50, 833, 150, 60, "Whipped cream"], "yi": [-3, -6]}, - {"xi": [58.3, 22, 85, 6, "Frosting"], "yi": [-6, -25]}, -] - -sampleMultiObjective5DimConfig = { - "baseEstimator": "GP", - "acqFunc": "EI", - "initialPoints": 3, - "kappa": 1.96, - "xi": 2, - "space": [ - {"type": "continuous", "name": "Sugar", "from": 0, "to": 100}, - {"type": "continuous", "name": "Flour", "from": 0, "to": 1000}, - {"type": "discrete", "name": "Temperature", "from": 0, "to": 300}, - {"type": "discrete", "name": "Time", "from": 0, "to": 120}, - {"type": "category", "name": "Finish", "categories": ["None", "Frosting", "Whipped cream"]}, - ], - "constraints": [], -} -``` - -(Check first whether equivalents already exist further down `tests/test_optimizer.py` — search for `sampleMultiObjective5Dim`. If they do, reuse them and skip the redefinition.) - -- [ ] **Step 2: Run test to verify it passes** - -Run: `python -m pytest tests/test_optimizer.py::test_equivalence_with_and_without_pickled_multi_objective -v` -Expected: PASS. If it FAILS with a plot divergence, that is a real regression — investigate before continuing. Likely culprits: the fast path silently skipping `selectedPoint`, or `_get_brownie_bee_1d_plot_safe` being called with different state on the two paths. - -- [ ] **Step 3: Commit** - -```bash -git add tests/test_optimizer.py -git commit -m "test: assert equivalence of pickled and full-run plots" -``` - ---- - -### Task 5: Fingerprint-mismatch test - -**Files:** -- Modify: `tests/test_optimizer.py` - -- [ ] **Step 1: Write the failing test** - -Add to `tests/test_optimizer.py`: - -```python -def test_pickled_fingerprint_mismatch_falls_through(caplog): - """A pickled produced from one data set is ignored when data changes.""" - import logging as _logging - - seed = optimizer.run(body={ - "data": sampleData, - "optimizerConfig": sampleConfig, - "extras": {"includeModel": "true"}, - }) - pickled_value = seed["result"]["pickled"] - assert len(pickled_value) > 0 - - altered_data = sampleData + [{"xi": [100, 100, 100, "Mus"], "yi": [0.5]}] - - with caplog.at_level(_logging.WARNING, logger="optimizerapi.pickled_state"): - result = optimizer.run(body={ - "data": altered_data, - "optimizerConfig": sampleConfig, - "extras": {"includeModel": "true", "pickled": pickled_value}, - }) - - assert result["result"]["extras"]["pickledUsed"] is False - assert any("fingerprint_mismatch" in r.message for r in caplog.records) - # Response still valid — server fell through to a full run. - assert "next" in result["result"] - assert len(result["result"]["next"]) > 0 -``` - -- [ ] **Step 2: Run test to verify it passes** - -Run: `python -m pytest tests/test_optimizer.py::test_pickled_fingerprint_mismatch_falls_through -v` -Expected: PASS. - -- [ ] **Step 3: Commit** - -```bash -git add tests/test_optimizer.py -git commit -m "test: fingerprint mismatch falls through with warning" -``` - ---- - -### Task 6: Warn when PNG + `selectedPoint` are combined - -**Files:** -- Modify: `optimizerapi/optimizer.py` (`process_result`) -- Modify: `tests/test_optimizer.py` - -- [ ] **Step 1: Write the failing test** - -Add to `tests/test_optimizer.py`: - -```python -def test_selectedPoint_with_png_logs_warning_and_is_ignored(caplog): - import logging as _logging - - with caplog.at_level(_logging.WARNING, logger="optimizerapi.optimizer"): - result = optimizer.run(body={ - "data": sampleData, - "optimizerConfig": sampleConfig, - "extras": { - "graphFormat": "png", - "selectedPoint": [651, 56, 722, "Ræv"], - "includeModel": "false", - }, - }) - - assert any("selectedPoint ignored on png path" in r.message for r in caplog.records) - # No crash, response is valid. - assert "plots" in result - assert "next" in result["result"] -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `python -m pytest tests/test_optimizer.py::test_selectedPoint_with_png_logs_warning_and_is_ignored -v` -Expected: FAIL — warning not emitted. - -- [ ] **Step 3: Add the warning in `process_result`** - -In `optimizerapi/optimizer.py`, inside `process_result`, right after the existing lines that read `graph_format` and `selected_point`: - -```python - graph_format = extras.get("graphFormat", "png") - # ... - pickle_model = json.loads(extras.get("includeModel", "true").lower()) - selected_point = extras.get("selectedPoint") -``` - -append: - -```python - if selected_point is not None and graph_format != "json": - logging.getLogger(__name__).warning( - "selectedPoint ignored on png path (graphFormat=%s)", graph_format - ) -``` - -(No other behavior change — the PNG branch already never reads `selected_point`.) - -- [ ] **Step 4: Run test to verify it passes** - -Run: `python -m pytest tests/test_optimizer.py::test_selectedPoint_with_png_logs_warning_and_is_ignored -v` -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add optimizerapi/optimizer.py tests/test_optimizer.py -git commit -m "feat: warn when selectedPoint is sent with png graphFormat" -``` - ---- - -### Task 7: Warn when `includeModel=false` + `pickled` are combined - -**Files:** -- Modify: `optimizerapi/optimizer.py` (`run`) -- Modify: `tests/test_optimizer.py` - -- [ ] **Step 1: Write the failing test** - -Add to `tests/test_optimizer.py`: - -```python -def test_includeModel_false_with_pickled_logs_chain_break_warning(caplog): - import logging as _logging - - seed = optimizer.run(body={ - "data": sampleData, - "optimizerConfig": sampleConfig, - "extras": {"includeModel": "true"}, - }) - pickled_value = seed["result"]["pickled"] - - with caplog.at_level(_logging.WARNING, logger="optimizerapi.optimizer"): - second = optimizer.run(body={ - "data": sampleData, - "optimizerConfig": sampleConfig, - "extras": {"pickled": pickled_value, "includeModel": "false"}, - }) - - assert any("includeModel=false with extras.pickled" in r.message for r in caplog.records) - # Contract preserved: empty pickled returned. - assert second["result"]["pickled"] == "" -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `python -m pytest tests/test_optimizer.py::test_includeModel_false_with_pickled_logs_chain_break_warning -v` -Expected: FAIL. - -- [ ] **Step 3: Add the warning in `run`** - -In `optimizerapi/optimizer.py`, inside `run`, after the `extras` dict is read and after `pickled_input` is computed (Task 3 put `pickled_input = extras.get("pickled", "")` right above the `cached = ...` line), add: - -```python - if pickled_input: - include_model_str = str(extras.get("includeModel", "true")).lower() - if include_model_str == "false": - logging.getLogger(__name__).warning( - "includeModel=false with extras.pickled — next call will pay the full cost" - ) -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `python -m pytest tests/test_optimizer.py::test_includeModel_false_with_pickled_logs_chain_break_warning -v` -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add optimizerapi/optimizer.py tests/test_optimizer.py -git commit -m "feat: warn when includeModel=false drops a returnable pickled cache" -``` - ---- - -### Task 8: OpenAPI spec updates - -**Files:** -- Modify: `optimizerapi/openapi/specification.yml` - -- [ ] **Step 1: Tighten `extras.pickled` description** - -In `specification.yml`, find: - -```yaml - pickled: - description: Previously-pickled model state to skip expensive GP model retraining - type: string -``` - -Replace the description with: - -```yaml - pickled: - description: > - Opaque cache hint produced by a previous response. The server - validates it matches the current data / optimizerConfig; on - mismatch it is ignored and a full run is performed. - type: string -``` - -- [ ] **Step 2: Tighten `extras.selectedPoint` description** - -Find: - -```yaml - selectedPoint: - description: Override the highlight point in JSON single plots with explicit X-space coordinates - type: array -``` - -Replace the description with: - -```yaml - selectedPoint: - description: > - Override the highlight point in single plots with explicit - X-space coordinates. Honored only when graphFormat is "json"; - ignored on the PNG path. - type: array -``` - -- [ ] **Step 3: Add `pickledUsed` to the response result schema** - -Locate the `result` schema (around line 264) and find the inner `result:` block: - -```yaml - result: - type: object - properties: - expected_minimum: - ... - pickled: - type: string - next: - ... - models: - ... - extras: - type: object -``` - -Change the trailing `extras:` block to: - -```yaml - extras: - type: object - properties: - pickledUsed: - description: True iff the server reused the pickled cache hint for this response. - type: boolean -``` - -- [ ] **Step 4: Confirm OpenAPI still parses** - -Run: `python -m optimizerapi.server` and confirm it boots without YAML/OpenAPI errors. Hit `Ctrl-C` once the line `Serving on http://...` (or the Connexion startup line) appears. Then close. - -Alternative non-interactive check: - -```bash -python -c "import yaml; yaml.safe_load(open('optimizerapi/openapi/specification.yml'))" -``` - -Expected: no traceback. - -- [ ] **Step 5: Commit** - -```bash -git add optimizerapi/openapi/specification.yml -git commit -m "docs(openapi): tighten descriptions for pickled/selectedPoint, add pickledUsed" -``` - ---- - -### Task 9: Refresh legacy pickled tests - -The payload structure is now `{"fingerprint", "result", "next", "optimizer"}`. Two existing tests need their expectations refreshed. - -**Files:** -- Modify: `tests/test_optimizer.py` - -- [ ] **Step 1: Update `test_old_format_pickled_falls_back`** - -The current test (at the location shown by `grep -n "test_old_format_pickled_falls_back" tests/test_optimizer.py`) constructs an old-format list and expects fall-through. Under the new code path this now flows through `bad_structure`. The functional assertion still holds; add an explicit warning assertion and a `pickledUsed=False` assertion to make it precise. - -Replace the test body with: - -```python -def test_old_format_pickled_falls_back(caplog): - """A pickled payload that decrypts but isn't the new dict shape falls through.""" - import logging as _logging - - old_format_data = ["some", "old", "data"] - old_pickled = pickleToString(old_format_data, get_crypto()) - - with caplog.at_level(_logging.WARNING, logger="optimizerapi.pickled_state"): - result = optimizer.run(body={ - "data": sampleData, - "optimizerConfig": sampleConfig, - "extras": {"pickled": old_pickled, "includeModel": "false"}, - }) - - assert "result" in result - assert len(result["result"]["next"]) > 0 - assert result["result"]["extras"]["pickledUsed"] is False - assert any("bad_structure" in r.message for r in caplog.records) -``` - -- [ ] **Step 2: Update `test_pickled_response_is_dict_format`** - -The existing test asserts the response pickled is a dict with `result`, `next`, `optimizer`. Add `fingerprint` to the assertions so the new contract is locked. - -Locate the test and replace its tail-end assertions (after `unpickled = unpickleFromString(pickled_value, get_crypto())`) with: - -```python - assert isinstance(unpickled, dict), f"Expected dict, got {type(unpickled)}" - assert set(unpickled.keys()) >= {"fingerprint", "result", "next", "optimizer"} - assert isinstance(unpickled["fingerprint"], str) and len(unpickled["fingerprint"]) == 64 -``` - -- [ ] **Step 3: Update `test_invalid_pickled_falls_back`** - -This test sends raw garbage and expects fall-through. Add `pickledUsed` and warning assertions for symmetry: - -```python -def test_invalid_pickled_falls_back(caplog): - """An undecodable pickled string falls through to a full run.""" - import logging as _logging - - with caplog.at_level(_logging.WARNING, logger="optimizerapi.pickled_state"): - result = optimizer.run(body={ - "data": sampleData, - "optimizerConfig": sampleConfig, - "extras": {"pickled": "this_is_not_valid_pickled_data_at_all", "includeModel": "false"}, - }) - - assert "result" in result - assert len(result["result"]["next"]) > 0 - assert result["result"]["extras"]["pickledUsed"] is False - assert any("decrypt_failed" in r.message for r in caplog.records) -``` - -- [ ] **Step 4: Run the three updated tests** - -Run: -```bash -python -m pytest tests/test_optimizer.py::test_old_format_pickled_falls_back \ - tests/test_optimizer.py::test_pickled_response_is_dict_format \ - tests/test_optimizer.py::test_invalid_pickled_falls_back -v -``` -Expected: all PASS. - -- [ ] **Step 5: Commit** - -```bash -git add tests/test_optimizer.py -git commit -m "test: refresh legacy pickled tests for new payload shape and warnings" -``` - ---- - -### Task 10: Full suite + lint - -**Files:** none new. - -- [ ] **Step 1: Run the full test suite** - -Run: `python -m pytest` -Expected: All tests pass except the pre-existing `test_multi_objective_json_single_plots` (out of scope per the spec). If anything else fails, stop and investigate; do not paper over by adjusting tests. - -- [ ] **Step 2: Run flake8** - -Run: `flake8 . --max-line-length=127` -Expected: no warnings. - -- [ ] **Step 3: Smoke-check the server boots** - -Run: `python -m optimizerapi.server` in one shell; wait for the startup line; then `Ctrl-C`. Confirms the OpenAPI parses end-to-end and there are no import errors from the new module. - -- [ ] **Step 4: Final commit (if any cleanup landed during steps 1–3)** - -If steps 1–3 surfaced fixes: - -```bash -git add -A -git commit -m "chore: final cleanup after pareto extras redesign" -``` - -Otherwise skip. - ---- - -## Out of scope (do not implement here) - -- Server-side session cache with short-ID `extras.pickleHandle`. -- Migrating `includeModel` to a real boolean. -- A structured `pickledRejectedReason` in the response. -- Fixing `test_multi_objective_json_single_plots`. -- Modifications to ProcessOptimizer source or `_get_brownie_bee_1d_plot_safe`. diff --git a/docs/superpowers/specs/2026-05-18-pareto-extras-redesign-design.md b/docs/superpowers/specs/2026-05-18-pareto-extras-redesign-design.md deleted file mode 100644 index 2760b51..0000000 --- a/docs/superpowers/specs/2026-05-18-pareto-extras-redesign-design.md +++ /dev/null @@ -1,130 +0,0 @@ -# Pareto `extras` Redesign — Design - -**Date:** 2026-05-18 -**Branch:** `langdal/ai-handoff-pareto` -**Status:** Design (awaiting plan) -**Supersedes the `extras.pickled` / `extras.selectedPoint` decisions in:** `.sisyphus/plans/pareto-point-selection.md` - -## 1. Goal - -Lock in the contract for the three `extras` fields that drive the pareto-point UI flow — `selectedPoint`, `pickled`, `includeModel` — under a single north-star principle, and produce a stable, equivalence-checkable API before merging the `pareto` branch. - -## 2. North-star principle - -> **The pickled model is an *efficiency* feature only. The full round-trip (user clicks a pareto point → server re-renders single plots) MUST work without `pickled`, just slower.** - -Made formal: - -> **Equivalence guarantee.** For any request `R`, the response to `R` and to `R ∪ {extras.pickled: P}` are observably equivalent — ignoring `result.pickled` and timing — provided `P` was produced by an earlier response whose `data` and `optimizerConfig` matched `R`. - -Authoritative inputs are `data` and `optimizerConfig`. The server NEVER trusts state inside `pickled` to override them. The client MUST always send `data`. - -## 3. Contract - -### 3.1 `extras.selectedPoint` - -- **Shape:** unchanged. Raw X-space coordinates as a list mixing numbers and category strings, matching the `space` dimensions in order. Example: `[50, 833, 150, 60, "Whipped cream"]`. -- **Rationale for keeping raw coords (vs. an index into the prior `pareto_data.front_x_data`):** stateless, self-contained, no coupling to prior response shape, no dependency on deterministic re-derivation of the pareto. The UI already has the coords; sending them back is free. -- **Honored when:** `extras.graphFormat == "json"`. PNG path does not support `selectedPoint`. -- **PNG + `selectedPoint`:** a `warning` is logged; the field is ignored; the response is otherwise unchanged. No error. -- **OpenAPI:** description updated to state the `graphFormat: "json"` precondition explicitly. - -### 3.2 `extras.pickled` - -- **Shape:** opaque string blob, as today. Clients SHOULD treat it as opaque and round-trip it verbatim. -- **Server-side payload structure (Fernet-encrypted, pickled):** - ``` - { - "fingerprint": "", - "result": , - "next": , - "optimizer": , - } - ``` - No version field — the encryption-key rotation (see §5) makes prior payloads undecryptable, so no in-tree migration logic is required. -- **Fingerprint definition:** `sha256_hex(canonical_json({"data": data, "optimizerConfig": optimizerConfig}))`, where `data` and `optimizerConfig` are the raw top-level request fields. `canonical_json` means sorted keys, no whitespace, stable number representation. The exact serializer choice is an implementation detail and lives in `pickled_state.py`. Fingerprinting on the raw request fields (rather than on internally reconstructed `space` / `hyperparams` / `constraints`) keeps the contract independent of internal refactoring. -- **Validation flow on incoming `extras.pickled`:** - 1. Decrypt with current `PICKLE_KEY`. Failure → fall through. - 2. Unpickle to a dict. Wrong type / missing keys → fall through. - 3. Compare `payload["fingerprint"]` to the fingerprint recomputed from the current request. Mismatch → fall through. - 4. All pass → fast path: reuse `payload["result"]` and `payload["optimizer"]`; skip `optimizer.tell(...)`. -- **Fall-through behavior:** a single `warning` log line with a reason tag (`decrypt_failed`, `bad_structure`, `fingerprint_mismatch`), and a full run is performed. The response is otherwise equivalent to one without `extras.pickled`. - -### 3.3 `extras.includeModel` - -- **Behavior:** unchanged. Controls whether `result.pickled` is populated in the response. -- **Typing:** unchanged — stringly-typed `"true"`/`"false"` parsed via `json.loads(...lower())`. Tightening to a real boolean is a separate, out-of-scope ticket; existing clients depend on the string form. -- **`includeModel: "false"` + `extras.pickled` sent in:** accepted, documented. The server uses the fast path and returns an empty `result.pickled`. A `warning` is logged so the chain-break is debuggable. The client is asking for exactly what it asked for; no auto-override. - -### 3.4 New response field: `result.extras.pickledUsed` - -- **Type:** boolean. -- **Set to `True` iff** the fast path was taken (all four validation steps in §3.2 passed). -- **Purpose:** lets the UI tell whether its cache hint was honored without inferring from timing. Optional for callers to read. - -### 3.5 Response schema (recap of changes) - -- `result.extras.pickledUsed: boolean` added. -- No other response-shape changes. - -## 4. Code organization - -- New module: `optimizerapi/pickled_state.py`, exposing: - - `compute_fingerprint(data, optimizerConfig) -> str` - - `pack(result, next_points, optimizer, fingerprint, crypto) -> str` - - `unpack_if_valid(blob, expected_fingerprint, crypto) -> Optional[dict]` — returns the inner dict on success, `None` on any failure, and logs the reason on failure. -- `optimizerapi/optimizer.py`: - - The unpickle block at ~lines 105–129 collapses to a single call to `unpack_if_valid(...)`. The `if/else/except` chain that exists today is replaced. - - The pickle block at ~line 395 calls `pack(...)` with the request fingerprint computed earlier in `run`. - - In `process_result`, set `result_details["extras"]["pickledUsed"]` according to whether the fast path ran. - - Add `warning` logs for: - - PNG + `selectedPoint` (in `process_result`, where `selected_point` is read). - - `includeModel: "false"` + `extras.pickled` (in `run`, after the pickled path decision is made). - -The point of pulling `pickled_state.py` out is to keep `optimizer.py:run` focused on optimizer-lifecycle code, and to make the fingerprint logic and fall-through paths independently testable. - -## 5. Encryption-key rotation (operational) - -- Rotate `PICKLE_KEY` in any environment that may hold pre-redesign pickled blobs. Operationally this means: deploy with a new key. No in-tree migration code is required; old blobs simply fail to decrypt and the existing fall-through path takes over. -- There are no in-production users of the `pickled` feature at this point, so this is risk-free. -- Mention the rotation requirement in the PR description / release notes for downstream consumers. - -## 6. OpenAPI changes (`specification.yml`) - -- `extras.pickled` description tightened to: "Opaque cache hint produced by a previous response. The server validates it matches the current `data` / `optimizerConfig`; on mismatch it is ignored and a full run is performed." -- `extras.selectedPoint` description appended with: "Honored only when `graphFormat` is `\"json\"`. Ignored on the PNG path." -- `result.extras` gains an optional `pickledUsed: boolean`. - -## 7. Tests (new) - -- **Equivalence test (the binding test for §2).** Run a multi-objective request twice — once without `pickled`, once with the `pickled` from the first response — both with the same `selectedPoint`. Assert that all `single_*` / `objective_*` plot entries are byte-equal (or numerically equal within tolerance for any floating-point reductions). -- **Fingerprint mismatch.** Send `pickled` from a previous response together with a modified `data` array. Assert: response matches a no-`pickled` run for the new `data`; `result.extras.pickledUsed == False`; one `warning` logged with reason `fingerprint_mismatch`. -- **Decrypt-failure fall-through.** Send a garbage `pickled` string. Assert: full run executed; `pickledUsed == False`; warning logged with reason `decrypt_failed`. -- **Bad-structure fall-through.** Decrypt-able but missing keys. Assert fall-through with reason `bad_structure`. -- **`pickledUsed` round-trip.** Two-call sequence: first call sets `pickledUsed == False`; second call (with returned `pickled`) sets `pickledUsed == True`. -- **PNG + `selectedPoint`.** Send PNG + `selectedPoint`. Assert: no error; warning logged; PNG plots returned and unaffected by `selectedPoint` value. -- **`includeModel: "false"` + `pickled`.** Existing `test_pickled_consumption_skips_training` already exercises this. Extend it to also assert that a warning is logged. - -Existing tests for `selectedPoint` and old-format pickled should be reviewed: the old-format test (`test_old_format_pickled_falls_back`) likely still passes via the decrypt-failure or bad-structure path under the rotated key, but its assertions may need refreshing. - -## 8. Out of scope - -- Server-side session cache + short-ID `extras.pickleHandle` (deferred; the opaque-blob contract preserves a clean swap path). -- Migrating `includeModel` to a real boolean. -- Surfacing a structured `pickledRejectedReason` to the UI. Logs are sufficient until proven otherwise. -- Modifying `ProcessOptimizer` source. -- Changing `expected_minimum` computation. -- The `_get_brownie_bee_1d_plot_safe` workaround — it stays. -- The pre-existing `test_multi_objective_json_single_plots` failure. - -## 9. Acceptance criteria - -A reviewer can verify the work is done by: - -1. The equivalence test in §7 passes. -2. All fall-through paths log exactly one `warning` with the documented reason tag. -3. `result.extras.pickledUsed` correctly reflects fast-path vs. full-run on a two-call sequence. -4. `specification.yml` reflects the description and schema changes in §6. -5. `optimizer.py:run` no longer contains inline unpickle/repack logic — it delegates to `pickled_state.py`. -6. PNG path is unchanged in behavior; `selectedPoint` is logged-and-ignored on that path. -7. `python -m pytest` is green except for the documented pre-existing `test_multi_objective_json_single_plots` failure; `flake8 . --max-line-length=127` is clean.