Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 13 additions & 3 deletions optimizerapi/optimizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -346,21 +346,31 @@ def process_result(
if optimizer.n_objectives == 1:
_set_expected_minimum(result_details, result[0], optimizer.space)

optimal_point = None
if optimizer.n_objectives == 2 and "pareto" in parsed.graphs_to_return:
emit_pareto_data(plots, optimizer)
optimal_point = emit_pareto_data(plots, optimizer)

if optimizer.n_objectives == 2 and "single" in parsed.graphs_to_return:
# Default the 1D plots to the model's optimal (compromise) point
# so both objectives describe the *same* settings. Without an
# explicit point each objective falls back to its own expected
# minimum, leaving the two rows misaligned.
point = (
parsed.selected_point
if parsed.selected_point is not None
else optimal_point
)
emit_json_single_plots(
plots,
result=result[0],
prefix="objective_1",
selected_point=parsed.selected_point,
selected_point=point,
)
emit_json_single_plots(
plots,
result=result[1],
prefix="objective_2",
selected_point=parsed.selected_point,
selected_point=point,
)

if parsed.include_model:
Expand Down
55 changes: 43 additions & 12 deletions optimizerapi/plot_emitters.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

import base64
import io
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, cast

import json_tricks
import matplotlib.pyplot as plt
Expand Down Expand Up @@ -78,26 +78,46 @@ def emit_json_single_plots(
X-space coordinates to highlight, or ``None`` to use the default.
"""
one_d_data = _get_brownie_bee_1d_plot_safe(result, x_eval=selected_point)
histogram_entry = one_d_data[-1]
for i, dim_data in enumerate(one_d_data[:-1]):
plots.append(
{"id": f"{prefix}_{i}", "plot": json_tricks.dumps({"data": dim_data})}
)
mean, std = _histogram_mean_std(result, selected_point, one_d_data[-1])
plots.append(
{
"id": f"{prefix}_{len(one_d_data) - 1}",
"plot": json_tricks.dumps(
{
"histogram": {
"mean": float(numpy.ravel(histogram_entry[0])[0]),
"std": float(numpy.ravel(histogram_entry[1])[0]),
}
}
),
"plot": json_tricks.dumps({"histogram": {"mean": mean, "std": std}}),
}
)


def _histogram_mean_std(
result: Any,
selected_point: "list[str | float] | None",
fallback_entry: Any,
) -> "tuple[float, float]":
"""Predicted-score mean/std for the histogram at ``selected_point``.

When no point is given, ``get_Brownie_Bee_1d_plot`` derives these from
``expected_minimum`` (correct), so we reuse its last entry. For an explicit
point it instead predicts on the *raw* point — but the model is fitted in
transformed space, so the raw point lands outside the normalized domain and
the prediction collapses to the prior mean (constant across Pareto points)
for numeric spaces, and errors for categorical ones. Predict on the
transformed point here so the histogram actually tracks the selection.
"""
if selected_point is None:
return (
float(numpy.ravel(fallback_entry[0])[0]),
float(numpy.ravel(fallback_entry[1])[0]),
)
model = result.models[-1]
mean, std = model.predict(
numpy.asarray(result.space.transform([selected_point])), return_std=True
)
return float(numpy.ravel(mean)[0]), float(numpy.ravel(std)[0])


def emit_png_plots(
plots: "list[Plot]",
*,
Expand Down Expand Up @@ -131,8 +151,15 @@ def emit_png_plots(
_emit_current_figure(plots, f"single_{idx}")


def emit_pareto_data(plots: "list[Plot]", optimizer: object) -> None:
"""Append the pareto-front payload (multi-objective only)."""
def emit_pareto_data(plots: "list[Plot]", optimizer: object) -> "list | None":
"""Append the pareto-front payload (multi-objective only).

Returns the x-space coordinates of the model's compromise/optimal point
(``front_x_data[best_idx]``) so the caller can use it as the default
``selected_point`` for the per-objective 1D plots — without it, each
objective's plots default to its own expected minimum and the two rows end
up describing different settings. Returns ``None`` if no front is available.
"""
front_x_data, front_y_data, obj1_error, obj2_error = get_Brownie_Bee_Pareto(
optimizer, n_points=200
)
Expand All @@ -146,6 +173,10 @@ def emit_pareto_data(plots: "list[Plot]", optimizer: object) -> None:
}
plots.append({"id": "pareto_data", "plot": json_tricks.dumps(pareto_data)})

if best_idx is None or len(front_x_data) == 0:
return None
return cast("list", front_x_data[best_idx].tolist())


def _figure_to_b64(figure) -> str:
buf = io.BytesIO()
Expand Down
118 changes: 118 additions & 0 deletions tests/test_e2e_pareto.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,39 @@
"constraints": [],
}

# Numeric-only multi-objective data (the built-in "CFPS multi objective"
# example). Unlike MULTI_OBJECTIVE_DATA above it has no categorical factor, so
# it exercises the all-numeric code path of get_Brownie_Bee_1d_plot — where the
# score histogram (predicted-score distribution at the selected point) used to
# be computed on the *raw* (untransformed) point and therefore never changed
# between Pareto points.
NUMERIC_MULTI_OBJECTIVE_DATA = [
{"xi": [1, 40, 0.5], "yi": [0.608, 0.0833]},
{"xi": [5, 120, 2.5], "yi": [1.9085, 2.0833]},
{"xi": [3, 80, 1.5], "yi": [1.3638, 0.75]},
{"xi": [5.5, 30, 0.25], "yi": [1.1523, 0.8542]},
{"xi": [2, 140, 2], "yi": [1.0272, 1.5556]},
{"xi": [0, 100, 1], "yi": [0.3739, 0.5556]},
{"xi": [4, 60, 2.75], "yi": [1.2057, 1.3958]},
{"xi": [3.5, 130, 0.75], "yi": [1.6283, 1.2431]},
{"xi": [1.5, 70, 2.25], "yi": [0.8189, 0.7986]},
{"xi": [0.5, 90, 0], "yi": [0.4268, 0.3472]},
]

NUMERIC_MULTI_OBJECTIVE_CONFIG = {
"baseEstimator": "GP",
"acqFunc": "EI",
"initialPoints": 4,
"kappa": 1.96,
"xi": 0.01,
"space": [
{"type": "continuous", "name": "Magnesium", "from": 0, "to": 6},
{"type": "discrete", "name": "Potassium", "from": 20, "to": 140},
{"type": "continuous", "name": "DTT", "from": 0, "to": 3},
],
"constraints": [],
}


@pytest.mark.filterwarnings("ignore::DeprecationWarning")
class TestE2EParetoFront:
Expand Down Expand Up @@ -182,6 +215,91 @@ def test_selectedPoint_highlights_correct_point(self, app_client, api_key):
"selectedPoint should be reflected in the single plot highlight"
)

def test_score_histogram_tracks_selected_point_numeric_space(
self, app_client, api_key
):
"""The score histogram must follow the selected Pareto point.

Regression for the "histograms never change when clicking the Pareto
front" bug: get_Brownie_Bee_1d_plot computed the histogram mean/std via
model.predict() on the *raw* point, but the GP is fitted in transformed
space. For an all-numeric space the raw point sits far outside the
normalized domain, so every point collapsed to the prior mean — the
histogram was identical regardless of selection (while the per-factor
plots, which transform x_eval themselves, did move).
"""

def histogram_for(selected_point):
payload = self.build_request(
data=NUMERIC_MULTI_OBJECTIVE_DATA,
optimizer_config=NUMERIC_MULTI_OBJECTIVE_CONFIG,
extras={
"graphs": ["single"],
"graphFormat": "json",
"selectedPoint": selected_point,
},
)
response = app_client.post(
f"{self.BASE_URL}?apikey={api_key}",
json=payload,
content_type="application/json",
)
assert response.status_code == 200
plots = response.get_json()["plots"]
# The histogram is the last entry per objective: one plot per space
# dimension (3) followed by the histogram at index 3.
out = {}
for obj in ("objective_1", "objective_2"):
hist_plot = next(p for p in plots if p["id"] == f"{obj}_3")
out[obj] = json.loads(hist_plot["plot"])["histogram"]
return out

# First fetch the Pareto front and pick two genuinely different
# trade-offs: best quality vs best cost.
front_payload = self.build_request(
data=NUMERIC_MULTI_OBJECTIVE_DATA,
optimizer_config=NUMERIC_MULTI_OBJECTIVE_CONFIG,
extras={"graphs": ["pareto"], "graphFormat": "json"},
)
front_resp = app_client.post(
f"{self.BASE_URL}?apikey={api_key}",
json=front_payload,
content_type="application/json",
)
assert front_resp.status_code == 200
pareto = json.loads(
next(p for p in front_resp.get_json()["plots"] if p["id"] == "pareto_data")[
"plot"
]
)
front_x = pareto["front_x_data"]
front_y = pareto["front_y_data"]
i_best_quality = min(range(len(front_y)), key=lambda i: front_y[i][0])
i_best_cost = min(range(len(front_y)), key=lambda i: front_y[i][1])

# Sanity: the front must actually offer a trade-off, else the test is
# vacuous.
assert front_y[i_best_quality][0] != front_y[i_best_cost][0]
assert front_y[i_best_quality][1] != front_y[i_best_cost][1]

at_quality = histogram_for(front_x[i_best_quality])
at_cost = histogram_for(front_x[i_best_cost])

for obj in ("objective_1", "objective_2"):
assert at_quality[obj]["mean"] != at_cost[obj]["mean"], (
f"{obj} histogram mean did not change between two different "
f"Pareto points (frozen histogram bug)"
)

# And the histogram mean should track the predicted score at the
# selected point (front_y), not collapse to a constant prior mean.
assert at_quality["objective_1"]["mean"] == pytest.approx(
front_y[i_best_quality][0], abs=0.1
)
assert at_cost["objective_2"]["mean"] == pytest.approx(
front_y[i_best_cost][1], abs=0.1
)

def test_full_pareto_exploration_workflow(self, app_client, api_key):
"""
Test the complete pareto exploration workflow:
Expand Down
30 changes: 28 additions & 2 deletions tests/test_plot_emitters.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,26 @@ def test_emit_json_single_plots_supports_different_prefixes(fake_brownie_1d):
assert ids[-1] == "objective_2_3"


class _FakeModel:
def predict(self, X, return_std=False):
# Make the histogram depend on the transformed point so the test would
# catch a regression that ignored it.
value = float(numpy.ravel(X)[0])
return numpy.array([value]), numpy.array([0.5])


class _FakeResult:
"""Minimal stand-in for an OptimizeResult with a transform + model."""

def __init__(self):
self.models = [_FakeModel()]
self.space = self

def transform(self, points):
# Identity transform; just records that it was called with the point.
return numpy.asarray(points, dtype=float)


def test_emit_json_single_plots_passes_selected_point_through(monkeypatch):
captured = {}

Expand All @@ -62,6 +82,12 @@ def _stub(result, x_eval=None):
)

plots: list = []
sp = [50, 833, "Whipped cream"]
emit_json_single_plots(plots, result=object(), prefix="single_0", selected_point=sp)
sp = [50, 833]
emit_json_single_plots(
plots, result=_FakeResult(), prefix="single_0", selected_point=sp
)
assert captured["x_eval"] == sp
# With a selected point the histogram is predicted at the transformed point,
# not taken from get_Brownie_Bee_1d_plot's (buggy) last entry.
histogram_payload = json.loads(plots[-1]["plot"])
assert histogram_payload["histogram"]["mean"] == 50.0
Loading