From 6916df7ee9b6bd55b22ae6859c299ac77a11292f Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 11 Jul 2026 14:28:56 +0200 Subject: [PATCH 1/2] perf: precompute constraint arrays for device_scheduler callbacks Pyomo calls each Param/Set initialize callback once per index tuple, so per-index pandas .iloc and boolean-mask lookups cost O(devices x timesteps) slow scalar accesses. Precompute plain numpy arrays (and a j -> quantity dict per commitment) once and index into those instead, preserving NaN semantics via float dtype. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Pxkeq64jtENY7fiWjwUsVS --- .../models/planning/linear_optimization.py | 98 ++++++++++++++----- 1 file changed, 75 insertions(+), 23 deletions(-) diff --git a/flexmeasures/data/models/planning/linear_optimization.py b/flexmeasures/data/models/planning/linear_optimization.py index 12b33fd70c..95720c2272 100644 --- a/flexmeasures/data/models/planning/linear_optimization.py +++ b/flexmeasures/data/models/planning/linear_optimization.py @@ -352,6 +352,58 @@ def commitment_time_device_groups_init(m): model.cjg = Set(dimen=3, initialize=commitment_time_device_groups_init) + # Precompute constraint columns as plain numpy arrays keyed by device/commitment + # index. Pyomo calls each ``initialize`` callback once per index tuple, so doing a + # pandas ``.iloc`` / scalar lookup inside a callback is O(devices x timesteps) + # individual (slow) pandas accesses. Indexing into precomputed numpy arrays gives + # identical values (NaN preserved via float dtype) at a fraction of the cost. + def _column_array(df, column): + """Return a float numpy array for ``column``, or None if the column is absent.""" + if column not in df.columns: + return None + return df[column].to_numpy(dtype=float) + + device_min_arr = [_column_array(d, "min") for d in device_constraints] + device_max_arr = [_column_array(d, "max") for d in device_constraints] + device_equals_arr = [_column_array(d, "equals") for d in device_constraints] + device_derivative_max_arr = [ + _column_array(d, "derivative max") for d in device_constraints + ] + device_derivative_min_arr = [ + _column_array(d, "derivative min") for d in device_constraints + ] + device_derivative_equals_arr = [ + _column_array(d, "derivative equals") for d in device_constraints + ] + device_stock_delta_arr = [ + _column_array(d, "stock delta") for d in device_constraints + ] + # Efficiency columns may be absent; a None entry signals "assume perfect efficiency". + device_efficiency_arr = [_column_array(d, "efficiency") for d in device_constraints] + device_derivative_down_efficiency_arr = [ + _column_array(d, "derivative down efficiency") for d in device_constraints + ] + device_derivative_up_efficiency_arr = [ + _column_array(d, "derivative up efficiency") for d in device_constraints + ] + ems_derivative_max_arr = [ + _column_array(e, "derivative max") for e in ems_constraints_list + ] + ems_derivative_min_arr = [ + _column_array(e, "derivative min") for e in ems_constraints_list + ] + # Commitment quantities are indexed by their (non-contiguous) "j" values, so map + # j -> quantity per commitment instead of doing a boolean mask lookup per (c, j). + commitment_quantity_arr = [ + dict( + zip( + commitments[c]["j"].to_numpy(), + commitments[c]["quantity"].to_numpy(dtype=float), + ) + ) + for c in range(len(commitments)) + ] + # Add parameters def price_down_select(m, c): if "downwards deviation price" not in commitments[c].columns: @@ -370,15 +422,15 @@ def price_up_select(m, c): return price def commitment_quantity_select(m, c, j): - quantity = commitments[c][commitments[c]["j"] == j]["quantity"].values[0] + quantity = commitment_quantity_arr[c][j] if np.isnan(quantity): return -infinity return quantity def device_max_select(m, d, j): - min_v = device_constraints[d]["min"].iloc[j] - max_v = device_constraints[d]["max"].iloc[j] - equal_v = device_constraints[d]["equals"].iloc[j] + min_v = device_min_arr[d][j] + max_v = device_max_arr[d][j] + equal_v = device_equals_arr[d][j] if np.isnan(max_v) and np.isnan(equal_v): return infinity else: @@ -389,9 +441,9 @@ def device_max_select(m, d, j): return np.nanmin([max_v, equal_v]) def device_min_select(m, d, j): - min_v = device_constraints[d]["min"].iloc[j] - max_v = device_constraints[d]["max"].iloc[j] - equal_v = device_constraints[d]["equals"].iloc[j] + min_v = device_min_arr[d][j] + max_v = device_max_arr[d][j] + equal_v = device_equals_arr[d][j] if np.isnan(min_v) and np.isnan(equal_v): return -infinity else: @@ -402,30 +454,30 @@ def device_min_select(m, d, j): return np.nanmax([min_v, equal_v]) def device_derivative_max_select(m, d, j): - max_v = device_constraints[d]["derivative max"].iloc[j] - equal_v = device_constraints[d]["derivative equals"].iloc[j] + max_v = device_derivative_max_arr[d][j] + equal_v = device_derivative_equals_arr[d][j] if np.isnan(max_v) and np.isnan(equal_v): return infinity else: return np.nanmin([max_v, equal_v]) def device_derivative_min_select(m, d, j): - min_v = device_constraints[d]["derivative min"].iloc[j] - equal_v = device_constraints[d]["derivative equals"].iloc[j] + min_v = device_derivative_min_arr[d][j] + equal_v = device_derivative_equals_arr[d][j] if np.isnan(min_v) and np.isnan(equal_v): return -infinity else: return np.nanmax([min_v, equal_v]) def ems_derivative_max_select(m, g, j): - v = ems_constraints_list[g]["derivative max"].iloc[j] + v = ems_derivative_max_arr[g][j] if np.isnan(v): return infinity else: return v def ems_derivative_min_select(m, g, j): - v = ems_constraints_list[g]["derivative min"].iloc[j] + v = ems_derivative_min_arr[g][j] if np.isnan(v): return -infinity else: @@ -433,36 +485,36 @@ def ems_derivative_min_select(m, g, j): def device_efficiency(m, d, j): """Assume perfect efficiency if no efficiency information is available.""" - try: - eff = device_constraints[d]["efficiency"].iloc[j] - except KeyError: + eff_arr = device_efficiency_arr[d] + if eff_arr is None: return 1 + eff = eff_arr[j] if np.isnan(eff): return 1 return eff def device_derivative_down_efficiency(m, d, j): """Assume perfect efficiency if no efficiency information is available.""" - try: - eff = device_constraints[d]["derivative down efficiency"].iloc[j] - except KeyError: + eff_arr = device_derivative_down_efficiency_arr[d] + if eff_arr is None: return 1 + eff = eff_arr[j] if np.isnan(eff): return 1 return eff def device_derivative_up_efficiency(m, d, j): """Assume perfect efficiency if no efficiency information is available.""" - try: - eff = device_constraints[d]["derivative up efficiency"].iloc[j] - except KeyError: + eff_arr = device_derivative_up_efficiency_arr[d] + if eff_arr is None: return 1 + eff = eff_arr[j] if np.isnan(eff): return 1 return eff def device_stock_delta(m, d, j): - return device_constraints[d]["stock delta"].iloc[j] + return device_stock_delta_arr[d][j] def grouped_commitment_equalities(m, c, j, g): """ From 40093425ac9b14a9cf06bc1d7f56b08a2d99b36e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Sat, 11 Jul 2026 15:58:11 +0200 Subject: [PATCH 2/2] docs: changelog entry for PR #2291 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Pxkeq64jtENY7fiWjwUsVS --- documentation/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 7f020f3647..89cccc7a3f 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -26,6 +26,7 @@ Infrastructure / Support ---------------------- * Upgraded dependencies [see `PR #1485 `_, `PR #2215 `_ and `PR #2243 `_] * Prepare the ``device_scheduler`` to deal with commitments per device group [see `PR #1934 `_] +* Speed up ``device_scheduler`` model construction by precomputing numpy constraint arrays for Pyomo parameter initialization [see `PR #2291 `_] * Support storing encrypted connection secrets on organisations and assets, including utility functions, encryption key configuration, CLI commands to set and delete secrets, and UI tables that show stored secret names and optional expiration times without exposing their values [see `PR #2236 `_] * Documentation section on the modelling choice for recording measurements, forecasts and schedules under one or multiple sensors [see `PR #2217 `_] * Document source filters better, and make use of the source-types filter in the PV curtailment tutorial [see `PR #2261 `_]