perf: precompute constraint arrays for device_scheduler callbacks

documentation/changelog.rst

@@ -26,6 +26,7 @@ Infrastructure / Support
 ----------------------
 * Upgraded dependencies [see `PR #1485 <https://www.github.com/FlexMeasures/flexmeasures/pull/1485>`_, `PR #2215 <https://www.github.com/FlexMeasures/flexmeasures/pull/2215>`_ and `PR #2243 <https://www.github.com/FlexMeasures/flexmeasures/pull/2243>`_]
 * Prepare the ``device_scheduler`` to deal with commitments per device group [see `PR #1934 <https://www.github.com/FlexMeasures/flexmeasures/pull/1934>`_]
+* Speed up ``device_scheduler`` model construction by precomputing numpy constraint arrays for Pyomo parameter initialization [see `PR #2291 <https://www.github.com/FlexMeasures/flexmeasures/pull/2291>`_]
 * Support storing encrypted connection secrets on organisations and assets, including utility functions, encryption key configuration, CLI commands to set and delete secrets, and UI tables that show stored secret names and optional expiration times without exposing their values [see `PR #2236 <https://www.github.com/FlexMeasures/flexmeasures/pull/2236>`_]
 * Documentation section on the modelling choice for recording measurements, forecasts and schedules under one or multiple sensors [see `PR #2217 <https://www.github.com/FlexMeasures/flexmeasures/pull/2217>`_]
 * Document source filters better, and make use of the source-types filter in the PV curtailment tutorial [see `PR #2261 <https://www.github.com/FlexMeasures/flexmeasures/pull/2261>`_]

flexmeasures/data/models/planning/linear_optimization.py

@@ -352,6 +352,58 @@ def commitment_time_device_groups_init(m):
 
     model.cjg = Set(dimen=3, initialize=commitment_time_device_groups_init)
 
+    # Precompute constraint columns as plain numpy arrays keyed by device/commitment
+    # index. Pyomo calls each ``initialize`` callback once per index tuple, so doing a
+    # pandas ``.iloc`` / scalar lookup inside a callback is O(devices x timesteps)
+    # individual (slow) pandas accesses. Indexing into precomputed numpy arrays gives
+    # identical values (NaN preserved via float dtype) at a fraction of the cost.
+    def _column_array(df, column):
+        """Return a float numpy array for ``column``, or None if the column is absent."""
+        if column not in df.columns:
+            return None
+        return df[column].to_numpy(dtype=float)
+
+    device_min_arr = [_column_array(d, "min") for d in device_constraints]
+    device_max_arr = [_column_array(d, "max") for d in device_constraints]
+    device_equals_arr = [_column_array(d, "equals") for d in device_constraints]
+    device_derivative_max_arr = [
+        _column_array(d, "derivative max") for d in device_constraints
+    ]
+    device_derivative_min_arr = [
+        _column_array(d, "derivative min") for d in device_constraints
+    ]
+    device_derivative_equals_arr = [
+        _column_array(d, "derivative equals") for d in device_constraints
+    ]
+    device_stock_delta_arr = [
+        _column_array(d, "stock delta") for d in device_constraints
+    ]
+    # Efficiency columns may be absent; a None entry signals "assume perfect efficiency".
+    device_efficiency_arr = [_column_array(d, "efficiency") for d in device_constraints]
+    device_derivative_down_efficiency_arr = [
+        _column_array(d, "derivative down efficiency") for d in device_constraints
+    ]
+    device_derivative_up_efficiency_arr = [
+        _column_array(d, "derivative up efficiency") for d in device_constraints
+    ]
+    ems_derivative_max_arr = [
+        _column_array(e, "derivative max") for e in ems_constraints_list
+    ]
+    ems_derivative_min_arr = [
+        _column_array(e, "derivative min") for e in ems_constraints_list
+    ]
+    # Commitment quantities are indexed by their (non-contiguous) "j" values, so map
+    # j -> quantity per commitment instead of doing a boolean mask lookup per (c, j).
+    commitment_quantity_arr = [
+        dict(
+            zip(
+                commitments[c]["j"].to_numpy(),
+                commitments[c]["quantity"].to_numpy(dtype=float),
+            )
+        )
+        for c in range(len(commitments))
+    ]
+
     # Add parameters
     def price_down_select(m, c):
         if "downwards deviation price" not in commitments[c].columns:
@@ -370,15 +422,15 @@ def price_up_select(m, c):
         return price
 
     def commitment_quantity_select(m, c, j):
-        quantity = commitments[c][commitments[c]["j"] == j]["quantity"].values[0]
+        quantity = commitment_quantity_arr[c][j]
         if np.isnan(quantity):
             return -infinity
         return quantity
 
     def device_max_select(m, d, j):
-        min_v = device_constraints[d]["min"].iloc[j]
-        max_v = device_constraints[d]["max"].iloc[j]
-        equal_v = device_constraints[d]["equals"].iloc[j]
+        min_v = device_min_arr[d][j]
+        max_v = device_max_arr[d][j]
+        equal_v = device_equals_arr[d][j]
         if np.isnan(max_v) and np.isnan(equal_v):
             return infinity
         else:
@@ -389,9 +441,9 @@ def device_max_select(m, d, j):
             return np.nanmin([max_v, equal_v])
 
     def device_min_select(m, d, j):
-        min_v = device_constraints[d]["min"].iloc[j]
-        max_v = device_constraints[d]["max"].iloc[j]
-        equal_v = device_constraints[d]["equals"].iloc[j]
+        min_v = device_min_arr[d][j]
+        max_v = device_max_arr[d][j]
+        equal_v = device_equals_arr[d][j]
         if np.isnan(min_v) and np.isnan(equal_v):
             return -infinity
         else:
@@ -402,67 +454,67 @@ def device_min_select(m, d, j):
             return np.nanmax([min_v, equal_v])
 
     def device_derivative_max_select(m, d, j):
-        max_v = device_constraints[d]["derivative max"].iloc[j]
-        equal_v = device_constraints[d]["derivative equals"].iloc[j]
+        max_v = device_derivative_max_arr[d][j]
+        equal_v = device_derivative_equals_arr[d][j]
         if np.isnan(max_v) and np.isnan(equal_v):
             return infinity
         else:
             return np.nanmin([max_v, equal_v])
 
     def device_derivative_min_select(m, d, j):
-        min_v = device_constraints[d]["derivative min"].iloc[j]
-        equal_v = device_constraints[d]["derivative equals"].iloc[j]
+        min_v = device_derivative_min_arr[d][j]
+        equal_v = device_derivative_equals_arr[d][j]
         if np.isnan(min_v) and np.isnan(equal_v):
             return -infinity
         else:
             return np.nanmax([min_v, equal_v])
 
     def ems_derivative_max_select(m, g, j):
-        v = ems_constraints_list[g]["derivative max"].iloc[j]
+        v = ems_derivative_max_arr[g][j]
         if np.isnan(v):
             return infinity
         else:
             return v
 
     def ems_derivative_min_select(m, g, j):
-        v = ems_constraints_list[g]["derivative min"].iloc[j]
+        v = ems_derivative_min_arr[g][j]
         if np.isnan(v):
             return -infinity
         else:
             return v
 
     def device_efficiency(m, d, j):
         """Assume perfect efficiency if no efficiency information is available."""
-        try:
-            eff = device_constraints[d]["efficiency"].iloc[j]
-        except KeyError:
+        eff_arr = device_efficiency_arr[d]
+        if eff_arr is None:
             return 1
+        eff = eff_arr[j]
         if np.isnan(eff):
             return 1
         return eff
 
     def device_derivative_down_efficiency(m, d, j):
         """Assume perfect efficiency if no efficiency information is available."""
-        try:
-            eff = device_constraints[d]["derivative down efficiency"].iloc[j]
-        except KeyError:
+        eff_arr = device_derivative_down_efficiency_arr[d]
+        if eff_arr is None:
             return 1
+        eff = eff_arr[j]
         if np.isnan(eff):
             return 1
         return eff
 
     def device_derivative_up_efficiency(m, d, j):
         """Assume perfect efficiency if no efficiency information is available."""
-        try:
-            eff = device_constraints[d]["derivative up efficiency"].iloc[j]
-        except KeyError:
+        eff_arr = device_derivative_up_efficiency_arr[d]
+        if eff_arr is None:
             return 1
+        eff = eff_arr[j]
         if np.isnan(eff):
             return 1
         return eff
 
     def device_stock_delta(m, d, j):
-        return device_constraints[d]["stock delta"].iloc[j]
+        return device_stock_delta_arr[d][j]
 
     def grouped_commitment_equalities(m, c, j, g):
         """