From 9983f05f2f025752035cd03796f104fc44dd7ec4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 10 Jul 2026 11:41:00 +0200 Subject: [PATCH 1/6] updating alembic.ini for actually logging errors during migrations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/data/migrations/alembic.ini | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/migrations/alembic.ini b/flexmeasures/data/migrations/alembic.ini index f8ed4801f7..ec9d45c26a 100644 --- a/flexmeasures/data/migrations/alembic.ini +++ b/flexmeasures/data/migrations/alembic.ini @@ -11,7 +11,7 @@ # Logging configuration [loggers] -keys = root,sqlalchemy,alembic +keys = root,sqlalchemy,alembic,flask_migrate [handlers] keys = console @@ -34,6 +34,11 @@ level = INFO handlers = qualname = alembic +[logger_flask_migrate] +level = INFO +handlers = +qualname = flask_migrate + [handler_console] class = StreamHandler args = (sys.stderr,) From 1446a16584fafd41d64b388659de9af1effca7a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 10 Jul 2026 14:26:54 +0200 Subject: [PATCH 2/6] add migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- ...timed_belief_horizon_to_integer_minutes.py | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 flexmeasures/data/migrations/versions/6d5c9e9d62cf_change_timed_belief_horizon_to_integer_minutes.py diff --git a/flexmeasures/data/migrations/versions/6d5c9e9d62cf_change_timed_belief_horizon_to_integer_minutes.py b/flexmeasures/data/migrations/versions/6d5c9e9d62cf_change_timed_belief_horizon_to_integer_minutes.py new file mode 100644 index 0000000000..e58bacf73d --- /dev/null +++ b/flexmeasures/data/migrations/versions/6d5c9e9d62cf_change_timed_belief_horizon_to_integer_minutes.py @@ -0,0 +1,81 @@ +"""change timed belief horizon to integer minutes + +Also delete duplicate beliefs that would have become duplicate primary keys after the change +(we keep the most recent beliefs). Those are beliefs with sub-minute differences in belieff horizons. +That last part is not reversible. + +Revision ID: 6d5c9e9d62cf +Revises: 55d8936a55f9 +Create Date: 2026-07-10 02:05:00.000000 + +""" + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +# revision identifiers, used by Alembic. +revision = "6d5c9e9d62cf" +down_revision = "55d8936a55f9" +branch_labels = None +depends_on = None + + +def upgrade(): + deleted_duplicate_beliefs = ( + op.get_bind() + .execute( + sa.text( + """ + WITH ranked_beliefs AS ( + SELECT + ctid, + ROW_NUMBER() OVER ( + PARTITION BY + source_id, + event_start, + FLOOR(EXTRACT(EPOCH FROM belief_horizon) / 60)::INTEGER, + cumulative_probability, + sensor_id + ORDER BY belief_horizon ASC + ) AS rank_in_future_primary_key + FROM timed_belief + ), + deleted_rows AS ( + DELETE FROM timed_belief + WHERE ctid IN ( + SELECT ctid + FROM ranked_beliefs + WHERE rank_in_future_primary_key > 1 + ) + RETURNING 1 + ) + SELECT COUNT(*) FROM deleted_rows + """ + ) + ) + .scalar_one() + ) + print( + "Deleted " + f"{deleted_duplicate_beliefs} timed_belief rows that would have become " + "duplicate primary keys after converting belief_horizon to integer minutes." + ) + op.alter_column( + "timed_belief", + "belief_horizon", + existing_type=postgresql.INTERVAL(), + type_=sa.Integer(), + postgresql_using="FLOOR(EXTRACT(EPOCH FROM belief_horizon) / 60)::INTEGER", + ) + + +def downgrade(): + op.alter_column( + "timed_belief", + "belief_horizon", + existing_type=sa.Integer(), + type_=postgresql.INTERVAL(), + postgresql_using="belief_horizon * interval '1 minute'", + ) From 5f0c31513c215a758ab334eea2c2d92e7f2da87e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 10 Jul 2026 14:27:34 +0200 Subject: [PATCH 3/6] small adaptations in logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/api/v3_0/tests/test_sensors_api.py | 1 + flexmeasures/data/services/sensors.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api.py b/flexmeasures/api/v3_0/tests/test_sensors_api.py index d8d033ca16..dac4356c00 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api.py @@ -744,6 +744,7 @@ def test_fetch_sensor_stats( for source, record in response_content.items(): assert record["First event start"] assert record["Last event end"] + assert record["Last recorded"] assert record["Min value"] assert record["Min value"] assert record["Max value"] diff --git a/flexmeasures/data/services/sensors.py b/flexmeasures/data/services/sensors.py index 82cb7f46e5..a74b47558d 100644 --- a/flexmeasures/data/services/sensors.py +++ b/flexmeasures/data/services/sensors.py @@ -830,7 +830,7 @@ def filtered_agg(func): sa.func.max( TimedBelief.event_start + sensor.event_resolution - - TimedBelief.belief_horizon + - TimedBelief.belief_horizon * sa.literal_column("interval '1 minute'") ).label("max_belief_time"), filtered_agg(sa.func.min).label("min_event_value"), filtered_agg(sa.func.max).label("max_event_value"), From aefcd62b8aba019ba4abb63ba7997b194dc81461 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Fri, 10 Jul 2026 14:28:28 +0200 Subject: [PATCH 4/6] add preliminary changelog entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- documentation/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 4fc2f13886..0c4ff6d478 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -24,6 +24,7 @@ New features Infrastructure / Support ---------------------- +* Store timed belief horizons as integer minutes in the database to reduce belief storage size; upgrading requires running the new migration with timely-beliefs support [see `PR #XXXX `_] * Upgraded dependencies [see `PR #1485 `_, `PR #2215 `_ and `PR #2243 `_] * Prepare the ``device_scheduler`` to deal with commitments per device group [see `PR #1934 `_] * Support storing encrypted connection secrets on organisations and assets, including utility functions, encryption key configuration, CLI commands to set and delete secrets, and UI tables that show stored secret names and optional expiration times without exposing their values [see `PR #2236 `_] From b3e8dce1d4d3188acfbbeb1c251f00c8008631f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Mon, 13 Jul 2026 00:46:55 +0200 Subject: [PATCH 5/6] move from horizons measures by minutes to seconds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- documentation/changelog.rst | 2 +- ...imed_belief_horizon_to_integer_seconds.py} | 37 ++++++++++++++++--- flexmeasures/data/services/sensors.py | 2 +- 3 files changed, 33 insertions(+), 8 deletions(-) rename flexmeasures/data/migrations/versions/{6d5c9e9d62cf_change_timed_belief_horizon_to_integer_minutes.py => 6d5c9e9d62cf_change_timed_belief_horizon_to_integer_seconds.py} (66%) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 0c4ff6d478..8eb9fbe187 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -24,7 +24,7 @@ New features Infrastructure / Support ---------------------- -* Store timed belief horizons as integer minutes in the database to reduce belief storage size; upgrading requires running the new migration with timely-beliefs support [see `PR #XXXX `_] +* Store timed belief horizons as integer seconds in the database to reduce belief storage size; upgrading requires running the new migration with timely-beliefs support [see `PR #XXXX `_] * Upgraded dependencies [see `PR #1485 `_, `PR #2215 `_ and `PR #2243 `_] * Prepare the ``device_scheduler`` to deal with commitments per device group [see `PR #1934 `_] * Support storing encrypted connection secrets on organisations and assets, including utility functions, encryption key configuration, CLI commands to set and delete secrets, and UI tables that show stored secret names and optional expiration times without exposing their values [see `PR #2236 `_] diff --git a/flexmeasures/data/migrations/versions/6d5c9e9d62cf_change_timed_belief_horizon_to_integer_minutes.py b/flexmeasures/data/migrations/versions/6d5c9e9d62cf_change_timed_belief_horizon_to_integer_seconds.py similarity index 66% rename from flexmeasures/data/migrations/versions/6d5c9e9d62cf_change_timed_belief_horizon_to_integer_minutes.py rename to flexmeasures/data/migrations/versions/6d5c9e9d62cf_change_timed_belief_horizon_to_integer_seconds.py index e58bacf73d..a447fe4ffd 100644 --- a/flexmeasures/data/migrations/versions/6d5c9e9d62cf_change_timed_belief_horizon_to_integer_minutes.py +++ b/flexmeasures/data/migrations/versions/6d5c9e9d62cf_change_timed_belief_horizon_to_integer_seconds.py @@ -1,7 +1,7 @@ -"""change timed belief horizon to integer minutes +"""change timed belief horizon to integer seconds Also delete duplicate beliefs that would have become duplicate primary keys after the change -(we keep the most recent beliefs). Those are beliefs with sub-minute differences in belieff horizons. +(we keep the most recent beliefs). Those are beliefs with sub-second differences in belief horizons. That last part is not reversible. Revision ID: 6d5c9e9d62cf @@ -23,6 +23,31 @@ def upgrade(): + out_of_range_beliefs = ( + op.get_bind() + .execute( + sa.text( + """ + SELECT COUNT(*) + FROM timed_belief + WHERE + EXTRACT(EPOCH FROM belief_horizon) < -2147483648 + OR EXTRACT(EPOCH FROM belief_horizon) > 2147483647 + """ + ) + ) + .scalar_one() + ) + if out_of_range_beliefs: + print( + f"Found {out_of_range_beliefs} timed_belief rows with belief_horizon " + "outside the supported integer-second range of about 68 years." + ) + raise RuntimeError( + "Cannot convert timed_belief.belief_horizon to integer seconds: " + f"{out_of_range_beliefs} rows exceed the signed INTEGER range." + ) + deleted_duplicate_beliefs = ( op.get_bind() .execute( @@ -35,7 +60,7 @@ def upgrade(): PARTITION BY source_id, event_start, - FLOOR(EXTRACT(EPOCH FROM belief_horizon) / 60)::INTEGER, + FLOOR(EXTRACT(EPOCH FROM belief_horizon))::INTEGER, cumulative_probability, sensor_id ORDER BY belief_horizon ASC @@ -60,14 +85,14 @@ def upgrade(): print( "Deleted " f"{deleted_duplicate_beliefs} timed_belief rows that would have become " - "duplicate primary keys after converting belief_horizon to integer minutes." + "duplicate primary keys after converting belief_horizon to integer seconds." ) op.alter_column( "timed_belief", "belief_horizon", existing_type=postgresql.INTERVAL(), type_=sa.Integer(), - postgresql_using="FLOOR(EXTRACT(EPOCH FROM belief_horizon) / 60)::INTEGER", + postgresql_using="FLOOR(EXTRACT(EPOCH FROM belief_horizon))::INTEGER", ) @@ -77,5 +102,5 @@ def downgrade(): "belief_horizon", existing_type=sa.Integer(), type_=postgresql.INTERVAL(), - postgresql_using="belief_horizon * interval '1 minute'", + postgresql_using="belief_horizon * interval '1 second'", ) diff --git a/flexmeasures/data/services/sensors.py b/flexmeasures/data/services/sensors.py index a74b47558d..445cae82a3 100644 --- a/flexmeasures/data/services/sensors.py +++ b/flexmeasures/data/services/sensors.py @@ -830,7 +830,7 @@ def filtered_agg(func): sa.func.max( TimedBelief.event_start + sensor.event_resolution - - TimedBelief.belief_horizon * sa.literal_column("interval '1 minute'") + - TimedBelief.belief_horizon * sa.literal_column("interval '1 second'") ).label("max_belief_time"), filtered_agg(sa.func.min).label("min_event_value"), filtered_agg(sa.func.max).label("max_event_value"), From 543a1a47ae9a8204d90f7f32efed7fb910fdc0f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20H=C3=B6ning?= Date: Mon, 13 Jul 2026 00:48:00 +0200 Subject: [PATCH 6/6] remove unnecessary job execution which led to double execution in seconds-level horizons, as queue was not cleared MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Nicolas Höning --- flexmeasures/data/tests/test_scheduling_sequential.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flexmeasures/data/tests/test_scheduling_sequential.py b/flexmeasures/data/tests/test_scheduling_sequential.py index 68d5689f2d..207018ba7c 100644 --- a/flexmeasures/data/tests/test_scheduling_sequential.py +++ b/flexmeasures/data/tests/test_scheduling_sequential.py @@ -88,7 +88,6 @@ def test_create_sequential_jobs(db, app, flex_description_sequential, smart_buil assert battery_power.empty # Work on jobs - queued_jobs[0].perform() work_on_rq(queue, handle_scheduling_exception) # Check that the jobs completed successfully