diff --git a/pyproject.toml b/pyproject.toml index 910fddd7..73922700 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,13 +27,13 @@ requires-python = ">=3.11" dependencies = [ "psycopg>=3.1.10,<4.0", "sqlalchemy>=2.0.8,<3.0", - "pandas>=2.2,<3.0", + "pandas>=3.0.0,<4.0", "pint>=0.23.0", "argon2_cffi>=23.1.0", "alembic>=1.8.0,<2.0", "click>=8.1.3,<9.0", "celery>=5.3.1,<6.0", - "redis>=4.3.4,<5.0", + "redis>=7.4.0,<8.0", "requests>=2.28.2", ] diff --git a/requirements/install-min.in b/requirements/install-min.in index b0c4f936..ab550bef 100644 --- a/requirements/install-min.in +++ b/requirements/install-min.in @@ -1,10 +1,10 @@ psycopg==3.1.10 sqlalchemy==2.0.8 -pandas==2.2 +pandas==3.0.0 argon2_cffi==23.1.0 alembic==1.8.0 click==8.1.3 celery==5.3.1 -redis==4.3.4 +redis==7.4.0 pint==0.23.0 requests==2.28.2 diff --git a/requirements/install-min.txt b/requirements/install-min.txt index 0e274b9a..06da0cca 100644 --- a/requirements/install-min.txt +++ b/requirements/install-min.txt @@ -37,8 +37,6 @@ click-plugins==1.1.1.2 # via celery click-repl==0.3.0 # via celery -deprecated==1.3.1 - # via redis greenlet==3.4.0 # via sqlalchemy idna==3.11 @@ -52,10 +50,8 @@ markupsafe==3.0.3 numpy==1.26.4 # via pandas packaging==26.1 - # via - # kombu - # redis -pandas==2.2.0 + # via kombu +pandas==3.0.0 # via -r requirements/install-min.in pint==0.23 # via -r requirements/install-min.in @@ -69,9 +65,7 @@ python-dateutil==2.9.0.post0 # via # celery # pandas -pytz==2026.1.post1 - # via pandas -redis==4.3.4 +redis==7.4.0 # via -r requirements/install-min.in requests==2.28.2 # via -r requirements/install-min.in @@ -90,7 +84,6 @@ tzdata==2026.1 # via # celery # kombu - # pandas urllib3==1.26.20 # via requests vine==5.1.0 @@ -100,5 +93,3 @@ vine==5.1.0 # kombu wcwidth==0.6.0 # via prompt-toolkit -wrapt==2.1.2 - # via deprecated diff --git a/requirements/install.txt b/requirements/install.txt index fc3c7356..2426de65 100644 --- a/requirements/install.txt +++ b/requirements/install.txt @@ -55,7 +55,7 @@ numpy==2.4.4 # via pandas packaging==26.1 # via kombu -pandas==2.3.3 +pandas==3.0.2 # via bemserver-core (pyproject.toml) pint==0.25.3 # via bemserver-core (pyproject.toml) @@ -71,9 +71,7 @@ python-dateutil==2.9.0.post0 # via # celery # pandas -pytz==2026.1.post1 - # via pandas -redis==4.6.0 +redis==7.4.0 # via bemserver-core (pyproject.toml) requests==2.33.1 # via bemserver-core (pyproject.toml) @@ -92,9 +90,7 @@ typing-extensions==4.15.0 # psycopg # sqlalchemy tzdata==2026.1 - # via - # kombu - # pandas + # via kombu tzlocal==5.3.1 # via celery urllib3==2.6.3 diff --git a/src/bemserver_core/__init__.py b/src/bemserver_core/__init__.py index 9c03dc42..4e966e4c 100644 --- a/src/bemserver_core/__init__.py +++ b/src/bemserver_core/__init__.py @@ -2,8 +2,6 @@ import os -import pandas as pd - from bemserver_core import ( common, database, @@ -18,9 +16,6 @@ from bemserver_core.exceptions import BEMServerCoreSettingsError from bemserver_core.process.weather import wdp -# Set pandas future flags to silence deprecation warnings -pd.set_option("future.no_silent_downcasting", True) - class BEMServerCore: def __init__(self): diff --git a/src/bemserver_core/input_output/timeseries_data_io.py b/src/bemserver_core/input_output/timeseries_data_io.py index 63cca72a..79207c01 100644 --- a/src/bemserver_core/input_output/timeseries_data_io.py +++ b/src/bemserver_core/input_output/timeseries_data_io.py @@ -388,8 +388,10 @@ def get_timeseries_data( data, columns=("timestamp", "id", "name", "value") ).set_index("timestamp") data_df["value"] = data_df["value"].astype(float) - data_df.index = pd.DatetimeIndex(data_df.index, tz="UTC").tz_convert( - ZoneInfo(timezone) + data_df.index = ( + pd.DatetimeIndex(data_df.index, tz="UTC") + .as_unit("us") + .tz_convert(ZoneInfo(timezone)) ) data_df = data_df.pivot(columns=col_label, values="value") @@ -482,7 +484,9 @@ def get_timeseries_buckets_data( ) if not timeseries: - return pd.DataFrame({}, index=complete_idx) + ret_df = pd.DataFrame({}, index=complete_idx) + ret_df.columns.name = col_label + return ret_df # At this stage, date_trunc can only aggregate by 1 x unit. # For a N x width bucket size, the remaining aggregation is @@ -513,7 +517,9 @@ def get_timeseries_buckets_data( data, columns=("timestamp", "id", "name", "value") ).set_index("timestamp") - data_df.index = pd.DatetimeIndex(data_df.index, tz="UTC").tz_convert(tz_info) + data_df.index = ( + pd.DatetimeIndex(data_df.index, tz="UTC").as_unit("us").tz_convert(tz_info) + ) # Pivot table to get timeseries in columns data_df = data_df.pivot(values="value", columns=col_label).fillna(fill_value) @@ -652,6 +658,7 @@ def delete(cls, start_dt, end_dt, timeseries, data_state): def to_utc_index(index): """Create UTC datetime index from timezone aware datetime list""" + # https://github.com/pandas-dev/pandas/issues/54995 try: # Cast to series so that output is series, with an apply method @@ -669,7 +676,7 @@ def to_utc_index(index): except TypeError as exc: raise TimeseriesDataIODatetimeError("Invalid or TZ-naive timestamp") from exc - return pd.DatetimeIndex(index, name="timestamp") + return pd.DatetimeIndex(index, name="timestamp").as_unit("us") class TimeseriesDataCSVIO(TimeseriesDataIO, BaseCSVIO): diff --git a/src/bemserver_core/process/cleanup.py b/src/bemserver_core/process/cleanup.py index cde25162..ea78c51a 100644 --- a/src/bemserver_core/process/cleanup.py +++ b/src/bemserver_core/process/cleanup.py @@ -37,10 +37,10 @@ def cleanup( ts_mins = Timeseries.get_property_for_many_timeseries(timeseries_ids, "Min") ts_maxs = Timeseries.get_property_for_many_timeseries(timeseries_ids, "Max") - for ts_id, (_, col) in zip(timeseries_ids, data_df.items(), strict=True): + for ts_id in timeseries_ids: if (ts_min := ts_mins[ts_id]) is not None: - col.loc[col < float(ts_min)] = np.nan + data_df.loc[data_df[ts_id] < float(ts_min), ts_id] = np.nan if (ts_max := ts_maxs[ts_id]) is not None: - col.loc[col > float(ts_max)] = np.nan + data_df.loc[data_df[ts_id] > float(ts_max), ts_id] = np.nan return data_df diff --git a/src/bemserver_core/process/energy_power.py b/src/bemserver_core/process/energy_power.py index bc2b5b32..4dc74446 100644 --- a/src/bemserver_core/process/energy_power.py +++ b/src/bemserver_core/process/energy_power.py @@ -137,7 +137,7 @@ def energyindex2power( power_s = power_s.resample(pd_freq, closed="left", label="left").agg("mean") # Convert to desired unit - convert_from = ureg.validate_unit(index_ts.unit_symbol) / ureg.validate_unit("ns") + convert_from = ureg.validate_unit(index_ts.unit_symbol) / ureg.validate_unit("us") power_s = pd.Series( ureg.convert(power_s.values, convert_from, convert_to), index=power_s.index, diff --git a/src/bemserver_core/process/weather.py b/src/bemserver_core/process/weather.py index dd3e46d6..2bee94f8 100644 --- a/src/bemserver_core/process/weather.py +++ b/src/bemserver_core/process/weather.py @@ -111,7 +111,7 @@ def get_weather_data( index=pd.DatetimeIndex( pd.to_datetime(timestamps, utc=True, unit="s"), name="timestamp", - ), + ).as_unit("us"), data=ret_data["data"], # Strip (unit) from column names to facilitate selection columns=[str(col).split()[0] for col in ret_data["columns"]], diff --git a/tests/common/test_units.py b/tests/common/test_units.py index 0d39c332..d8132eac 100644 --- a/tests/common/test_units.py +++ b/tests/common/test_units.py @@ -5,6 +5,7 @@ import pytest import pandas as pd +from pandas.testing import assert_frame_equal from bemserver_core import BEMServerCore from bemserver_core.common import ureg @@ -66,7 +67,7 @@ def test_ureg_convert_dimensionality_error(self): def test_ureg_convert_df(self): data_df = pd.DataFrame({"id": [0, 1, 2]}) ureg.convert_df(data_df, {"id": "km"}, {"id": "m"}) - assert data_df.equals(1000.0 * pd.DataFrame({"id": [0, 1, 2]})) + assert_frame_equal(data_df, (1000.0 * pd.DataFrame({"id": [0, 1, 2]}))) def test_ureg_convert_df_undefined_unit(self): data_df = pd.DataFrame({"id": [0, 1, 2]}) diff --git a/tests/input_output/test_timeseries_data_io.py b/tests/input_output/test_timeseries_data_io.py index a0637b19..c8f22066 100644 --- a/tests/input_output/test_timeseries_data_io.py +++ b/tests/input_output/test_timeseries_data_io.py @@ -62,7 +62,7 @@ def test_timeseries_data_io_set_timeseries_data_as_admin( "2020-01-01T03:00:00+00:00", ], name="timestamp", - ) + ).as_unit("us") val_0 = [0, 1, 2, 3] val_2 = [10, 11, 12, np.nan] data_df = pd.DataFrame( @@ -205,7 +205,7 @@ def test_timeseries_data_io_set_timeseries_data_as_user( "2020-01-01T03:00:00+00:00", ], name="timestamp", - ) + ).as_unit("us") val_0 = [0, 1, 2, 3] val_2 = [10, 11, 12, 13] @@ -260,7 +260,7 @@ def test_timeseries_data_io_import_set_timeseries_data_timeseries_error( "2020-01-01T03:00:00+00:00", ], name="timestamp", - ) + ).as_unit("us") val_0 = [0, 1, 2, 3] data_df = pd.DataFrame( @@ -292,7 +292,7 @@ def test_timeseries_data_io_import_set_timeseries_data_columns_type_error( "2020-01-01T03:00:00+00:00", ], name="timestamp", - ) + ).as_unit("us") val_0 = [0, 1, 2, 3] data_df = pd.DataFrame( @@ -326,14 +326,14 @@ def test_timeseries_data_io_import_set_timeseries_data_empty_dataframe( ds_1 = TimeseriesDataState.get(name="Raw").first() # Empty dataframe - index = pd.DatetimeIndex([]) + index = pd.DatetimeIndex([]).as_unit("us") data_df = pd.DataFrame({}, index=index) # Nothing happens. No crash. with CurrentUser(admin_user): tsdio.set_timeseries_data(data_df, ds_1, campaign) - index = pd.DatetimeIndex(["2020-01-01T00:00:00+00:00"]) + index = pd.DatetimeIndex(["2020-01-01T00:00:00+00:00"]).as_unit("us") val_0 = [np.nan] # Not exactly empty but NaN-only dataframe @@ -684,12 +684,13 @@ def test_timeseries_data_io_get_timeseries_data_as_admin( data_df = tsdio.get_timeseries_data( start_dt, end_dt, ts_l, ds_1, col_label="name" ) - index = pd.DatetimeIndex([], name="timestamp", tz="UTC") + index = pd.DatetimeIndex([], name="timestamp", tz="UTC").as_unit("us") no_data_df = pd.DataFrame( {ts_0.name: [], ts_2.name: [], ts_4.name: []}, index=index, ) - assert data_df.equals(no_data_df) + no_data_df.columns.name = "name" + assert_frame_equal(data_df, no_data_df) # Create data timestamps = pd.date_range( @@ -710,7 +711,7 @@ def test_timeseries_data_io_get_timeseries_data_as_admin( ], name="timestamp", tz="UTC", - ) + ).as_unit("us") val_0 = [0.0, 1.0, 2.0] val_2 = [np.nan, np.nan, np.nan] val_4 = [10.0, 12.0, np.nan] @@ -718,25 +719,26 @@ def test_timeseries_data_io_get_timeseries_data_as_admin( {ts_0.id: val_0, ts_2.id: val_2, ts_4.id: val_4}, index=index, ) - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "id" + assert_frame_equal(data_df, expected_data_df) # Get with no start_date data_df = tsdio.get_timeseries_data(None, end_dt, ts_l, ds_1) - assert data_df.equals(expected_data_df) + assert_frame_equal(data_df, expected_data_df) # Get with no end date data_df = tsdio.get_timeseries_data(start_dt, None, ts_l, ds_1) - assert data_df.equals(expected_data_df) + assert_frame_equal(data_df, expected_data_df) # Get with no start/end date data_df = tsdio.get_timeseries_data(None, None, ts_l, ds_1) - assert data_df.equals(expected_data_df) + assert_frame_equal(data_df, expected_data_df) # Get outside data range: no data data_df = tsdio.get_timeseries_data( end_dt, None, ts_l, ds_1, col_label="name" ) - assert data_df.equals(no_data_df) + assert_frame_equal(data_df, no_data_df) # Get with TZ data_df = tsdio.get_timeseries_data( @@ -754,8 +756,8 @@ def test_timeseries_data_io_get_timeseries_data_as_admin( ], name="timestamp", tz=ZoneInfo("Europe/Paris"), - ) - assert data_df.equals(expected_data_df) + ).as_unit("us") + assert_frame_equal(data_df, expected_data_df) # Test conversions with OpenBar(): @@ -776,7 +778,7 @@ def test_timeseries_data_io_get_timeseries_data_as_admin( ], name="timestamp", tz="UTC", - ) + ).as_unit("us") val_0 = [1000 * 0.0, 1000 * 1.0, 1000 * 2.0] val_2 = [np.nan, np.nan, np.nan] val_4 = [10.0, 12.0, np.nan] @@ -784,7 +786,8 @@ def test_timeseries_data_io_get_timeseries_data_as_admin( {ts_0.id: val_0, ts_2.id: val_2, ts_4.id: val_4}, index=index, ) - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "id" + assert_frame_equal(data_df, expected_data_df) # Undefined: dummy with pytest.raises(BEMServerCoreUndefinedUnitError): data_df = tsdio.get_timeseries_data( @@ -815,7 +818,7 @@ def test_timeseries_data_io_get_timeseries_data_as_admin( ], name="timestamp", tz="UTC", - ) + ).as_unit("us") val_0 = [1.0, 2.0] val_2 = [np.nan, np.nan] val_4 = [12.0, np.nan] @@ -823,25 +826,26 @@ def test_timeseries_data_io_get_timeseries_data_as_admin( {ts_2.id: val_2, ts_0.id: val_0, ts_4.id: val_4}, index=index, ) + expected_data_df.columns.name = "id" data_df = tsdio.get_timeseries_data( h1_dt, h2_dt, ts_l, ds_1, inclusive="both" ) - assert data_df.equals(expected_data_df) + assert_frame_equal(data_df, expected_data_df) data_df = tsdio.get_timeseries_data( h1_dt, h2_dt, ts_l, ds_1, inclusive="neither" ) mask = (expected_data_df.index > h1_dt) & (expected_data_df.index < h2_dt) - assert data_df.equals(expected_data_df.loc[mask]) + assert_frame_equal(data_df, expected_data_df.loc[mask]) data_df = tsdio.get_timeseries_data( h1_dt, h2_dt, ts_l, ds_1, inclusive="left" ) mask = (expected_data_df.index >= h1_dt) & (expected_data_df.index < h2_dt) - assert data_df.equals(expected_data_df.loc[mask]) + assert_frame_equal(data_df, expected_data_df.loc[mask]) data_df = tsdio.get_timeseries_data( h1_dt, h2_dt, ts_l, ds_1, inclusive="right" ) mask = (expected_data_df.index > h1_dt) & (expected_data_df.index <= h2_dt) - assert data_df.equals(expected_data_df.loc[mask]) + assert_frame_equal(data_df, expected_data_df.loc[mask]) @pytest.mark.parametrize("campaigns", (2,), indirect=True) @pytest.mark.parametrize("timeseries", (5,), indirect=True) @@ -895,7 +899,7 @@ def test_timeseries_data_io_get_timeseries_data_as_user( ], name="timestamp", tz="UTC", - ) + ).as_unit("us") val_1 = [0.0, 1.0, 2.0] val_3 = [10.0, 12.0, np.nan] expected_data_df = pd.DataFrame( @@ -905,7 +909,8 @@ def test_timeseries_data_io_get_timeseries_data_as_user( }, index=index, ) - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = col_label + assert_frame_equal(data_df, expected_data_df) @pytest.mark.parametrize("timeseries", (5,), indirect=True) @pytest.mark.parametrize("agg", ("avg", "min", "max", "count")) @@ -945,7 +950,7 @@ def test_timeseries_data_io_get_aggreagate_data( {agg: (0, 0, 0) if agg == "count" else (np.nan, np.nan, np.nan)}, index=pd.Index((getattr(ts, col_label) for ts in ts_l), name=col_label), ) - assert data_df.equals(expected_data_df) + assert_frame_equal(data_df, expected_data_df) values_1 = range(24 * 3) create_timeseries_data(ts_0, ds_1, timestamps, values_1) @@ -973,7 +978,7 @@ def test_timeseries_data_io_get_aggreagate_data( {agg: expected_values}, index=pd.Index((getattr(ts, col_label) for ts in ts_l), name=col_label), ) - assert data_df.equals(expected_data_df) + assert_frame_equal(data_df, expected_data_df) @pytest.mark.parametrize("timeseries", (5,), indirect=True) def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( @@ -1016,7 +1021,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( ], name="timestamp", tz="UTC", - ) + freq="D", + ).as_unit("us") expected_data_df = pd.DataFrame( { ts_0.id: [np.nan, np.nan, np.nan], @@ -1025,7 +1031,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( }, index=index, ) - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "id" + assert_frame_equal(data_df, expected_data_df) values_1 = range(24 * 3) create_timeseries_data(ts_0, ds_1, timestamps, values_1) @@ -1053,7 +1060,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( ], name="timestamp", tz="UTC", - ) + freq="D", + ).as_unit("us") expected_data_df = pd.DataFrame( { ts_0.name: [24, 24, 24], @@ -1062,7 +1070,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( }, index=index, ) - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "name" + assert_frame_equal(data_df, expected_data_df) # UTC count 1 day with gapfill data_df = tsdio.get_timeseries_buckets_data( @@ -1087,7 +1096,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( ], name="timestamp", tz="UTC", - ) + freq="D", + ).as_unit("us") expected_data_df = pd.DataFrame( { ts_0.name: [24, 24, 24, 0, 0, 0], @@ -1096,7 +1106,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( }, index=index, ) - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "name" + assert_frame_equal(data_df, expected_data_df) # UTC count 1 day, 3 hour (and a half) offset # start time is floored to round to interval @@ -1119,7 +1130,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( ], name="timestamp", tz="UTC", - ) + freq="D", + ).as_unit("us") expected_data_df = pd.DataFrame( { ts_0.name: [24, 24, 24], @@ -1128,8 +1140,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( }, index=index, ) - - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "name" + assert_frame_equal(data_df, expected_data_df) # Local TZ count 1 week data_df = tsdio.get_timeseries_buckets_data( @@ -1152,7 +1164,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( ], name="timestamp", tz=ZoneInfo("Europe/Paris"), - ) + freq="W-MON", + ).as_unit("us") expected_data_df = pd.DataFrame( { ts_0.name: [72, 0], @@ -1161,8 +1174,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( }, index=index, ) - - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "name" + assert_frame_equal(data_df, expected_data_df) # UTC count 12 hours data_df = tsdio.get_timeseries_buckets_data( @@ -1187,7 +1200,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( ], name="timestamp", tz="UTC", - ) + freq="12h", + ).as_unit("us") expected_data_df = pd.DataFrame( { ts_0.name: 6 * [12], @@ -1196,7 +1210,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( }, index=index, ) - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "name" + assert_frame_equal(data_df, expected_data_df) # Local TZ avg data_df = tsdio.get_timeseries_buckets_data( @@ -1218,7 +1233,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( ], name="timestamp", tz=ZoneInfo("Europe/Paris"), - ) + freq="D", + ).as_unit("us") expected_data_df = pd.DataFrame( { ts_0.name: [11.0, 34.5, 58.5], @@ -1227,8 +1243,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( }, index=index, ) - - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "name" + assert_frame_equal(data_df, expected_data_df) # UTC sum, with gapfill data_df = tsdio.get_timeseries_buckets_data( @@ -1251,7 +1267,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( ], name="timestamp", tz="UTC", - ) + freq="D", + ).as_unit("us") expected_data_df = pd.DataFrame( { ts_0.name: [276.0, 852.0, 1428.0, np.nan], @@ -1260,8 +1277,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( }, index=index, ) - - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "name" + assert_frame_equal(data_df, expected_data_df) # UTC min, with gapfill data_df = tsdio.get_timeseries_buckets_data( @@ -1284,7 +1301,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( ], name="timestamp", tz="UTC", - ) + freq="D", + ).as_unit("us") expected_data_df = pd.DataFrame( { ts_0.name: [0.0, 24.0, 48.0, np.nan], @@ -1293,8 +1311,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( }, index=index, ) - - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "name" + assert_frame_equal(data_df, expected_data_df) # UTC max, with gapfill data_df = tsdio.get_timeseries_buckets_data( @@ -1307,7 +1325,6 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( "max", col_label="name", ) - index = pd.DatetimeIndex( [ "2020-01-01T00:00:00", @@ -1317,7 +1334,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( ], name="timestamp", tz="UTC", - ) + freq="D", + ).as_unit("us") expected_data_df = pd.DataFrame( { ts_0.name: [23.0, 47.0, 71.0, np.nan], @@ -1326,8 +1344,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( }, index=index, ) - - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "name" + assert_frame_equal(data_df, expected_data_df) # Test conversions with OpenBar(): @@ -1350,7 +1368,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( ], name="timestamp", tz="UTC", - ) + freq="D", + ).as_unit("us") val_0 = [1000 * 11.5, 1000 * 35.5, 1000 * 59.5] val_2 = [np.nan, np.nan, np.nan] val_4 = [33.0, 81.0, np.nan] @@ -1358,7 +1377,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( {ts_0.id: val_0, ts_2.id: val_2, ts_4.id: val_4}, index=index, ) - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "id" + assert_frame_equal(data_df, expected_data_df) # Undefined: dummy with pytest.raises(BEMServerCoreUndefinedUnitError): data_df = tsdio.get_timeseries_buckets_data( @@ -1412,7 +1432,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( ], name="timestamp", tz="UTC", - ) + freq="D", + ).as_unit("us") val_0 = [24, 24, 24] val_2 = [0, 0, 0] val_4 = [24, 24, 0] @@ -1420,7 +1441,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( {ts_0.id: val_0, ts_2.id: val_2, ts_4.id: val_4}, index=index, ) - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "id" + assert_frame_equal(data_df, expected_data_df) # Purposely set order different than ID order to check output ordering ts_l = (ts_2, ts_0, ts_4) @@ -1434,7 +1456,6 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( 1, "day", "count", - col_label="id", ) index = pd.DatetimeIndex( @@ -1445,7 +1466,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( ], name="timestamp", tz="UTC", - ) + freq="D", + ).as_unit("us") expected_data_df = pd.DataFrame( { ts_2.id: [0, 0, 0], @@ -1454,7 +1476,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( }, index=index, ) - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "id" + assert_frame_equal(data_df, expected_data_df) # No timeseries data_df = tsdio.get_timeseries_buckets_data( @@ -1469,9 +1492,11 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_as_admin( ], name="timestamp", tz="UTC", - ) + freq="D", + ).as_unit("us") expected_data_df = pd.DataFrame({}, index=index) - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "id" + assert_frame_equal(data_df, expected_data_df) # Invalid aggregation with pytest.raises(TimeseriesDataIOInvalidAggregationError): @@ -1556,17 +1581,23 @@ def test_timeseries_data_io_get_timeseries_buckets_data_dst_as_admin( col_label="name", ) - index = pd.DatetimeIndex( - [ - "2020-10-25T00:00:00", - "2020-10-25T01:00:00", - ], - name="timestamp", - tz="UTC", - ).tz_convert(ZoneInfo("Europe/Paris")) + index = ( + pd.DatetimeIndex( + [ + "2020-10-25T00:00:00", + "2020-10-25T01:00:00", + ], + name="timestamp", + tz="UTC", + freq="h", + ) + .as_unit("us") + .tz_convert(ZoneInfo("Europe/Paris")) + ) expected_data_df = pd.DataFrame({ts_0.name: [1, 1]}, index=index) + expected_data_df.columns.name = "name" - assert data_df.equals(expected_data_df) + assert_frame_equal(data_df, expected_data_df) def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_dst_as_admin( self, users, timeseries @@ -1618,9 +1649,11 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_dst_as_admin( ["2020-03-28T00:00:00", "2020-03-29T00:00:00"], name="timestamp", tz=ZoneInfo("Europe/Paris"), - ) + freq="D", + ).as_unit("us") expected_data_df = pd.DataFrame({ts_0.name: [24, 23]}, index=index) - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "name" + assert_frame_equal(data_df, expected_data_df) # local TZ count 1 day - Fall back data_df = tsdio.get_timeseries_buckets_data( @@ -1630,9 +1663,11 @@ def test_timeseries_data_io_get_timeseries_buckets_data_fixed_size_dst_as_admin( ["2020-10-24T00:00:00", "2020-10-25T00:00:00"], name="timestamp", tz=ZoneInfo("Europe/Paris"), - ) + freq="D", + ).as_unit("us") expected_data_df = pd.DataFrame({ts_0.name: [24, 25]}, index=index) - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "name" + assert_frame_equal(data_df, expected_data_df) @pytest.mark.parametrize("timeseries", (5,), indirect=True) def test_timeseries_data_io_get_timeseries_buckets_data_variable_size_as_admin( @@ -1674,7 +1709,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_variable_size_as_admin( ], name="timestamp", tz="UTC", - ) + freq="YS", + ).as_unit("us") expected_data_df = pd.DataFrame( { ts_0.name: [24 * 366, 24 * 365], @@ -1683,8 +1719,9 @@ def test_timeseries_data_io_get_timeseries_buckets_data_variable_size_as_admin( }, index=index, ) + expected_data_df.columns.name = "name" - assert data_df.equals(expected_data_df) + assert_frame_equal(data_df, expected_data_df) # UTC count year, with gapfill data_df = tsdio.get_timeseries_buckets_data( @@ -1706,7 +1743,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_variable_size_as_admin( ], name="timestamp", tz="UTC", - ) + freq="YS", + ).as_unit("us") expected_data_df = pd.DataFrame( { ts_0.name: [24 * 366, 24 * 365, 0], @@ -1715,7 +1753,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_variable_size_as_admin( }, index=index, ) - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "name" + assert_frame_equal(data_df, expected_data_df) # Local TZ count year - start_dt floored data_df = tsdio.get_timeseries_buckets_data( @@ -1737,7 +1776,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_variable_size_as_admin( ], name="timestamp", tz=ZoneInfo("Europe/Paris"), - ) + freq="YS", + ).as_unit("us") expected_data_df = pd.DataFrame( { ts_0.name: [ @@ -1754,7 +1794,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_variable_size_as_admin( }, index=index, ) - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "name" + assert_frame_equal(data_df, expected_data_df) # UTC avg month, with gapfill data_df = tsdio.get_timeseries_buckets_data( @@ -1778,7 +1819,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_variable_size_as_admin( ], name="timestamp", tz="UTC", - ) + freq="MS", + ).as_unit("us") expected_data_df = pd.DataFrame( { @@ -1788,8 +1830,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_variable_size_as_admin( }, index=index, ) - - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "name" + assert_frame_equal(data_df, expected_data_df) # UTC avg month data_df = tsdio.get_timeseries_buckets_data( @@ -1811,7 +1853,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_variable_size_as_admin( ], name="timestamp", tz="UTC", - ) + freq="MS", + ).as_unit("us") expected_data_df = pd.DataFrame( { ts_0.name: [371.5, 1091.5, 1811.5], @@ -1820,8 +1863,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_variable_size_as_admin( }, index=index, ) - - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "name" + assert_frame_equal(data_df, expected_data_df) # Local TZ avg month data_df = tsdio.get_timeseries_buckets_data( @@ -1845,7 +1888,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_variable_size_as_admin( ], name="timestamp", tz=ZoneInfo("Europe/Paris"), - ) + freq="MS", + ).as_unit("us") expected_data_df = pd.DataFrame( { ts_0.name: [371.0, 1090.5, 1810.0, 2541.5], @@ -1854,8 +1898,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_variable_size_as_admin( }, index=index, ) - - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "name" + assert_frame_equal(data_df, expected_data_df) # UTC sum month data_df = tsdio.get_timeseries_buckets_data( @@ -1877,7 +1921,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_variable_size_as_admin( ], name="timestamp", tz="UTC", - ) + freq="MS", + ).as_unit("us") expected_data_df = pd.DataFrame( { ts_0.name: [276396.0, 759684.0, 1347756.0], @@ -1886,8 +1931,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_variable_size_as_admin( }, index=index, ) - - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "name" + assert_frame_equal(data_df, expected_data_df) # UTC min month data_df = tsdio.get_timeseries_buckets_data( @@ -1909,7 +1954,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_variable_size_as_admin( ], name="timestamp", tz="UTC", - ) + freq="MS", + ).as_unit("us") expected_data_df = pd.DataFrame( { ts_0.name: [0.0, 744.0, 1440.0], @@ -1918,8 +1964,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_variable_size_as_admin( }, index=index, ) - - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "name" + assert_frame_equal(data_df, expected_data_df) # UTC max month data_df = tsdio.get_timeseries_buckets_data( @@ -1941,7 +1987,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_variable_size_as_admin( ], name="timestamp", tz="UTC", - ) + freq="MS", + ).as_unit("us") expected_data_df = pd.DataFrame( { ts_0.name: [743.0, 1439.0, 2183.0], @@ -1950,8 +1997,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_variable_size_as_admin( }, index=index, ) - - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "name" + assert_frame_equal(data_df, expected_data_df) # UTC count year by ID data_df = tsdio.get_timeseries_buckets_data( @@ -1965,7 +2012,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_variable_size_as_admin( ], name="timestamp", tz="UTC", - ) + freq="YS", + ).as_unit("us") expected_data_df = pd.DataFrame( { ts_0.id: [24 * 366, 24 * 365], @@ -1974,8 +2022,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_variable_size_as_admin( }, index=index, ) - - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "id" + assert_frame_equal(data_df, expected_data_df) # No timeseries data_df = tsdio.get_timeseries_buckets_data( @@ -1989,9 +2037,11 @@ def test_timeseries_data_io_get_timeseries_buckets_data_variable_size_as_admin( ], name="timestamp", tz="UTC", - ) + freq="YS", + ).as_unit("us") expected_data_df = pd.DataFrame({}, index=index) - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "id" + assert_frame_equal(data_df, expected_data_df) with pytest.raises(BEMServerCorePeriodError): # 2 years @@ -2060,7 +2110,8 @@ def test_timeseries_data_io_get_timeseries_buckets_data_as_user( ], name="timestamp", tz="UTC", - ) + freq="D", + ).as_unit("us") expected_data_df = pd.DataFrame( { ts_1.name if col_label == "name" else ts_1.id: [11.5, 35.5, 59.5], @@ -2068,8 +2119,9 @@ def test_timeseries_data_io_get_timeseries_buckets_data_as_user( }, index=index, ) + expected_data_df.columns.name = col_label - assert data_df.equals(expected_data_df) + assert_frame_equal(data_df, expected_data_df) @pytest.mark.parametrize("timeseries", (5,), indirect=True) def test_timeseries_data_io_delete_as_admin( @@ -2356,13 +2408,13 @@ def test_timeseries_data_io_import_csv_error( # Value not float ("2020-01-01T00:00:00+00:00,a", TimeseriesDataCSVIOError), # Too many columns - ("2500-01-01T00:00:00+00:00,1,2,3,4,5", TimeseriesDataCSVIOError), + ("2020-01-01T00:00:00+00:00,1,2,3,4,5", TimeseriesDataCSVIOError), # Naive datetime ("2020-01-01T00:00:00,12", TimeseriesDataIODatetimeError), # Invalid timestamp ("dummy,1", TimeseriesDataIODatetimeError), ("0,1", TimeseriesDataIODatetimeError), - ("2500-01-01T00:00:00+00:00,12", TimeseriesDataIODatetimeError), + ("10000-01-01T00:00:00+00:00,12", TimeseriesDataIODatetimeError), ), ) @pytest.mark.usefixtures("timeseries") @@ -2378,7 +2430,7 @@ def test_timeseries_data_io_import_csv_row_error( with OpenBar(): ds_1 = TimeseriesDataState.get(name="Raw").first() - header = "Datetime,Timeseries 0\n" if for_campaign else "Datetime,1\n" + header = "Datetime,Timeseries 1\n" if for_campaign else "Datetime,1\n" csv_data = header + row with CurrentUser(admin_user): @@ -2970,7 +3022,7 @@ def test_timeseries_data_io_import_json_as_user( # Invalid timestamp ('{"1": {"dummy": 1}}', TimeseriesDataIODatetimeError), ('{"1": [{"dummy": 1}]}', TimeseriesDataIODatetimeError), - ('{"1": {"2500-01-01T00:00:00+00:00": 1}}', TimeseriesDataIODatetimeError), + ('{"1": {"10000-01-01T00:00:00+00:00": 1}}', TimeseriesDataIODatetimeError), # Invalid value ('{"1": {"2020-01-01T00:00:00+00:00": "lol"}}', TimeseriesDataJSONIOError), ), diff --git a/tests/process/test_cleanup.py b/tests/process/test_cleanup.py index b0168617..8a7d76cc 100644 --- a/tests/process/test_cleanup.py +++ b/tests/process/test_cleanup.py @@ -6,6 +6,7 @@ import numpy as np import pandas as pd +from pandas.testing import assert_frame_equal from bemserver_core.authorization import CurrentUser, OpenBar from bemserver_core.model import ( @@ -79,7 +80,10 @@ def test_cleanup_process(self, users, timeseries): ts_2.id: [0, 13, 33, 42, 69], ts_3.id: [np.nan, np.nan, np.nan, np.nan, np.nan], }, - index=pd.DatetimeIndex(timestamps, name="timestamp"), + index=pd.DatetimeIndex(timestamps, name="timestamp", freq=None).as_unit( + "us" + ), dtype=float, ) - assert ret.equals(expected) + expected.columns.name = "id" + assert_frame_equal(ret, expected) diff --git a/tests/process/test_energy_power.py b/tests/process/test_energy_power.py index 213979a6..a298106c 100644 --- a/tests/process/test_energy_power.py +++ b/tests/process/test_energy_power.py @@ -84,7 +84,9 @@ def test_power2energy_process(self, users, timeseries): ) expected_data_s = pd.Series( [0.0, 0.0, 0.5, 0.5, 0.0, 0.0, 0.5, 0.5], - index=pd.DatetimeIndex(timestamps, name="timestamp", freq="1800s"), + index=pd.DatetimeIndex( + timestamps, name="timestamp", freq="1800s" + ).as_unit("us"), ) assert_series_equal(data_s, expected_data_s) @@ -94,7 +96,9 @@ def test_power2energy_process(self, users, timeseries): ] expected_data_s = pd.Series( [0.0, 1.0, 0.0, 1.0], - index=pd.DatetimeIndex(timestamps, name="timestamp", freq="3600s"), + index=pd.DatetimeIndex( + timestamps, name="timestamp", freq="3600s" + ).as_unit("us"), ) assert_series_equal(data_s, expected_data_s) @@ -104,7 +108,9 @@ def test_power2energy_process(self, users, timeseries): ] expected_data_s = pd.Series( [1.0, 1.0], - index=pd.DatetimeIndex(timestamps, name="timestamp", freq="7200s"), + index=pd.DatetimeIndex( + timestamps, name="timestamp", freq="7200s" + ).as_unit("us"), ) assert_series_equal(data_s, expected_data_s) @@ -114,7 +120,9 @@ def test_power2energy_process(self, users, timeseries): ] expected_data_s = pd.Series( [0.0, 1000.0, 0.0, 1000.0], - index=pd.DatetimeIndex(timestamps, name="timestamp", freq="3600s"), + index=pd.DatetimeIndex( + timestamps, name="timestamp", freq="3600s" + ).as_unit("us"), ) assert_series_equal(data_s, expected_data_s) @@ -125,7 +133,9 @@ def test_power2energy_process(self, users, timeseries): ] expected_data_s = pd.Series( [0.0, 12.0, 15.0, 18.0], - index=pd.DatetimeIndex(timestamps, name="timestamp", freq="7200s"), + index=pd.DatetimeIndex( + timestamps, name="timestamp", freq="7200s" + ).as_unit("us"), ) assert_series_equal(data_s, expected_data_s) @@ -135,7 +145,9 @@ def test_power2energy_process(self, users, timeseries): ] expected_data_s = pd.Series( [np.nan, 168000.0, 168000.0], - index=pd.DatetimeIndex(timestamps, name="timestamp", freq="14400s"), + index=pd.DatetimeIndex( + timestamps, name="timestamp", freq="14400s" + ).as_unit("us"), ) assert_series_equal(data_s, expected_data_s) @@ -145,7 +157,9 @@ def test_power2energy_process(self, users, timeseries): ] expected_data_s = pd.Series( [np.nan, np.nan, np.nan], - index=pd.DatetimeIndex(timestamps, name="timestamp", freq="14400s"), + index=pd.DatetimeIndex( + timestamps, name="timestamp", freq="14400s" + ).as_unit("us"), ) assert_series_equal(data_s, expected_data_s) @@ -174,7 +188,9 @@ def test_power2energy_process(self, users, timeseries): ) expected_data_s = pd.Series( [0.0, 0.0, 0.5, 0.5, 0.0, 0.0, 0.5, 0.5], - index=pd.DatetimeIndex(timestamps, name="timestamp", freq="1800s"), + index=pd.DatetimeIndex( + timestamps, name="timestamp", freq="1800s" + ).as_unit("us"), ) assert_series_equal(data_s, expected_data_s) @@ -254,7 +270,7 @@ def test_energy2power_process(self, users, timeseries): ] expected_data_s = pd.Series( [0.0, 1.0, 0.0, 1.0], - index=pd.DatetimeIndex(timestamps, name="timestamp"), + index=pd.DatetimeIndex(timestamps, name="timestamp").as_unit("us"), ) assert_series_equal(data_s, expected_data_s) @@ -264,7 +280,7 @@ def test_energy2power_process(self, users, timeseries): ] expected_data_s = pd.Series( [0.0, 1000.0, 0.0, 1000.0], - index=pd.DatetimeIndex(timestamps, name="timestamp"), + index=pd.DatetimeIndex(timestamps, name="timestamp").as_unit("us"), ) assert_series_equal(data_s, expected_data_s) @@ -274,7 +290,7 @@ def test_energy2power_process(self, users, timeseries): ] expected_data_s = pd.Series( [6.0, 9.0], - index=pd.DatetimeIndex(timestamps, name="timestamp"), + index=pd.DatetimeIndex(timestamps, name="timestamp").as_unit("us"), ) assert_series_equal(data_s, expected_data_s) @@ -284,7 +300,7 @@ def test_energy2power_process(self, users, timeseries): [ 21000.0, ], - index=pd.DatetimeIndex(timestamps, name="timestamp"), + index=pd.DatetimeIndex(timestamps, name="timestamp").as_unit("us"), ) assert_series_equal(data_s, expected_data_s) @@ -292,7 +308,9 @@ def test_energy2power_process(self, users, timeseries): timestamps = [] expected_data_s = pd.Series( [], - index=pd.DatetimeIndex(timestamps, name="timestamp", tz=dt.UTC), + index=pd.DatetimeIndex(timestamps, name="timestamp", tz=dt.UTC).as_unit( + "us" + ), dtype=float, ) assert_series_equal(data_s, expected_data_s) @@ -315,7 +333,7 @@ def test_energy2power_process(self, users, timeseries): ] expected_data_s = pd.Series( [0.0, 1.0, 0.0, 1.0], - index=pd.DatetimeIndex(timestamps, name="timestamp"), + index=pd.DatetimeIndex(timestamps, name="timestamp").as_unit("us"), ) assert_series_equal(data_s, expected_data_s) @@ -373,7 +391,9 @@ def test_energyindex2power_process(self, users, timeseries): ] expected_data_s = pd.Series( [1.0, 3.0, 5.0, 7.0, 9.0, np.nan], - index=pd.DatetimeIndex(timestamps, name="timestamp", freq="h"), + index=pd.DatetimeIndex(timestamps, name="timestamp", freq="h").as_unit( + "us" + ), ) assert_series_equal(data_s, expected_data_s) @@ -383,7 +403,9 @@ def test_energyindex2power_process(self, users, timeseries): ] expected_data_s = pd.Series( [2.0, 6.0, 9.0], - index=pd.DatetimeIndex(timestamps, name="timestamp", freq="2h"), + index=pd.DatetimeIndex(timestamps, name="timestamp", freq="2h").as_unit( + "us" + ), ) assert_series_equal(data_s, expected_data_s) @@ -394,7 +416,9 @@ def test_energyindex2power_process(self, users, timeseries): ] expected_data_s = pd.Series( [1000.0, 3000.0, 5000.0, 7000.0, 9000.0, np.nan], - index=pd.DatetimeIndex(timestamps, name="timestamp", freq="h"), + index=pd.DatetimeIndex(timestamps, name="timestamp", freq="h").as_unit( + "us" + ), ) assert_series_equal(data_s, expected_data_s) @@ -405,7 +429,9 @@ def test_energyindex2power_process(self, users, timeseries): ] expected_data_s = pd.Series( [np.nan, np.nan, 14.0, 14.0, 18.0, 18.0, np.nan, np.nan], - index=pd.DatetimeIndex(timestamps, name="timestamp", freq="h"), + index=pd.DatetimeIndex(timestamps, name="timestamp", freq="h").as_unit( + "us" + ), ) assert_series_equal(data_s, expected_data_s) @@ -416,7 +442,9 @@ def test_energyindex2power_process(self, users, timeseries): ] expected_data_s = pd.Series( [6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 14.0, 14.0, 18.0, 18.0, np.nan, np.nan], - index=pd.DatetimeIndex(timestamps, name="timestamp", freq="h"), + index=pd.DatetimeIndex(timestamps, name="timestamp", freq="h").as_unit( + "us" + ), ) assert_series_equal(data_s, expected_data_s) @@ -426,7 +454,9 @@ def test_energyindex2power_process(self, users, timeseries): ] expected_data_s = pd.Series( 12 * [np.nan], - index=pd.DatetimeIndex(timestamps, name="timestamp", freq="h"), + index=pd.DatetimeIndex(timestamps, name="timestamp", freq="h").as_unit( + "us" + ), ) assert_series_equal(data_s, expected_data_s) @@ -439,7 +469,7 @@ def test_energyindex2power_process(self, users, timeseries): 3 * [np.nan], index=pd.DatetimeIndex( timestamps, name="timestamp", tz=dt.UTC, freq="4h" - ), + ).as_unit("us"), dtype=float, ) assert_series_equal(data_s, expected_data_s) @@ -461,7 +491,9 @@ def test_energyindex2power_process(self, users, timeseries): ] expected_data_s = pd.Series( [1.0, 3.0, 5.0, 7.0, 9.0, np.nan], - index=pd.DatetimeIndex(timestamps, name="timestamp", freq="h"), + index=pd.DatetimeIndex(timestamps, name="timestamp", freq="h").as_unit( + "us" + ), ) assert_series_equal(data_s, expected_data_s) @@ -519,7 +551,9 @@ def test_energyindex2energy_process(self, users, timeseries): ] expected_data_s = pd.Series( [1.0, 3.0, 5.0, 7.0, 9.0, np.nan], - index=pd.DatetimeIndex(timestamps, name="timestamp", freq="h"), + index=pd.DatetimeIndex(timestamps, name="timestamp", freq="h").as_unit( + "us" + ), ) assert_series_equal(data_s, expected_data_s) @@ -529,7 +563,9 @@ def test_energyindex2energy_process(self, users, timeseries): ] expected_data_s = pd.Series( [4.0, 12.0, 18.0], - index=pd.DatetimeIndex(timestamps, name="timestamp", freq="2h"), + index=pd.DatetimeIndex(timestamps, name="timestamp", freq="2h").as_unit( + "us" + ), ) assert_series_equal(data_s, expected_data_s) @@ -540,7 +576,9 @@ def test_energyindex2energy_process(self, users, timeseries): ] expected_data_s = pd.Series( [1000.0, 3000.0, 5000.0, 7000.0, 9000.0, np.nan], - index=pd.DatetimeIndex(timestamps, name="timestamp", freq="h"), + index=pd.DatetimeIndex(timestamps, name="timestamp", freq="h").as_unit( + "us" + ), ) assert_series_equal(data_s, expected_data_s) @@ -551,7 +589,9 @@ def test_energyindex2energy_process(self, users, timeseries): ] expected_data_s = pd.Series( [np.nan, np.nan, 14.0, 14.0, 18.0, 18.0, np.nan, np.nan], - index=pd.DatetimeIndex(timestamps, name="timestamp", freq="h"), + index=pd.DatetimeIndex(timestamps, name="timestamp", freq="h").as_unit( + "us" + ), ) assert_series_equal(data_s, expected_data_s) @@ -562,7 +602,9 @@ def test_energyindex2energy_process(self, users, timeseries): ] expected_data_s = pd.Series( [6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 14.0, 14.0, 18.0, 18.0, np.nan, np.nan], - index=pd.DatetimeIndex(timestamps, name="timestamp", freq="h"), + index=pd.DatetimeIndex(timestamps, name="timestamp", freq="h").as_unit( + "us" + ), ) assert_series_equal(data_s, expected_data_s) @@ -572,7 +614,9 @@ def test_energyindex2energy_process(self, users, timeseries): ] expected_data_s = pd.Series( 12 * [np.nan], - index=pd.DatetimeIndex(timestamps, name="timestamp", freq="h"), + index=pd.DatetimeIndex(timestamps, name="timestamp", freq="h").as_unit( + "us" + ), ) assert_series_equal(data_s, expected_data_s) @@ -585,7 +629,7 @@ def test_energyindex2energy_process(self, users, timeseries): 3 * [np.nan], index=pd.DatetimeIndex( timestamps, name="timestamp", tz=dt.UTC, freq="4h" - ), + ).as_unit("us"), dtype=float, ) assert_series_equal(data_s, expected_data_s) @@ -607,7 +651,9 @@ def test_energyindex2energy_process(self, users, timeseries): ] expected_data_s = pd.Series( [1.0, 3.0, 5.0, 7.0, 9.0, np.nan], - index=pd.DatetimeIndex(timestamps, name="timestamp", freq="h"), + index=pd.DatetimeIndex(timestamps, name="timestamp", freq="h").as_unit( + "us" + ), ) assert_series_equal(data_s, expected_data_s) diff --git a/tests/process/test_forward_fill.py b/tests/process/test_forward_fill.py index bf6ac835..d75dbdd5 100644 --- a/tests/process/test_forward_fill.py +++ b/tests/process/test_forward_fill.py @@ -70,7 +70,7 @@ def test_ffill_process(self, users, timeseries): ts_2.id: [np.nan, np.nan, 42.0, 42.0, np.nan, 42.0], ts_3.id: [np.nan, np.nan, np.nan, np.nan, np.nan, np.nan], }, - index=pd.DatetimeIndex(timestamps, name="timestamp"), + index=pd.DatetimeIndex(timestamps, name="timestamp").as_unit("us"), ) assert_frame_equal(data_df, expected_data_df) @@ -90,7 +90,7 @@ def test_ffill_process(self, users, timeseries): ts_2.id: [42.0, 42.0, np.nan, 42.0], ts_3.id: [np.nan, np.nan, np.nan, np.nan], }, - index=pd.DatetimeIndex(timestamps, name="timestamp"), + index=pd.DatetimeIndex(timestamps, name="timestamp").as_unit("us"), ) assert_frame_equal(data_df, expected_data_df) @@ -110,7 +110,7 @@ def test_ffill_process(self, users, timeseries): 3: [np.nan, np.nan, 42.0, 42.0, 42.0], 4: [0.0, 1.0, 1.0, 1.0, 1.0], }, - index=pd.DatetimeIndex(timestamps, name="timestamp"), + index=pd.DatetimeIndex(timestamps, name="timestamp").as_unit("us"), ) expected_data_df.columns = [ts_3.id, ts_2.id, ts_2.id, ts_0.id] assert_frame_equal(data_df, expected_data_df) @@ -138,6 +138,6 @@ def test_ffill_process(self, users, timeseries): ts_2.id: [np.nan, 42.0, 42.0, 42.0, 42.0], ts_3.id: [np.nan, np.nan, np.nan, np.nan, np.nan], }, - index=pd.DatetimeIndex(timestamps, name="timestamp"), + index=pd.DatetimeIndex(timestamps, name="timestamp").as_unit("us"), ) assert_frame_equal(data_df, expected_data_df) diff --git a/tests/process/test_weather.py b/tests/process/test_weather.py index 23d5aa62..aa8fb6fe 100644 --- a/tests/process/test_weather.py +++ b/tests/process/test_weather.py @@ -96,7 +96,7 @@ def test_get_weather_data(self, mock_get, forecast): ["2020-01-01T00:00:00+00:00", "2020-01-01T01:00:00+00:00"], name="timestamp", tz="UTC", - ) + ).as_unit("us") expected_data_df = pd.DataFrame( {"AIR_TEMPERATURE": [2.45, 2.59], "RELATIVE_HUMIDITY": [0.78, 0.78]}, index=index, @@ -354,7 +354,7 @@ def test_get_weather_data_for_site( ], name="timestamp", tz="UTC", - ) + ).as_unit("us") expected_data_df = pd.DataFrame( {"Timeseries 1": [2.45, 2.59], "Timeseries 2": [np.nan, np.nan]}, index=index, @@ -379,7 +379,7 @@ def test_get_weather_data_for_site( ], name="timestamp", tz="UTC", - ) + ).as_unit("us") expected_data_df = pd.DataFrame( {"Timeseries 1": [2.45, 2.59], "Timeseries 2": [78.0, 79.0]}, index=index ) diff --git a/tests/scheduled_tasks/test_cleanup.py b/tests/scheduled_tasks/test_cleanup.py index f22522c6..2c14716b 100644 --- a/tests/scheduled_tasks/test_cleanup.py +++ b/tests/scheduled_tasks/test_cleanup.py @@ -5,6 +5,7 @@ import pytest import pandas as pd +from pandas.testing import assert_frame_equal from bemserver_core.authorization import OpenBar from bemserver_core.database import db @@ -56,13 +57,15 @@ def test_cleanup_data(self, users, timeseries, campaigns): ], name="timestamp", tz="UTC", - ) + ).as_unit("us") val_0 = [13.0] expected_data_df = pd.DataFrame({ts_0.id: val_0}, index=index) - assert data_df.equals(expected_data_df) + expected_data_df.columns.name = "id" + assert_frame_equal(data_df, expected_data_df) # Campaign 2 (not cleaned), TS 1 -> no clean data data_df = tsdio.get_timeseries_data(start_dt, end_dt, (ts_1,), ds_2) - index = pd.DatetimeIndex([], name="timestamp", tz="UTC") + index = pd.DatetimeIndex([], name="timestamp", tz="UTC").as_unit("us") no_data_df = pd.DataFrame({ts_1.id: []}, index=index) - assert data_df.equals(no_data_df) + no_data_df.columns.name = "id" + assert_frame_equal(data_df, no_data_df) diff --git a/tests/scheduled_tasks/test_download_weather_data.py b/tests/scheduled_tasks/test_download_weather_data.py index efe2ed05..663ad53c 100644 --- a/tests/scheduled_tasks/test_download_weather_data.py +++ b/tests/scheduled_tasks/test_download_weather_data.py @@ -141,7 +141,7 @@ def test_download_weather_data( ], name="timestamp", tz="UTC", - ) + ).as_unit("us") if forecast is False: expected_data_df = pd.DataFrame( {"Timeseries 1": [2.45, 2.59], "Timeseries 2": [np.nan, np.nan]},