From 070552469a0a30d8d1df057c323a3203ccb895b8 Mon Sep 17 00:00:00 2001 From: stewit <> Date: Tue, 20 Oct 2026 22:36:45 +0200 Subject: [PATCH] fixes around fallback behaviour (null, propagation to weaker fields) --- src/hdhelpers/metadata/_helpers.py | 51 ++++- src/hdhelpers/metadata/_specs.py | 6 + src/hdhelpers/metadata/helpers.py | 101 +++++++--- tests/metadata/test_metadata_fallbacks.py | 227 ++++++++++++++++++++++ 4 files changed, 351 insertions(+), 34 deletions(-) create mode 100644 tests/metadata/test_metadata_fallbacks.py diff --git a/src/hdhelpers/metadata/_helpers.py b/src/hdhelpers/metadata/_helpers.py index 0120981..573e031 100644 --- a/src/hdhelpers/metadata/_helpers.py +++ b/src/hdhelpers/metadata/_helpers.py @@ -63,15 +63,46 @@ def get_value_dimension_info( ) +def follows_current_convention(attrs: Any) -> bool: + """Whether metadata follows the current metadata convention, i.e. has a list of metrics""" + return isinstance(attrs, dict) and isinstance(attrs.get("metrics"), list) + + +def value_dimension_info_with_fallbacks( + data: pd.DataFrame | pd.Series, fields: tuple[str, ...], legacy_spec: Spec +) -> defaultdict[str, defaultdict[str, Any]]: + """Obtain a field of the value dimensions, falling back to further fields + + For metadata following the current metadata convention, each field is looked up on all + levels described in get_value_dimension_info (the value dimension, the metric for the + value dimension "value" and "value_dimensions_shared") before the next field is tried. + So e.g. a display name of the metric wins over the name of its value dimension. Fields + provided as null are treated as missing. + + Older metadata structures are handled via legacy_spec, keeping their established behaviour. + """ + if not follows_current_convention(data.attrs): + return get_value_dimension_info(data, legacy_spec) + + result: defaultdict[str, defaultdict[str, Any]] = defaultdict(lambda: defaultdict(lambda: None)) + for field in fields: + info_by_metric = get_value_dimension_info(data, Coalesce(field, default=None)) + for metric, info_by_value_dimension in info_by_metric.items(): + for value_dimension, info in info_by_value_dimension.items(): + if result[metric][value_dimension] is None: + result[metric][value_dimension] = info + return result + + _SINGLE_METRIC_KEY = "single_metric" def _select_single_metric(info_by_metric: dict, attrs: Any, empty: Any) -> Any: - """Pick the entry of the one metric of a SingleTSFrame out of a by-metric mapping + """Pick the entry of the one metric of a SingleTSFrame or Series out of a by-metric mapping The single metric is identified via _SINGLE_METRIC_KEY in "dataset_metadata". If that is - missing but exactly one metric is present, that one is used, since a SingleTSFrame cannot - be ambiguous in this respect. Otherwise `empty` is returned. + missing but exactly one metric is present, that one is used, since a SingleTSFrame or + Series cannot be ambiguous in this respect. Otherwise `empty` is returned. """ metric = extract_from_metadata(attrs, key=_SINGLE_METRIC_KEY) if metric is not None and metric in info_by_metric: @@ -106,6 +137,20 @@ def singlets_value_dimension_info( ) +def singlets_value_dimension_info_with_fallbacks( + singletsframe: pd.DataFrame, fields: tuple[str, ...], legacy_spec: Spec +) -> defaultdict[str, Any]: + """Counterpart of value_dimension_info_with_fallbacks for SingleTSFrames + + Like singlets_value_dimension_info, the result is keyed by value dimension only. + """ + return _select_single_metric( + value_dimension_info_with_fallbacks(singletsframe, fields, legacy_spec), + singletsframe.attrs, + empty=defaultdict(lambda: None), + ) + + def singlets_metric_info(singletsframe: pd.DataFrame, metric_info: str | Spec) -> Any: """Metadata of the single metric of a SingleTSFrame diff --git a/src/hdhelpers/metadata/_specs.py b/src/hdhelpers/metadata/_specs.py index e5749bf..13cbab5 100644 --- a/src/hdhelpers/metadata/_specs.py +++ b/src/hdhelpers/metadata/_specs.py @@ -12,6 +12,12 @@ DISPLAY_NAME: Spec = Coalesce("display_name", "name", default=None) SHORT_DISPLAY_NAME: Spec = Coalesce("short_display_name", "display_name", "name", default=None) +# The fallback chains of the display names for metadata following the current metadata +# convention: Each field is looked up on all levels (value dimension, metric, shared value +# dimensions) before the next field is tried, see value_dimension_info_with_fallbacks. +DISPLAY_NAME_FIELDS = ("display_name", "name") +SHORT_DISPLAY_NAME_FIELDS = ("short_display_name", "display_name", "name") + def by_metric_key_by_val_dimension(metadatum_key: str | Spec) -> Spec: """Providesglom spec that extracts a metadatum by metric by value dimension diff --git a/src/hdhelpers/metadata/helpers.py b/src/hdhelpers/metadata/helpers.py index d0ff0c4..e845e4a 100644 --- a/src/hdhelpers/metadata/helpers.py +++ b/src/hdhelpers/metadata/helpers.py @@ -14,19 +14,22 @@ from typing import Any, cast import pandas as pd -from glom import Coalesce, Spec, glom +from glom import Coalesce, GlomError, Spec, glom +import hdhelpers.metadata._specs as _specs from hdhelpers.metadata._helpers import ( check_dataframe, check_series, check_series_or_dataframe, extract_from_metadata, + follows_current_convention, get_value_dimension_info, singlets_metric_info, singlets_value_dimension_info, + singlets_value_dimension_info_with_fallbacks, spec_not_none, + value_dimension_info_with_fallbacks, ) -import hdhelpers.metadata._specs as _specs from hdhelpers.metadata._specs import spec_by_metric_key @@ -135,6 +138,9 @@ def get_display_names(multitsframe: pd.DataFrame) -> defaultdict[str, defaultdic Returns: defaultdict[str, defaultdict[str, str | None]]: Dictionary of metrics containing the display names. If the display name of the metrics is not present it returns the result of :func:`hdhelpers.metadata.get_names`. + For metadata following the current convention, the display name is looked up on the value dimension, on + the metric (for the value dimension "value") and in "value_dimensions_shared" before falling back to the + name. Fields provided as null count as missing. Raises: TypeError: If `multitsframe` is not a DataFrame. @@ -169,7 +175,9 @@ def get_display_names(multitsframe: pd.DataFrame) -> defaultdict[str, defaultdic """ check_dataframe(multitsframe) - return get_value_dimension_info(multitsframe, _specs.DISPLAY_NAME) + return value_dimension_info_with_fallbacks( + multitsframe, _specs.DISPLAY_NAME_FIELDS, _specs.DISPLAY_NAME + ) def get_short_display_names( @@ -183,6 +191,9 @@ def get_short_display_names( Returns: defaultdict[str, defaultdict[str, str | None]]: Dictionary of metrics containing the short display names. If the short display name of the metrics is not present it returns the result of :func:`hdhelpers.metadata.get_display_names`. + For metadata following the current convention, each of short display name, display name and name is + looked up on the value dimension, on the metric (for the value dimension "value") and in + "value_dimensions_shared" before falling back to the next one. Fields provided as null count as missing. Raises: TypeError: If `multitsframe` is not a DataFrame. @@ -220,7 +231,9 @@ def get_short_display_names( """ check_dataframe(multitsframe) - return get_value_dimension_info(multitsframe, _specs.SHORT_DISPLAY_NAME) + return value_dimension_info_with_fallbacks( + multitsframe, _specs.SHORT_DISPLAY_NAME_FIELDS, _specs.SHORT_DISPLAY_NAME + ) def get_measurements(multitsframe: pd.DataFrame) -> defaultdict[str, defaultdict[str, str | None]]: @@ -264,7 +277,7 @@ def get_metric_info(multitsframe: pd.DataFrame, metric_info: str | Spec) -> defa metric_info (str | Spec): Name of information to retrieve. Note that `metric_info` is interpreted as a glom Spec. Returns: - defaultdict[str, Any]: Dictionary, where keys are the entries of the metrics metadata specified via "metric_key" in "dataset_metadata" and values are the entries specified via "metric_info" in the metrics metadata + defaultdict[str, Any]: Dictionary, where keys are the entries of the metrics metadata specified via "metric_key" in "dataset_metadata" and values are the entries specified via "metric_info" in the metrics metadata. Defaults to None, also if there is no metric metadata at all. Code example: @@ -311,8 +324,11 @@ def get_metric_info(multitsframe: pd.DataFrame, metric_info: str | Spec) -> defa """ check_dataframe(multitsframe) spec = spec_by_metric_key(metric_info) - metric_info = glom(multitsframe.attrs, spec) - return defaultdict(lambda: None, metric_info) + try: + info_by_metric = glom(multitsframe.attrs, spec) + except GlomError: # no metric metadata present at all + info_by_metric = {} + return defaultdict(lambda: None, info_by_metric) def get_series_info(series: pd.Series, value_dim_info: str | Spec) -> Any: @@ -344,10 +360,7 @@ def get_series_info(series: pd.Series, value_dim_info: str | Spec) -> Any: # metadata, we can reuse the code that extracts value_dimension metadata for # this value dimension. check_series(series) - series_metric_key = extract_from_metadata(series.attrs, key="single_metric", default="series") - from_new_convention = get_value_dimension_info(series, value_dim_info)[series_metric_key].get( - "value" - ) + from_new_convention = _series_value(get_value_dimension_info(series, value_dim_info), series) if from_new_convention is not None: return from_new_convention @@ -373,6 +386,37 @@ def get_series_info(series: pd.Series, value_dim_info: str | Spec) -> Any: ) +def _series_value(info_by_metric: dict, series: pd.Series) -> Any: + """Info of the value dimension "value" of the metric of a Series + + The metric is identified via "single_metric" in "dataset_metadata", defaulting to "series" + as used by older metadata structures. If it is not found but exactly one metric is present, + that one is used, since a Series cannot be ambiguous in this respect. + """ + series_metric_key = extract_from_metadata(series.attrs, key="single_metric", default="series") + if series_metric_key in info_by_metric: + return info_by_metric[series_metric_key].get("value") + if len(info_by_metric) == 1: + return next(iter(info_by_metric.values())).get("value") + return None + + +def _series_info_with_fallbacks(series: pd.Series, fields: tuple[str, ...]) -> Any: + """Info of a Series, falling back to further fields + + For metadata following the current metadata convention, each field is looked up on the + value dimension and the metric before the next field is tried, see + value_dimension_info_with_fallbacks. Older metadata structures keep their established + behaviour of using the first of the fields present on the first level providing any. + """ + legacy_spec = Coalesce(*(spec_not_none(field) for field in fields)) + if follows_current_convention(series.attrs): + return _series_value( + value_dimension_info_with_fallbacks(series, fields, legacy_spec), series + ) + return get_series_info(series, legacy_spec) + + def get_series_unit(series: pd.Series) -> str | None: """Gets name of the series from metadata @@ -492,14 +536,7 @@ def get_series_display_name(series: pd.Series) -> str | None: check_series(series) return cast( str | None, - get_series_info( - series, - Coalesce( - spec_not_none("display_name"), - spec_not_none("name"), - spec_not_none("short_display_name"), - ), - ), + _series_info_with_fallbacks(series, ("display_name", "name", "short_display_name")), ) @@ -530,14 +567,7 @@ def get_series_short_display_name(series: pd.Series) -> str | None: check_series(series) return cast( str | None, - get_series_info( - series, - Coalesce( - spec_not_none("short_display_name"), - spec_not_none("display_name"), - spec_not_none("name"), - ), - ), + _series_info_with_fallbacks(series, ("short_display_name", "display_name", "name")), ) @@ -714,7 +744,10 @@ def get_singlets_names(singletsframe: pd.DataFrame) -> defaultdict[str, str | No def get_singlets_display_names(singletsframe: pd.DataFrame) -> defaultdict[str, str | None]: """Gets display names of the value dimensions of a SingleTSFrame from metadata - Falls back to the name of the value dimension if no display name is given. + Falls back to the name of the value dimension if no display name is given. The display name + is looked up on the value dimension, on the metric (for the value dimension "value") and in + "value_dimensions_shared" before falling back to the name. Fields provided as null count as + missing. Args: singletsframe (pd.DataFrame): STSF with metadata following the convention. @@ -749,14 +782,18 @@ def get_singlets_display_names(singletsframe: pd.DataFrame) -> defaultdict[str, """ check_dataframe(singletsframe) - return singlets_value_dimension_info(singletsframe, _specs.DISPLAY_NAME) + return singlets_value_dimension_info_with_fallbacks( + singletsframe, _specs.DISPLAY_NAME_FIELDS, _specs.DISPLAY_NAME + ) def get_singlets_short_display_names(singletsframe: pd.DataFrame) -> defaultdict[str, str | None]: """Gets short display names of the value dimensions of a SingleTSFrame from metadata Falls back to the display name and then to the name of the value dimension if no short - display name is given. + display name is given. Each of them is looked up on the value dimension, on the metric (for + the value dimension "value") and in "value_dimensions_shared" before falling back to the next + one. Fields provided as null count as missing. Args: singletsframe (pd.DataFrame): STSF with metadata following the convention. @@ -791,7 +828,9 @@ def get_singlets_short_display_names(singletsframe: pd.DataFrame) -> defaultdict """ check_dataframe(singletsframe) - return singlets_value_dimension_info(singletsframe, _specs.SHORT_DISPLAY_NAME) + return singlets_value_dimension_info_with_fallbacks( + singletsframe, _specs.SHORT_DISPLAY_NAME_FIELDS, _specs.SHORT_DISPLAY_NAME + ) def get_singlets_measurements(singletsframe: pd.DataFrame) -> defaultdict[str, str | None]: diff --git a/tests/metadata/test_metadata_fallbacks.py b/tests/metadata/test_metadata_fallbacks.py new file mode 100644 index 0000000..05e7023 --- /dev/null +++ b/tests/metadata/test_metadata_fallbacks.py @@ -0,0 +1,227 @@ +"""Fallbacks for metadata following the current metadata convention + +Metadata providers like the hetida platform send fields without value explicitly as null. +These count as missing: The accessors fall back to the next level or field as if the field +was not provided at all. +""" + +from copy import deepcopy + +import pandas as pd +import pytest + +from hdhelpers.metadata import ( + get_display_names, + get_measurements, + get_metric_info, + get_names, + get_series_display_name, + get_series_measurement, + get_series_name, + get_series_short_display_name, + get_series_unit, + get_short_display_names, + get_singlets_display_names, + get_singlets_short_display_names, + get_singlets_units, + get_units, +) + +SIGNAL_ID = "7ab8c419-8745-4714-b588-57c0c772cc95" + + +def value_dimension(**fields): + """Value dimension "value" as the hetida platform sends it, with nulls for missing fields""" + return { + "column": "value", + "name": None, + "display_name": None, + "short_display_name": None, + "description": None, + "unit": None, + "measurement": None, + "value_data_type": "float", + } | fields + + +def metric(**fields): + """Metric as the hetida platform sends it, with nulls for missing fields""" + return { + "id": SIGNAL_ID, + "external_id": None, + "name": None, + "display_name": None, + "description": None, + "inherited": {}, + "comments": [], + "value_dimensions": [value_dimension()], + "default_aggregation_method": None, + } | fields + + +# what the hetida platform sends for a timeseries of a SERIES read +platform_series_attrs = { + "dataset_metadata": { + "metric_key": "id", + "single_metric": SIGNAL_ID, + "queried_metrics": [SIGNAL_ID], + "ref_interval_start_timestamp": "2026-10-18T00:00:00Z", + "ref_interval_end_timestamp": "2026-10-19T00:00:00Z", + "ref_interval_type": "closed", + }, + "metrics": [ + metric( + external_id="water.north.level", + name="Water-lvl-north", + value_dimensions=[ + value_dimension(name="Water-lvl-north", unit="m", measurement="level") + ], + default_aggregation_method="AVG", + ) + ], +} + +# what the hetida platform sends for a MULTITSFRAME read of a signal group: display_name of +# a metric is the display name configured for the member of the group +platform_multitsframe_attrs = { + "dataset_metadata": {"metric_key": "id", "queried_metrics": ["level", "flow"]}, + "metrics": [ + metric( + id="level", + name="Water-lvl-north", + value_dimensions=[value_dimension(name="Water-lvl-north", unit="m")], + ), + metric( + id="flow", + name="Pump-flow", + display_name="Flow pump 1", + value_dimensions=[value_dimension(name="Pump-flow")], + ), + ], +} + + +def series(attrs): + ser = pd.Series([1.0], index=pd.to_datetime(["2026-10-18T06:00:00Z"])) + ser.attrs = deepcopy(attrs) + return ser + + +def frame(attrs): + df = pd.DataFrame() + df.attrs = deepcopy(attrs) + return df + + +def test_series_from_platform(): + platform_series = series(platform_series_attrs) + + assert get_series_unit(platform_series) == "m" + assert get_series_measurement(platform_series) == "level" + assert get_series_name(platform_series) == "Water-lvl-north" + assert get_series_display_name(platform_series) == "Water-lvl-north" + assert get_series_short_display_name(platform_series) == "Water-lvl-north" + + +def test_multitsframe_from_platform(): + multitsframe = frame(platform_multitsframe_attrs) + + assert get_units(multitsframe)["level"]["value"] == "m" + assert get_units(multitsframe)["flow"]["value"] is None + assert get_names(multitsframe)["flow"]["value"] == "Pump-flow" + assert get_measurements(multitsframe)["level"]["value"] is None + # a null display name falls back to the name + assert get_display_names(multitsframe)["level"]["value"] == "Water-lvl-north" + assert get_short_display_names(multitsframe)["level"]["value"] == "Water-lvl-north" + # the display name of the metric wins over the name of its value dimension + assert get_display_names(multitsframe)["flow"]["value"] == "Flow pump 1" + assert get_short_display_names(multitsframe)["flow"]["value"] == "Flow pump 1" + + +def test_each_field_is_looked_up_on_all_levels_before_the_next_field(): + attrs = { + "metrics": [ + metric( + id="a", + name="metric name", + value_dimensions=[ + value_dimension(name="value name"), + value_dimension(column="state", name="state name"), + ], + ), + metric(id="b", name="metric name", display_name="metric display name"), + metric( + id="c", + display_name="metric display name", + value_dimensions=[value_dimension(display_name="value display name")], + ), + ], + "value_dimensions_shared": [{"column": "state", "display_name": "shared display name"}], + } + + display_names = get_display_names(frame(attrs)) + + assert display_names["a"]["value"] == "value name" + assert display_names["a"]["state"] == "shared display name" + assert display_names["b"]["value"] == "metric display name" + assert display_names["c"]["value"] == "value display name" + + +def test_singletsframe_display_names(): + attrs = { + "dataset_metadata": {"single_metric": "abc.temp", "metric_key": "external_id"}, + "metrics": [ + metric( + external_id="abc.temp", + name="ABC temperature", + display_name="ABC temp.", + value_dimensions=[ + value_dimension(name="temperature", unit="°C"), + value_dimension(column="state", name="measurement state"), + ], + ) + ], + } + singletsframe = frame(attrs) + + assert get_singlets_units(singletsframe)["value"] == "°C" + # only the value dimension "value" falls back to the metric + assert get_singlets_display_names(singletsframe)["value"] == "ABC temp." + assert get_singlets_display_names(singletsframe)["state"] == "measurement state" + assert get_singlets_short_display_names(singletsframe)["state"] == "measurement state" + + +def test_series_display_name_of_metric_wins_over_value_dimension_name(): + attrs = deepcopy(platform_series_attrs) + attrs["metrics"][0]["display_name"] = "Level north" + attrs["metrics"][0]["short_display_name"] = "north" + + assert get_series_display_name(series(attrs)) == "Level north" + assert get_series_short_display_name(series(attrs)) == "north" + + +def test_series_without_single_metric_uses_its_only_metric(): + attrs = deepcopy(platform_series_attrs) + del attrs["dataset_metadata"]["single_metric"] + + assert get_series_unit(series(attrs)) == "m" + assert get_series_display_name(series(attrs)) == "Water-lvl-north" + + +@pytest.mark.parametrize( + "attrs", + [{}, {"dataset_metadata": {"metric_key": "id"}}, {"metrics": None}], +) +def test_metric_info_without_metric_metadata(attrs): + """Components should not require metadata, so missing metric metadata must not raise""" + assert get_metric_info(frame(attrs), "name")["any metric"] is None + + +def test_metric_info_from_platform(): + multitsframe = frame(platform_multitsframe_attrs) + multitsframe.attrs["metrics"][0]["default_aggregation_method"] = "SUM" + + aggregation_methods = get_metric_info(multitsframe, "default_aggregation_method") + + assert aggregation_methods["level"] == "SUM" + assert aggregation_methods["flow"] is None