From 9e21d9b48783e30ef67cb13c54cd371f47fed3ef Mon Sep 17 00:00:00 2001 From: Matt Fulp <8397318+fulpm@users.noreply.github.com> Date: Wed, 8 Jul 2026 17:54:18 -0400 Subject: [PATCH 1/8] lint fix --- howso/utilities/feature_attributes/base.py | 306 +++++++++--------- .../infer_feature_attributes.py | 6 +- .../feature_attributes/suggestions.py | 3 +- 3 files changed, 152 insertions(+), 163 deletions(-) diff --git a/howso/utilities/feature_attributes/base.py b/howso/utilities/feature_attributes/base.py index c962438d..c2dd5deb 100644 --- a/howso/utilities/feature_attributes/base.py +++ b/howso/utilities/feature_attributes/base.py @@ -11,13 +11,13 @@ from pathlib import Path import platform import typing as t +from typing import Self import warnings from zoneinfo import ZoneInfo from dateutil.parser import isoparse, parse as dt_parse import numpy as np import pandas as pd -from typing_extensions import Self import yaml from howso.utilities.fanout_features import infer_fanout_feature_config @@ -208,7 +208,7 @@ def from_json( ) if json_path: - with open(json_path, mode="r") as fp: + with open(json_path) as fp: obj_dict = json.load(fp, object_pairs_hook=feature_attributes_pairs_hook) else: obj_dict = json.loads(json_str or "", object_pairs_hook=feature_attributes_pairs_hook) @@ -233,11 +233,11 @@ def to_dataframe(self, *, include_all: bool = False) -> pd.DataFrame: pandas.DataFrame A DataFrame representation of the inferred feature attributes. """ - raise NotImplementedError('Function not yet implemented for all subclasses of `FeatureAttributesBase`') + raise NotImplementedError("Function not yet implemented for all subclasses of `FeatureAttributesBase`") - def get_names(self, *, types: t.Optional[str | Container] = None, - data_types: t.Optional[str | Container] = None, - without: t.Optional[str | Iterable[str]] = None, + def get_names(self, *, types: str | Container | None = None, + data_types: str | Container | None = None, + without: str | Iterable[str] | None = None, ) -> list[str]: """ Get feature names associated with this FeatureAttributes object. @@ -263,24 +263,24 @@ def get_names(self, *, types: t.Optional[str | Container] = None, if without: for feature in without: if feature not in self.keys(): - raise ValueError(f'Feature {feature} does not exist in this FeatureAttributes ' - 'object') + raise ValueError(f"Feature {feature} does not exist in this FeatureAttributes " + "object") names = self.keys() if types: if isinstance(types, str): - types = [types, ] + types = [types ] else: types = [] if data_types: if isinstance(data_types, str): - data_types = [data_types, ] + data_types = [data_types ] else: data_types = [] names = [ name for name in names - if (self[name].get('type') in types or not types) - and (self[name].get('data_type') in data_types or not data_types) + if (self[name].get("type") in types or not types) + and (self[name].get("data_type") in data_types or not data_types) ] return [ @@ -288,11 +288,11 @@ def get_names(self, *, types: t.Optional[str | Container] = None, if without is None or key not in without ] - def _validate_bounds(self, data: pd.DataFrame, feature: str, # noqa: C901 + def _validate_bounds(self, data: pd.DataFrame, feature: str, attributes: FeatureAttributes) -> list[str]: """Validate the feature bounds of the provided DataFrame.""" # Import here to avoid circular import - from howso.utilities import date_to_epoch + from howso.utilities import date_to_epoch # noqa: PLC0415 errors = [] @@ -302,29 +302,29 @@ def _validate_bounds(self, data: pd.DataFrame, feature: str, # noqa: C901 # Gather some data to use for validation series = data[feature] - bounds = attributes['bounds'] # pyright: ignore[reportTypedDictNotRequiredAccess] - min_bound = bounds.get('min') - max_bound = bounds.get('max') + bounds = attributes["bounds"] # pyright: ignore[reportTypedDictNotRequiredAccess] + min_bound = bounds.get("min") + max_bound = bounds.get("max") # Get unique values but exclude NoneTypes unique_values = series.dropna().unique() additional_errors = 0 - if bounds.get('allowed'): + if bounds.get("allowed"): # Check nominal bounds - allowed_values = attributes['bounds']['allowed'] # pyright: ignore[reportTypedDictNotRequiredAccess] + allowed_values = attributes["bounds"]["allowed"] # pyright: ignore[reportTypedDictNotRequiredAccess] out_of_band_values = set(unique_values) - set(allowed_values) if pd.isna(list(out_of_band_values)).all(): # Placeholder for behavior when columns contain nans pass elif out_of_band_values: errors.append(f"'{feature}' contains out-of-band values: {out_of_band_values}") - elif attributes.get('date_time_format'): + elif attributes.get("date_time_format"): # Time-only attributes have bounds represented in seconds - if attributes.get('original_type', {}).get('data_type') == 'time': + if attributes.get("original_type", {}).get("data_type") == "time": unique_time_values = pd.to_datetime( series, - format=attributes['date_time_format'], # pyright: ignore[reportTypedDictNotRequiredAccess] - errors='coerce' + format=attributes["date_time_format"], # pyright: ignore[reportTypedDictNotRequiredAccess] + errors="coerce" ).dropna().unique() for value in unique_time_values: value_in_seconds = time_to_seconds(value.time()) @@ -340,11 +340,11 @@ def _validate_bounds(self, data: pd.DataFrame, feature: str, # noqa: C901 else: try: if min_bound: - min_bound_epoch = date_to_epoch(min_bound, time_format=attributes['date_time_format']) + min_bound_epoch = date_to_epoch(min_bound, time_format=attributes["date_time_format"]) if max_bound: - max_bound_epoch = date_to_epoch(max_bound, time_format=attributes['date_time_format']) + max_bound_epoch = date_to_epoch(max_bound, time_format=attributes["date_time_format"]) for value in unique_values: - epoch = date_to_epoch(value, time_format=attributes['date_time_format']) + epoch = date_to_epoch(value, time_format=attributes["date_time_format"]) if (max_bound and epoch > max_bound_epoch) or (min_bound and epoch < min_bound_epoch): if len(errors) < 5: errors.append( @@ -354,7 +354,7 @@ def _validate_bounds(self, data: pd.DataFrame, feature: str, # noqa: C901 else: additional_errors += 1 except ValueError as err: - errors.append(f'Could not validate datetime bounds due to the following error: {err}') + errors.append(f"Could not validate datetime bounds due to the following error: {err}") elif min_bound or max_bound: # Check int/float bounds for value in unique_values: @@ -371,7 +371,7 @@ def _validate_bounds(self, data: pd.DataFrame, feature: str, # noqa: C901 f'"{feature}" had {additional_errors} additional values outside of bounds that were not displayed.') return errors - def _validate_dtype(self, data: pd.DataFrame, feature: str, # noqa: C901 + def _validate_dtype(self, data: pd.DataFrame, feature: str, expected_dtype: str | pd.CategoricalDtype, coerced_df: pd.DataFrame, coerce: bool = False, localize_datetimes: bool = True) -> list[str]: """Validate the data type of a feature and optionally attempt to coerce.""" @@ -389,40 +389,38 @@ def _validate_dtype(self, data: pd.DataFrame, feature: str, # noqa: C901 is_valid = True except Exception: # noqa: Intentionally broad pass - elif expected_dtype == 'datetime64': + elif expected_dtype == "datetime64": try: - format = self[feature]['date_time_format'] # pyright: ignore[reportTypedDictNotRequiredAccess] + format = self[feature]["date_time_format"] # pyright: ignore[reportTypedDictNotRequiredAccess] if ".%f" in format: format = "ISO8601" series = pd.to_datetime(coerced_df[feature], format=format) if coerce: if localize_datetimes and not isinstance(series, pd.DatetimeTZDtype): coerced_df[feature] = series.dt.tz_localize( - 'UTC', ambiguous='infer', nonexistent='NaT' + "UTC", ambiguous="infer", nonexistent="NaT" ) else: coerced_df[feature] = series is_valid = True except Exception: # noqa: Intentionally broad pass + # Else, compare the dtype directly + elif data[feature].dtype.name == expected_dtype: + is_valid = True + # If the feature can be converted, consider it valid (slightly differing numeric types, etc.) else: - # Else, compare the dtype directly - if data[feature].dtype.name == expected_dtype: + try: + series = series.astype(expected_dtype) + if coerce: + coerced_df[feature] = series is_valid = True - # If the feature can be converted, consider it valid (slightly differing numeric types, etc.) - else: - try: - series = series.astype(expected_dtype) - if coerce: - coerced_df[feature] = series + except pd.errors.IntCastingNaNError: + # If this happens, there is a null value, thus a float dtype is OK + if pd.api.types.is_float_dtype(series): is_valid = True - except pd.errors.IntCastingNaNError: - # If this happens, there is a null value, thus a float dtype is OK - if pd.api.types.is_float_dtype(series): - is_valid = True - except Exception as err: # noqa: Intentionally broad - coerce_err = str(err) - pass + except Exception as err: # noqa: Intentionally broad + coerce_err = str(err) # Raise warnings if the types do not match if not is_valid: @@ -439,11 +437,11 @@ def _validate_dtype(self, data: pd.DataFrame, feature: str, # noqa: C901 @staticmethod def _allows_null(attributes: FeatureAttributes) -> bool: """Return whether the given attributes indicates the allowance of null values.""" - return 'bounds' in attributes and attributes['bounds'].get('allow_null', False) + return "bounds" in attributes and attributes["bounds"].get("allow_null", False) - def _validate_df(self, data: pd.DataFrame, coerce: bool = False, # noqa: C901 - raise_errors: bool = False, table_name: t.Optional[str] = None, validate_bounds=True, - allow_missing_features: bool = False, localize_datetimes=True, nullable_int_dtype='Int64'): + def _validate_df(self, data: pd.DataFrame, coerce: bool = False, + raise_errors: bool = False, table_name: str | None = None, validate_bounds=True, + allow_missing_features: bool = False, localize_datetimes=True, nullable_int_dtype="Int64"): errors = [] coerced_df = data.copy(deep=True) features = t.cast(dict[str, "FeatureAttributes"], self[table_name] if table_name else self) @@ -452,23 +450,23 @@ def _validate_df(self, data: pd.DataFrame, coerce: bool = False, # noqa: C901 if feature not in data.columns: # Check if column is missing (and not supposed to be) if not ( - feature.startswith('.') + feature.startswith(".") or ( - attributes.get('auto_derive_on_train', False) - and 'derived_feature_code' in attributes + attributes.get("auto_derive_on_train", False) + and "derived_feature_code" in attributes ) or allow_missing_features ): - errors.append(f'{feature} is missing from the dataframe') + errors.append(f"{feature} is missing from the dataframe") # OK if it's an internal feature or is being processed by Validator continue # Check nominal types - if attributes['type'] == 'nominal': - if attributes.get('data_type') == 'number': + if attributes["type"] == "nominal": + if attributes.get("data_type") == "number": # Check type (float) - if attributes.get('decimal_places', 0) > 0: - errors.extend(self._validate_dtype(data, feature, 'float64', + if attributes.get("decimal_places", 0) > 0: + errors.extend(self._validate_dtype(data, feature, "float64", coerced_df, coerce=coerce)) # Check type (nullable Int) elif self._allows_null(attributes): @@ -476,34 +474,34 @@ def _validate_df(self, data: pd.DataFrame, coerce: bool = False, # noqa: C901 coerced_df, coerce=coerce)) # Check type (int) else: - errors.extend(self._validate_dtype(data, feature, 'int64', + errors.extend(self._validate_dtype(data, feature, "int64", coerced_df, coerce=coerce)) - elif attributes.get('data_type') == 'boolean': + elif attributes.get("data_type") == "boolean": # Check type (boolean) - errors.extend(self._validate_dtype(data, feature, 'bool', + errors.extend(self._validate_dtype(data, feature, "bool", coerced_df, coerce=coerce)) - elif attributes.get('bounds') and attributes['bounds'].get('allowed'): # pyright: ignore[reportTypedDictNotRequiredAccess] + elif attributes.get("bounds") and attributes["bounds"].get("allowed"): # pyright: ignore[reportTypedDictNotRequiredAccess] # Check type (categorical) - schema_dtype = pd.CategoricalDtype(attributes['bounds']['allowed'], # pyright: ignore[reportTypedDictNotRequiredAccess] + schema_dtype = pd.CategoricalDtype(attributes["bounds"]["allowed"], # pyright: ignore[reportTypedDictNotRequiredAccess] ordered=True) errors.extend(self._validate_dtype(data, feature, schema_dtype, coerced_df, coerce=coerce)) else: # Else, should be an object - errors.extend(self._validate_dtype(data, feature, 'object', + errors.extend(self._validate_dtype(data, feature, "object", coerced_df, coerce=coerce)) # Check ordinal types - elif attributes['type'] == 'ordinal': - if attributes.get('bounds') and attributes['bounds'].get('allowed'): # pyright: ignore[reportTypedDictNotRequiredAccess] + elif attributes["type"] == "ordinal": + if attributes.get("bounds") and attributes["bounds"].get("allowed"): # pyright: ignore[reportTypedDictNotRequiredAccess] # Check type (categorical) - schema_dtype = pd.CategoricalDtype(attributes['bounds']['allowed'], # pyright: ignore[reportTypedDictNotRequiredAccess] + schema_dtype = pd.CategoricalDtype(attributes["bounds"]["allowed"], # pyright: ignore[reportTypedDictNotRequiredAccess] ordered=True) errors.extend(self._validate_dtype(data, feature, schema_dtype, coerced_df, coerce=coerce)) # Check type (float) - elif attributes.get('decimal_places', 0) > 0: - errors.extend(self._validate_dtype(data, feature, 'float64', + elif attributes.get("decimal_places", 0) > 0: + errors.extend(self._validate_dtype(data, feature, "float64", coerced_df, coerce=coerce)) # Check type (nullable Int) elif self._allows_null(attributes): @@ -511,55 +509,53 @@ def _validate_df(self, data: pd.DataFrame, coerce: bool = False, # noqa: C901 coerced_df, coerce=coerce)) # Check type (int) else: - errors.extend(self._validate_dtype(data, feature, 'int64', + errors.extend(self._validate_dtype(data, feature, "int64", coerced_df, coerce=coerce)) # Check continuous types - else: - if 'date_time_format' in attributes: - # Check type (datetime) - errors.extend(self._validate_dtype(data, feature, 'datetime64', - coerced_df, coerce=coerce, - localize_datetimes=localize_datetimes)) - - # Check semi-structured type (object) - elif attributes.get("data_type") in {"json", "yaml", "amalgam", "string", "string_mixable"}: - errors.extend(self._validate_dtype(data, feature, "object", coerced_df, coerce=coerce)) - - # Check type (float) - elif attributes.get('decimal_places', -1) > 0: - errors.extend(self._validate_dtype(data, feature, 'float64', - coerced_df, coerce=coerce)) - # Check type (nullable Int) - elif self._allows_null(attributes): - errors.extend(self._validate_dtype(data, feature, nullable_int_dtype, - coerced_df, coerce=coerce)) - # Check type (int) - elif attributes.get('decimal_places', -1) == 0: - errors.extend(self._validate_dtype(data, feature, 'int64', - coerced_df, coerce=coerce)) - elif attributes.get('data_type') == 'number': - # If feature is continuous and not a datetime, it should have a numeric data_type. - # If it cannot be casted to a float, then add an error. - if len(self._validate_dtype(data, feature, 'float64', - coerced_df, coerce=True)): - errors.extend([f"Feature '{feature}' should be numeric" - " when 'type' is 'continuous' and " - "'data_type' is 'number'."]) + elif "date_time_format" in attributes: + # Check type (datetime) + errors.extend(self._validate_dtype(data, feature, "datetime64", + coerced_df, coerce=coerce, + localize_datetimes=localize_datetimes)) + + # Check semi-structured type (object) + elif attributes.get("data_type") in {"json", "yaml", "amalgam", "string", "string_mixable"}: + errors.extend(self._validate_dtype(data, feature, "object", coerced_df, coerce=coerce)) + + # Check type (float) + elif attributes.get("decimal_places", -1) > 0: + errors.extend(self._validate_dtype(data, feature, "float64", + coerced_df, coerce=coerce)) + # Check type (nullable Int) + elif self._allows_null(attributes): + errors.extend(self._validate_dtype(data, feature, nullable_int_dtype, + coerced_df, coerce=coerce)) + # Check type (int) + elif attributes.get("decimal_places", -1) == 0: + errors.extend(self._validate_dtype(data, feature, "int64", + coerced_df, coerce=coerce)) + elif attributes.get("data_type") == "number": + # If feature is continuous and not a datetime, it should have a numeric data_type. + # If it cannot be casted to a float, then add an error. + if len(self._validate_dtype(data, feature, "float64", + coerced_df, coerce=True)): + errors.extend([f"Feature '{feature}' should be numeric" + " when 'type' is 'continuous' and " + "'data_type' is 'number'."]) # Check feature bounds if validate_bounds: errors.extend(self._validate_bounds(data, feature, attributes)) if errors: - msg = ('Failed to validate DataFrame against feature attributes due to the ' - 'following errors:\n') + msg = ("Failed to validate DataFrame against feature attributes due to the " + "following errors:\n") for error in errors: - msg = msg + f'{error}\n' + msg = msg + f"{error}\n" if raise_errors: raise ValueError(msg) - else: - warnings.warn(msg) + warnings.warn(msg) if coerce: return coerced_df @@ -595,7 +591,7 @@ def validate(self, data: t.Any, coerce: bool = False, raise_errors: bool = False None | DataFrame None or the coerced DataFrame if 'coerce' is True and there were no errors. """ - raise NotImplementedError() + raise NotImplementedError @staticmethod def merge(attributes: dict[str, dict], entries: dict[str, dict]) -> FeatureAttributesBase: @@ -632,18 +628,18 @@ def merge(attributes: dict[str, dict], entries: dict[str, dict]) -> FeatureAttri validate_features(entries) # Compare to existing attributes for feature_name in entries.keys(): - orig_type = attributes.get(feature_name, {}).get('type') - new_type = entries[feature_name].get('type') + orig_type = attributes.get(feature_name, {}).get("type") + new_type = entries[feature_name].get("type") # TODO 22059: Allow ordinals here when we can attempt to infer values - if new_type == 'ordinal' and not ( - attributes.get(feature_name, {}).get('bounds', {}).get('allowed') or - entries.get(feature_name, {}).get('bounds', {}).get('allowed') + if new_type == "ordinal" and not ( + attributes.get(feature_name, {}).get("bounds", {}).get("allowed") or + entries.get(feature_name, {}).get("bounds", {}).get("allowed") ): - raise ValueError('Inference of ordinal values is not yet supported. Please ' - 'preset ordinal features with their ordered values using ' - '`ordinal_feature_values`.') + raise ValueError("Inference of ordinal values is not yet supported. Please " + "preset ordinal features with their ordered values using " + "`ordinal_feature_values`.") # Sanity check: booleans must be nominal - elif entries[feature_name].get('data_type') == 'boolean' and orig_type and orig_type != 'nominal': + if entries[feature_name].get("data_type") == "boolean" and orig_type and orig_type != "nominal": warnings.warn( f'Feature "{feature_name}" was preset as {orig_type} ' 'but was detected to be a boolean. Booleans ' @@ -651,7 +647,7 @@ def merge(attributes: dict[str, dict], entries: dict[str, dict]) -> FeatureAttri ) # In otherwise valid cases, ensure that existing types are not overwritten elif orig_type and new_type: - del entries[feature_name]['type'] + del entries[feature_name]["type"] # Finally, update the dict with all remaining attributes if feature_name not in attributes.keys(): attributes[feature_name] = entries[feature_name] @@ -664,8 +660,6 @@ def merge(attributes: dict[str, dict], entries: dict[str, dict]) -> FeatureAttri class MultiTableFeatureAttributes(FeatureAttributesBase): """A dict-like object containing feature attributes for multiple tables.""" - pass - class SingleTableFeatureAttributes(FeatureAttributesBase): """A dict-like object containing feature attributes for a single table or DataFrame.""" @@ -707,7 +701,7 @@ def validate(data: t.Any, **kwargs): @validate.register def _(self, data: pd.DataFrame, coerce=False, raise_errors=False, validate_bounds=True, allow_missing_features=False, localize_datetimes=True, - nullable_int_dtype: str | np.dtype | pd.api.extensions.ExtensionDtype = 'Int64'): + nullable_int_dtype: str | np.dtype | pd.api.extensions.ExtensionDtype = "Int64"): return self._validate_df(data, coerce=coerce, raise_errors=raise_errors, validate_bounds=validate_bounds, allow_missing_features=allow_missing_features, @@ -742,7 +736,7 @@ def to_dataframe(self, *, include_all: bool = False) -> pd.DataFrame: pandas.DataFrame A DataFrame representation of the inferred feature attributes. """ - sep = '|' + sep = "|" key_order = [ "sample", "type", @@ -773,7 +767,7 @@ def to_dataframe(self, *, include_all: bool = False) -> pd.DataFrame: df = pd.json_normalize(attributes, sep=sep) # Update the column names to create a MultiIndex df.columns = pd.MultiIndex.from_tuples([ - tuple(c.split(sep)) if sep in c else (c, '') + tuple(c.split(sep)) if sep in c else (c, "") for c in df.columns ]) # Set the outer key (e.g., 'f0') as the index @@ -1091,8 +1085,7 @@ def _process(self, suggestion = f"Please verify that cases in '{feature_name}' are of a consistent data type." raise ValueError(f"The following error was raised while trying to compute bounds for feature " f"'{feature_name}':\n\n {err}\n\n{suggestion}") from err - else: - raise + raise if bounds: # Use `update` on the bounds dictionary in case `allowed` ordinal values have already been set bounds.update(self.attributes[feature_name].get("bounds", {})) @@ -1235,14 +1228,13 @@ def _infer_string_attributes(self, feature_name: str) -> dict: "data_type": "formatted_date_time", "date_time_format": fmt } - else: - # It isn't clear how this method would be called on a feature - # if it has no data, but just in case... - return { - "type": "continuous", - "data_type": "number", - } - elif self._is_json_feature(feature_name): + # It isn't clear how this method would be called on a feature + # if it has no data, but just in case... + return { + "type": "continuous", + "data_type": "number", + } + if self._is_json_feature(feature_name): typing_attrs = { "type": "continuous", "data_type": "json", @@ -1253,34 +1245,32 @@ def _infer_string_attributes(self, feature_name: str) -> dict: if isinstance(first_non_null, Set): typing_attrs["original_type"]["coercion"] = "set" return typing_attrs - elif self._is_yaml_feature(feature_name): + if self._is_yaml_feature(feature_name): return { "type": "continuous", "data_type": "yaml" } - else: - # The user may have pre-set the type as "continuous" to force it to be considered a tokenizable string; - # but that may also be the case for string ints or floats. Check that first. - is_tokenizable_string = False - if self.attributes.get(feature_name, {}).get("type") == "continuous": - try: - # If the column can be converted to float, and was set to be "continuous", - # it is probably not a tokenizable string. - col = self.data[feature_name] - col.astype("float") - except Exception: # noqa: Intentionally broad - # If it cannot be converted to float, but it was set to be "continuous", - # it is probably a tokenizable string. - is_tokenizable_string = True - if is_tokenizable_string: - return { - "type": "continuous", - "data_type": "json", - # Also set the original_type here so that we do not need to re-check _is_tokenizable_string - "original_type": {"data_type": FeatureType.TOKENIZABLE_STRING.value}, - } - else: - return self._infer_unknown_attributes(feature_name) + # The user may have pre-set the type as "continuous" to force it to be considered a tokenizable string; + # but that may also be the case for string ints or floats. Check that first. + is_tokenizable_string = False + if self.attributes.get(feature_name, {}).get("type") == "continuous": + try: + # If the column can be converted to float, and was set to be "continuous", + # it is probably not a tokenizable string. + col = self.data[feature_name] + col.astype("float") + except Exception: # noqa: Intentionally broad + # If it cannot be converted to float, but it was set to be "continuous", + # it is probably a tokenizable string. + is_tokenizable_string = True + if is_tokenizable_string: + return { + "type": "continuous", + "data_type": "json", + # Also set the original_type here so that we do not need to re-check _is_tokenizable_string + "original_type": {"data_type": FeatureType.TOKENIZABLE_STRING.value}, + } + return self._infer_unknown_attributes(feature_name) def _infer_unknown_attributes(self, *args: t.Any) -> dict: """Get inferred attributes for the given unknown-type column.""" @@ -1478,7 +1468,7 @@ def _validate_date_times(self) -> None: f'The feature "{feature_name}" must have a `date_time_format` defined ' f'when its `data_type` is "{data_type}".' ) - elif dt_format and data_type in {"formatted_date_time", "formatted_time"}: + if dt_format and data_type in {"formatted_date_time", "formatted_time"}: # If the date/time format does not include a time zone, warn the user that # the default of UTC will be used. However, due to potential multiprocessing, # and to avoid an excess of warnings if done per-feature, stash the offending @@ -1492,7 +1482,7 @@ def _validate_date_times(self) -> None: rand_val = self._get_random_value(feature_name) if isinstance(rand_val, datetime.datetime): # Some datetime objects might have a time zone attribute not visible as a string - if getattr(rand_val, 'tzinfo', None) is not None and isinstance(rand_val.tzinfo, ZoneInfo): + if getattr(rand_val, "tzinfo", None) is not None and isinstance(rand_val.tzinfo, ZoneInfo): continue # Warn in case of UTC offset -- could lead to unexpected results due to time zone # differences diff --git a/howso/utilities/feature_attributes/infer_feature_attributes.py b/howso/utilities/feature_attributes/infer_feature_attributes.py index 1dceb177..e39f2fb0 100644 --- a/howso/utilities/feature_attributes/infer_feature_attributes.py +++ b/howso/utilities/feature_attributes/infer_feature_attributes.py @@ -401,8 +401,8 @@ def infer_feature_attributes(data: pd.DataFrame | IFACompatibleADCProtocol | SQL elif isinstance(data, IFACompatibleADCProtocol): infer = IFATimeSeriesADC(data, time_feature_name) else: - raise NotImplementedError('`infer_feature_attributes` for time series only supported for DataFrames and ' - 'AbstractData classes.') + raise NotImplementedError("`infer_feature_attributes` for time series only supported for DataFrames and " + "AbstractData classes.") elif time_feature_name: raise ValueError("'time_feature_name' was included, but 'data' must be of type DataFrame " "for time series feature attributes to be calculated.") @@ -417,6 +417,6 @@ def infer_feature_attributes(data: pd.DataFrame | IFACompatibleADCProtocol | SQL elif isinstance(data, IFACompatibleADCProtocol): infer = InferFeatureAttributesAbstractData(data) else: - raise NotImplementedError('Data not recognized as a DataFrame, AbstractData class, or compatible datastore.') + raise NotImplementedError("Data not recognized as a DataFrame, AbstractData class, or compatible datastore.") return infer(**kwargs) diff --git a/howso/utilities/feature_attributes/suggestions.py b/howso/utilities/feature_attributes/suggestions.py index f87bce10..9605ee15 100644 --- a/howso/utilities/feature_attributes/suggestions.py +++ b/howso/utilities/feature_attributes/suggestions.py @@ -1,12 +1,11 @@ from abc import ABC, abstractmethod from collections.abc import Mapping, Sequence import textwrap -from typing import Any, cast +from typing import Any, cast, Self import warnings from rich.console import Console from rich.table import Table -from typing import Self # Fanout features parameters # -------------------------- From 00f94ee8c7d1b51ded23cd3a01bf4fba969152cb Mon Sep 17 00:00:00 2001 From: Matt Fulp <8397318+fulpm@users.noreply.github.com> Date: Wed, 8 Jul 2026 18:24:52 -0400 Subject: [PATCH 2/8] misc fixes and docs --- howso/utilities/feature_attributes/base.py | 12 +++------ .../infer_feature_attributes.py | 27 ++++++++++++------- .../feature_attributes/suggestions.py | 2 ++ 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/howso/utilities/feature_attributes/base.py b/howso/utilities/feature_attributes/base.py index c2dd5deb..970a9371 100644 --- a/howso/utilities/feature_attributes/base.py +++ b/howso/utilities/feature_attributes/base.py @@ -11,7 +11,6 @@ from pathlib import Path import platform import typing as t -from typing import Self import warnings from zoneinfo import ZoneInfo @@ -56,9 +55,6 @@ LINUX_DT_MAX = "2262-04-11" WIN_DT_MAX = "6053-01-24" -# Define a TypeVar which is FeatureAttributesBase or any subclass. -FeatureAttributesBaseType = t.TypeVar("FeatureAttributesBaseType", bound="FeatureAttributesBase") - SIGNIFICANT_THRESHOLD_DEFAULT: int = 30 @@ -87,7 +83,7 @@ def __init__(self, feature_attributes: Mapping, params: dict | None = None, unsu self.update(feature_attributes) self.unsupported = unsupported or [] self.warnings_collector = IFAWarningCollector() - self.suggestions_collector = suggestions_collector or "You have no suggestions." + self.suggestions_collector = suggestions_collector or IFASuggestionCollector() def __copy__(self) -> FeatureAttributesBase: """Return a (deep)copy of this instance of FeatureAttributesBase.""" @@ -175,11 +171,11 @@ def to_json(self, archive: bool = False, json_path: Path | None = None) -> str: @classmethod def from_json( - cls: Self, + cls: type[FeatureAttributesBase], json_str: str | None = None, *, json_path: str | None = None - ) -> Self: + ) -> FeatureAttributesBase: """ Reconstruct a FeatureAttributesBase from JSON. @@ -192,7 +188,7 @@ def from_json( Returns ------- - FeatureAttributesBaseType + FeatureAttributesBase An instance of FeatureAttributesBase or any of its subclasses. """ if json_path and json_str: diff --git a/howso/utilities/feature_attributes/infer_feature_attributes.py b/howso/utilities/feature_attributes/infer_feature_attributes.py index e39f2fb0..01b7edf4 100644 --- a/howso/utilities/feature_attributes/infer_feature_attributes.py +++ b/howso/utilities/feature_attributes/infer_feature_attributes.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections.abc import Iterable -import typing as t +from typing import Any import pandas as pd @@ -17,16 +17,22 @@ from howso.utilities.feature_attributes.time_series import IFATimeSeriesADC, IFATimeSeriesPandas -def infer_feature_attributes(data: pd.DataFrame | IFACompatibleADCProtocol | SQLRelationalDatastoreProtocol, *, - tables: t.Optional[Iterable[TableNameProtocol]] = None, - time_feature_name: t.Optional[str] = None, - **kwargs - ) -> FeatureAttributesBase: +def infer_feature_attributes( + data: pd.DataFrame | IFACompatibleADCProtocol | SQLRelationalDatastoreProtocol, + *, + tables: Iterable[TableNameProtocol] | None = None, + time_feature_name: str | None = None, + **kwargs: Any, +) -> FeatureAttributesBase: """ - Return a dict-like feature attributes object with useful accessor methods. + Infer the feature attributes of a given data source. - The returned object is a subclass of FeatureAttributesBase that is appropriate for the - provided data type. + Feature attributes provide Howso with a schema like representation of the data and is required in order to + ingest the data into Howso. It is important to inspect the result of this method in order to verify the accuracy + of what is inferred and so that any inaccuracies can be corrected before data ingestion. This method makes a best + guess at the feature attributes and in some cases may not automatically set certain attributes if there is + ambiguity, instead raising suggestions for further consideration which can be reviewed by accessing the + :attr:`~FeatureAttributesBase.suggestions` attribute. Parameters ---------- @@ -347,7 +353,8 @@ def infer_feature_attributes(data: pd.DataFrame | IFACompatibleADCProtocol | SQL FeatureAttributesBase A subclass of ``FeatureAttributesBase`` (Single/MultiTableFeatureAttributes) that extends ``dict``, thus providing dict-like access to feature - attributes and useful accessor methods. + attributes and useful accessor methods. The subclass variant is dependent + on the provided data source. Examples -------- diff --git a/howso/utilities/feature_attributes/suggestions.py b/howso/utilities/feature_attributes/suggestions.py index 9605ee15..ba364dfa 100644 --- a/howso/utilities/feature_attributes/suggestions.py +++ b/howso/utilities/feature_attributes/suggestions.py @@ -401,6 +401,8 @@ def __getattr__(self, key: str) -> IFASuggestion: def __repr__(self) -> str: """Print a helpful description of the available suggestions.""" + if not self._suggestions: + return "You have no suggestions." table = Table(title="Suggestions for Potential Data Quality Improvements", caption="To view a more detailed description of a suggestion, access its `name` as a property " "(e.g., `your_attributes_object.suggestions.preserve_rare_values`).\n\nTo apply all suggestions," From 6c18458cf001b93cbcc533a865f71dd5bf7f4a41 Mon Sep 17 00:00:00 2001 From: Matt Fulp <8397318+fulpm@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:42:52 -0400 Subject: [PATCH 3/8] type hint ifa --- howso/utilities/feature_attributes/base.py | 2 +- .../infer_feature_attributes.py | 111 ++++++++++++++---- .../feature_attributes/suggestions.py | 8 +- .../feature_attributes/time_series.py | 2 +- 4 files changed, 93 insertions(+), 30 deletions(-) diff --git a/howso/utilities/feature_attributes/base.py b/howso/utilities/feature_attributes/base.py index 970a9371..326cb429 100644 --- a/howso/utilities/feature_attributes/base.py +++ b/howso/utilities/feature_attributes/base.py @@ -819,7 +819,7 @@ def _process(self, mode_bound_features: Iterable[str] | None = None, num_series: int = 1, nominal_substitution_config: dict[str, dict] | None = None, - ordinal_feature_values: dict[str, list[str]] | None = None, + ordinal_feature_values: dict[str, list[t.Any]] | None = None, preserve_rare_values_map: PreserveRareValuesMap | t.Literal["all", "off"] | None = None, preserve_rare_values_config: PreserveRareValuesConfig | FullPreserveRareValuesConfig | None = None, significance_threshold: int = SIGNIFICANT_THRESHOLD_DEFAULT, diff --git a/howso/utilities/feature_attributes/infer_feature_attributes.py b/howso/utilities/feature_attributes/infer_feature_attributes.py index 01b7edf4..9d2a7106 100644 --- a/howso/utilities/feature_attributes/infer_feature_attributes.py +++ b/howso/utilities/feature_attributes/infer_feature_attributes.py @@ -1,7 +1,7 @@ from __future__ import annotations -from collections.abc import Iterable -from typing import Any +from collections.abc import Iterable, Sequence +from typing import Any, Literal, overload, TypeAlias, TypedDict, Unpack import pandas as pd @@ -16,6 +16,67 @@ from howso.utilities.feature_attributes.relational import InferFeatureAttributesSQLDatastore from howso.utilities.feature_attributes.time_series import IFATimeSeriesADC, IFATimeSeriesPandas +FeatureType: TypeAlias = Literal["continuous", "ordinal", "nominal"] + + +class InferOptions(TypedDict, total=False): + """Infer feature attributes options.""" + + attempt_infer_extended_nominals: bool + datetime_feature_formats: dict[str, str | tuple[str, str]] + default_time_zone: str | None + dependent_features: dict[str, list[str]] + enable_suggestions: bool + fanout_feature_map: dict[tuple[str, ...] | str, list[str]] + id_feature_name: str | Sequence[str] | None + include_extended_nominal_probabilities: bool + include_sample: bool + infer_bounds: bool + max_distilled_cases: int | None + max_rows_to_eval: int + max_workers: int | None + memory_warning_threshold: int | None + mode_bound_features: Iterable[str] + nominal_substitution_config: dict[str, dict[str, Any]] + ordinal_feature_values: dict[str, list[Any] | tuple[str]] + tight_bounds: Iterable[str] + types: dict[str, FeatureType] | dict[FeatureType, list[str]] + + +class InferTimeSeriesOptions(InferOptions, total=False): + """Infer feature attributes options for time-series data.""" + + delta_boundaries: dict[str, dict[Literal["min", "max"], dict[int | str, Any]]] + derived_orders: dict[str, int] + lags: dict[str, int | list[int]] | list[int] + num_lags: int | dict[str, int] + orders_of_derivatives: dict[str, int] + rate_boundaries: dict[str, dict[Literal["min", "max"], dict[int | str, Any]]] + time_feature_is_universal: bool | None + time_invariant_features: Iterable[str] + time_series_type_default: Literal["rate", "delta", "covariate"] + time_series_types_override: dict[str, Literal["rate", "delta", "covariate"]] + + +@overload +def infer_feature_attributes( + data: pd.DataFrame | IFACompatibleADCProtocol | SQLRelationalDatastoreProtocol, + *, + tables: Iterable[TableNameProtocol] | None = None, + time_feature_name: str, + **kwargs: Unpack[InferTimeSeriesOptions], +) -> FeatureAttributesBase: ... + + +@overload +def infer_feature_attributes( + data: pd.DataFrame | IFACompatibleADCProtocol | SQLRelationalDatastoreProtocol, + *, + tables: Iterable[TableNameProtocol] | None = None, + time_feature_name: None = None, + **kwargs: Unpack[InferOptions], +) -> FeatureAttributesBase: ... + def infer_feature_attributes( data: pd.DataFrame | IFACompatibleADCProtocol | SQLRelationalDatastoreProtocol, @@ -48,7 +109,7 @@ def infer_feature_attributes( Please refer to ``kwargs`` for other parameters related to extended nominals. - datetime_feature_formats : dict, default None + datetime_feature_formats : dict, optional (Optional) Dict defining custom (non-ISO8601) datetime or time-only formats. By default, datetime features are assumed to be in ISO8601 format. Non-English datetimes must have locales specified. If locale is omitted, the default system locale is used. @@ -69,7 +130,7 @@ def infer_feature_attributes( ``datetime_feature_formats`` and it is not inferred from the data. If not specified anywhere, the Howso Engine will default to UTC. - delta_boundaries : dict, default None + delta_boundaries : dict, optional (Optional) For time series, specify the delta boundaries in the form {"feature" : {"min|max" : {order : value}}}. Works with partial values by specifying only particular order of derivatives you would like to @@ -122,7 +183,7 @@ def infer_feature_attributes( "measurement_amount": [ "measurement" ] } - derived_orders : dict, default None + derived_orders : dict, optional (Optional) Dict of features to the number of orders of derivatives that should be derived instead of synthesized. For example, for a feature with a 3rd order of derivative, setting its derived_orders @@ -149,7 +210,7 @@ def infer_feature_attributes( (Optional) If True, bounds will be inferred for the features if the feature column has at least one non NaN value - lags : list or dict, default None + lags : list or dict, optional (Optional) A list containing the specific indices of the desired lag features to derive for each feature (not including the series time feature). Specifying derived lag features for the feature specified by @@ -195,18 +256,18 @@ def infer_feature_attributes( (Optional) Maximum number of bytes that a feature's per-case average can compute to without raising a warning about memory usage (Pandas DataFrame only). - mode_bound_features : list of str, default None + mode_bound_features : list of str, optional (Optional) Explicit list of feature names to use mode bounds for when inferring loose bounds. If None, assumes all features. A mode bound is used instead of a loose bound when the mode for the feature is the same as an original bound, as it may represent an application-specific min/max. - nominal_substitution_config : dict of dicts, default None + nominal_substitution_config : dict of dicts, optional (Optional) Configuration of the nominal substitution engine and the nominal generators and detectors. - num_lags : int or dict, default None + num_lags : int or dict, optional (Optional) An integer specifying the number of lag features to derive for each feature (not including the series time feature). Specifying derived lag features for the feature specified by @@ -218,14 +279,14 @@ def infer_feature_attributes( The num_lags parameter will be overridden by the lags parameter per feature. - orders_of_derivatives : dict, default None + orders_of_derivatives : dict, optional (Optional) Dict of features and their corresponding order of derivatives for the specified type (delta/rate). If provided will generate the specified number of derivatives and boundary values. If set to 0, will not generate any delta/rate features. By default all continuous features have an order value of 1. - ordinal_feature_values : dict, default None + ordinal_feature_values : dict, optional (optional) Dict for ordinal string features defining an ordered list of string values for each feature, ordered low to high. If specified will set 'type' to be 'ordinal' for all features in @@ -238,7 +299,7 @@ def infer_feature_attributes( "size" : [ "small", "medium", "large", "huge" ] } - preserve_rare_values_config : dict, default None + preserve_rare_values_config : dict, optional (Optional) A map of feature name to a list of dict specifying a protected value and a case weight multiplier. Enables case weight rebalancing for data distillation workflows such that protected values do not lose signal. Compute automatically by providing @@ -258,12 +319,12 @@ def infer_feature_attributes( the format that can be expected if your `preserve_rare_values_config` comes from a suggestion after calling `infer_feature_attributes`. - preserve_rare_values_map : dict or "all", default None + preserve_rare_values_map : dict or "all", optional (Optional) A map of feature name to list of values that should be protected during data distillation. If set to "all", will infer and attempt to preserve all detected rare values. - rate_boundaries : dict, default None + rate_boundaries : dict, optional (Optional) For time series, specify the rate boundaries in the form {"feature" : {"min|max" : {order : value}}}. Works with partial values by specifying only particular order of derivatives you would like to @@ -298,7 +359,7 @@ def infer_feature_attributes( normal. Disabling suggestions can meaningfully reduce runtime on large datasets when suggestions are not needed. - tight_bounds: Iterable of str, default None + tight_bounds: Iterable of str, optional (Optional) Set tight min and max bounds for the features specified in the Iterable. @@ -313,7 +374,7 @@ def infer_feature_attributes( time_feature_name : str, default None (Optional, required for time series) The name of the time feature. - time_invariant_features : list of str, default None + time_invariant_features : list of str, optional (Optional) Names of time-invariant features. If none are provided, they will be inferred automatically. @@ -328,12 +389,12 @@ def infer_feature_attributes( feature values are predicted via interpolation at each timestep rather than being derived using a delta or rate. - time_series_types_override : dict, default None + time_series_types_override : dict, optional (Optional) Dict of features and their corresponding time series type, one of 'rate', 'delta', or 'covariate'. Used to override ``time_series_type_default`` for the specified features. - types: dict, default None + types: dict, optional (Optional) Dict of features and their intended type (i.e., "nominal," "ordinal," or "continuous"), or types mapped to MutableSequences of feature names. Any types provided here will override the types that would @@ -408,18 +469,20 @@ def infer_feature_attributes( elif isinstance(data, IFACompatibleADCProtocol): infer = IFATimeSeriesADC(data, time_feature_name) else: - raise NotImplementedError("`infer_feature_attributes` for time series only supported for DataFrames and " - "AbstractData classes.") + raise NotImplementedError( + "`infer_feature_attributes` for time series only supported for DataFrames and AbstractData classes." + ) elif time_feature_name: - raise ValueError("'time_feature_name' was included, but 'data' must be of type DataFrame " - "for time series feature attributes to be calculated.") + raise ValueError( + "'time_feature_name' was included, but 'data' must be of type DataFrame " + "for time series feature attributes to be calculated." + ) # Else, check data type elif isinstance(data, pd.DataFrame): infer = InferFeatureAttributesDataFrame(data) elif isinstance(data, SQLRelationalDatastoreProtocol): if tables is None: - raise TypeError("'tables' is a required parameter if 'data' implements " - "SQLRelationalDatastoreProtocol.") + raise TypeError("'tables' is a required parameter if 'data' implements SQLRelationalDatastoreProtocol.") infer = InferFeatureAttributesSQLDatastore(data, tables) elif isinstance(data, IFACompatibleADCProtocol): infer = InferFeatureAttributesAbstractData(data) diff --git a/howso/utilities/feature_attributes/suggestions.py b/howso/utilities/feature_attributes/suggestions.py index ba364dfa..e3170a70 100644 --- a/howso/utilities/feature_attributes/suggestions.py +++ b/howso/utilities/feature_attributes/suggestions.py @@ -138,8 +138,8 @@ def __repr__(self) -> str: rows.extend([ ( - "Get a reusable `fanout_features_map`", - "You may provide `fanout_features_map` as a parameter to " + "Get a reusable `fanout_feature_map`", + "You may provide `fanout_feature_map` as a parameter to " "`infer_feature_attributes` if you wish to adjust the fan-out feature " "configuration. Our detected fan-out feature configuration may be a " "good starting point.", @@ -148,7 +148,7 @@ def __repr__(self) -> str: ), ( "Apply suggestion to this feature attributes object", - "Save the suggested candidate `fanout_features_map` " + "Save the suggested candidate `fanout_feature_map` " "to this feature attributes object.", "Call `apply_suggestion()` on the feature attributes object: " '`apply_suggestion("fanout_features")`' @@ -188,7 +188,7 @@ def apply(self, attributes: dict) -> None: attributes[f]["fanout_on"] = list(_key_features) def get_fanout_feature_map(self) -> FanoutFeaturesMap: - """Get the `fanout_features_map` for use in future calls to `infer_feature_attributes`.""" + """Get the `fanout_feature_map` for use in future calls to `infer_feature_attributes`.""" return self._fanout_features def merge(self, other: IFASuggestion) -> None: diff --git a/howso/utilities/feature_attributes/time_series.py b/howso/utilities/feature_attributes/time_series.py index 8c1533c0..8f9b8c00 100644 --- a/howso/utilities/feature_attributes/time_series.py +++ b/howso/utilities/feature_attributes/time_series.py @@ -365,7 +365,7 @@ def _process( # noqa: C901 nominal_substitution_config: t.Optional[dict[str, dict]] = None, num_lags: t.Optional[int | dict] = None, orders_of_derivatives: t.Optional[dict] = None, - ordinal_feature_values: t.Optional[dict[str, list[str]]] = None, + ordinal_feature_values: t.Optional[dict[str, list[t.Any]]] = None, rate_boundaries: t.Optional[dict] = None, time_invariant_features: t.Optional[Iterable[str]] = None, tight_bounds: t.Optional[Iterable[str]] = None, From cb4df53fb158fca1f658785da7ba8beadaf8e7a2 Mon Sep 17 00:00:00 2001 From: Matt Fulp <8397318+fulpm@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:41:33 -0400 Subject: [PATCH 4/8] don't use t for types --- howso/utilities/feature_attributes/base.py | 47 ++++++++++++---------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/howso/utilities/feature_attributes/base.py b/howso/utilities/feature_attributes/base.py index 05343d37..6821edd9 100644 --- a/howso/utilities/feature_attributes/base.py +++ b/howso/utilities/feature_attributes/base.py @@ -10,7 +10,7 @@ import math from pathlib import Path import platform -import typing as t +from typing import Any, cast, Literal, Self, TYPE_CHECKING import warnings from zoneinfo import ZoneInfo @@ -40,7 +40,7 @@ time_to_seconds, ) -if t.TYPE_CHECKING: +if TYPE_CHECKING: from howso.client.typing import FeatureAttributes logger = logging.getLogger(__name__) @@ -61,14 +61,19 @@ class FeatureAttributesBase(dict[str, "FeatureAttributes"]): """Provides accessor methods for and dict-like access to inferred feature attributes.""" - def __init__(self, feature_attributes: Mapping, params: dict | None = None, unsupported: list[str] | None = None, - suggestions_collector: IFASuggestionCollector | None = None) -> None: + def __init__( + self, + feature_attributes: Mapping[str, Any], + params: dict[str, Any] | None = None, + unsupported: list[str] | None = None, + suggestions_collector: IFASuggestionCollector | None = None, + ) -> None: """ Instantiate this FeatureAttributesBase object. Parameters ---------- - feature_attributes : dict + feature_attributes : Mapping The feature attributes dictionary to be wrapped by this object. params : dict (Optional) The parameters used in the call to infer_feature_attributes. @@ -119,7 +124,7 @@ def suggestions(self) -> IFASuggestionCollector: """Get the suggestions for this FeatureAttributesBase object.""" return self.suggestions_collector - def get_parameters(self) -> dict: + def get_parameters(self) -> dict[str, Any]: """ Get the keyword arguments used with the initial call to infer_feature_attributes. @@ -171,11 +176,11 @@ def to_json(self, archive: bool = False, json_path: Path | None = None) -> str: @classmethod def from_json( - cls: type[FeatureAttributesBase], + cls, json_str: str | None = None, *, json_path: str | None = None - ) -> FeatureAttributesBase: + ) -> Self: """ Reconstruct a FeatureAttributesBase from JSON. @@ -440,7 +445,7 @@ def _validate_df(self, data: pd.DataFrame, coerce: bool = False, allow_missing_features: bool = False, localize_datetimes=True, nullable_int_dtype="Int64"): errors = [] coerced_df = data.copy(deep=True) - features = t.cast(dict[str, "FeatureAttributes"], self[table_name] if table_name else self) + features = cast(dict[str, "FeatureAttributes"], self[table_name] if table_name else self) for feature, attributes in features.items(): if feature not in data.columns: @@ -558,7 +563,7 @@ def _validate_df(self, data: pd.DataFrame, coerce: bool = False, return None - def validate(self, data: t.Any, coerce: bool = False, raise_errors: bool = False, validate_bounds: bool = True, + def validate(self, data: Any, coerce: bool = False, raise_errors: bool = False, validate_bounds: bool = True, allow_missing_features: bool = False, localize_datetimes: bool = True) -> None | pd.DataFrame: """ Validate the given data against this FeatureAttributes object. @@ -661,7 +666,7 @@ class SingleTableFeatureAttributes(FeatureAttributesBase): """A dict-like object containing feature attributes for a single table or DataFrame.""" @singledispatchmethod - def validate(data: t.Any, **kwargs): + def validate(data: Any, **kwargs: Any): """ Validate the given single table data against this FeatureAttributes object. @@ -819,8 +824,8 @@ def _process(self, mode_bound_features: Iterable[str] | None = None, num_series: int = 1, nominal_substitution_config: dict[str, dict] | None = None, - ordinal_feature_values: dict[str, list[t.Any]] | None = None, - preserve_rare_values_map: PreserveRareValuesMap | t.Literal["all", "off"] | None = None, + ordinal_feature_values: dict[str, list[Any]] | None = None, + preserve_rare_values_map: PreserveRareValuesMap | Literal["all", "off"] | None = None, preserve_rare_values_config: PreserveRareValuesConfig | FullPreserveRareValuesConfig | None = None, significance_threshold: int = SIGNIFICANT_THRESHOLD_DEFAULT, tight_bounds: Iterable[str] | None = None, @@ -1276,7 +1281,7 @@ def _infer_string_attributes(self, feature_name: str) -> dict: } return self._infer_unknown_attributes(feature_name) - def _infer_unknown_attributes(self, *args: t.Any) -> dict: + def _infer_unknown_attributes(self, *args: Any) -> dict: """Get inferred attributes for the given unknown-type column.""" return { "type": "nominal", @@ -1771,7 +1776,7 @@ def _get_n_random_rows(self, samples: int = 5000, seed: int | None = None) -> pd """Get random samples from the given data as a DataFrame.""" @abstractmethod - def _get_random_value(self, feature_name: str, no_nulls: bool = False) -> t.Any: + def _get_random_value(self, feature_name: str, no_nulls: bool = False) -> Any: """Retrieve a random value from the data.""" @abstractmethod @@ -1779,7 +1784,7 @@ def _has_unique_constraint(self, feature_name: str) -> bool: """Return whether this feature has a unique constraint.""" @abstractmethod - def _get_first_non_null(self, feature_name: str) -> t.Any: + def _get_first_non_null(self, feature_name: str) -> Any: """ Get the first non-null value in the given column. @@ -1804,7 +1809,7 @@ def _get_unique_count(self, feature_name: str | Iterable[str]) -> int: """Get the number of unique values in the provided feature(s).""" @abstractmethod - def _get_unique_values(self, feature_name: str) -> Collection[t.Any]: + def _get_unique_values(self, feature_name: str) -> Collection[Any]: """Get a set of the unique values for the given feature_name.""" @abstractmethod @@ -1812,7 +1817,7 @@ def _get_row_count(self) -> int: """Get the total number of rows in the data.""" @abstractmethod - def _get_value_count(self, feature_name: str, value: t.Any) -> int: + def _get_value_count(self, feature_name: str, value: Any) -> int: """Get the number of occurrences of the provided value of the provided feature.""" def _find_protected_value_candidates(self, max_distilled_cases: int, @@ -1864,7 +1869,7 @@ def _find_protected_value_candidates(self, max_distilled_cases: int, top_five = sorted(value_counts, key=lambda d: d["count"], reverse=True)[:5] return pvm, top_five - def _compute_unprotected_multiplier(self, feature: str, protected_values_multipliers: Sequence[dict[str, t.Any]], + def _compute_unprotected_multiplier(self, feature: str, protected_values_multipliers: Sequence[dict[str, Any]], *, row_count: int | None = None) -> float: """ Compute the unprotected multiplier for the provided feature given a list of rare values with multipliers. @@ -1905,7 +1910,7 @@ def _compute_unprotected_multiplier(self, feature: str, protected_values_multipl def _compute_preserve_rare_values_config( self, max_distilled_cases: int, - preserve_rare_values_map: PreserveRareValuesMap | t.Literal["all"], + preserve_rare_values_map: PreserveRareValuesMap | Literal["all"], significance_threshold: int ) -> FullPreserveRareValuesConfig: """ @@ -1956,7 +1961,7 @@ def _compute_preserve_rare_values_config( ) return prvc - def _process_rare_values(self, preserve_rare_values_map: PreserveRareValuesMap | t.Literal["all", "off"] | None, # noqa: PLR0912, PLR0915 + def _process_rare_values(self, preserve_rare_values_map: PreserveRareValuesMap | Literal["all", "off"] | None, # noqa: PLR0912, PLR0915 preserve_rare_values_config: PreserveRareValuesConfig | None, max_distilled_cases: int, significance_threshold: int, enable_suggestions: bool = True) -> None: """Procesess `preserve_rare_values` configuration or make recommendation.""" From 55883d38cd54f67b1935e4244eeee6cddf4d7c6d Mon Sep 17 00:00:00 2001 From: Matt Fulp <8397318+fulpm@users.noreply.github.com> Date: Thu, 16 Jul 2026 11:48:03 -0400 Subject: [PATCH 5/8] description --- .../feature_attributes/infer_feature_attributes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/howso/utilities/feature_attributes/infer_feature_attributes.py b/howso/utilities/feature_attributes/infer_feature_attributes.py index 9d2a7106..e3c654ed 100644 --- a/howso/utilities/feature_attributes/infer_feature_attributes.py +++ b/howso/utilities/feature_attributes/infer_feature_attributes.py @@ -90,10 +90,10 @@ def infer_feature_attributes( Feature attributes provide Howso with a schema like representation of the data and is required in order to ingest the data into Howso. It is important to inspect the result of this method in order to verify the accuracy - of what is inferred and so that any inaccuracies can be corrected before data ingestion. This method makes a best + of what was inferred and so that any inaccuracies can be corrected before data ingestion. This method makes a best guess at the feature attributes and in some cases may not automatically set certain attributes if there is - ambiguity, instead raising suggestions for further consideration which can be reviewed by accessing the - :attr:`~FeatureAttributesBase.suggestions` attribute. + ambiguity, instead providing suggestions for further consideration which can be reviewed by accessing the + :attr:`~howso.utilities.feature_attributes.base.FeatureAttributesBase.suggestions` attribute. Parameters ---------- From 9bd7447317b67315727089d59fb61358e91274ba Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 16 Jul 2026 16:46:40 +0000 Subject: [PATCH 6/8] Trigger build after approval From 8f962cc50997af7ddbafdfb134a0271cb95de395 Mon Sep 17 00:00:00 2001 From: Matt Fulp <8397318+fulpm@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:06:26 -0400 Subject: [PATCH 7/8] stronger typing --- howso/utilities/feature_attributes/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/howso/utilities/feature_attributes/base.py b/howso/utilities/feature_attributes/base.py index 6821edd9..670adf52 100644 --- a/howso/utilities/feature_attributes/base.py +++ b/howso/utilities/feature_attributes/base.py @@ -63,7 +63,7 @@ class FeatureAttributesBase(dict[str, "FeatureAttributes"]): def __init__( self, - feature_attributes: Mapping[str, Any], + feature_attributes: Mapping[str, FeatureAttributes], params: dict[str, Any] | None = None, unsupported: list[str] | None = None, suggestions_collector: IFASuggestionCollector | None = None, @@ -236,8 +236,8 @@ def to_dataframe(self, *, include_all: bool = False) -> pd.DataFrame: """ raise NotImplementedError("Function not yet implemented for all subclasses of `FeatureAttributesBase`") - def get_names(self, *, types: str | Container | None = None, - data_types: str | Container | None = None, + def get_names(self, *, types: str | Container[str] | None = None, + data_types: str | Container[str] | None = None, without: str | Iterable[str] | None = None, ) -> list[str]: """ From 3fdb897dff962ee740a4ea53c96c8ded7a22e532 Mon Sep 17 00:00:00 2001 From: Matt Fulp <8397318+fulpm@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:01:47 -0400 Subject: [PATCH 8/8] add missing options --- .../feature_attributes/infer_feature_attributes.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/howso/utilities/feature_attributes/infer_feature_attributes.py b/howso/utilities/feature_attributes/infer_feature_attributes.py index e3c654ed..e1d5dc9d 100644 --- a/howso/utilities/feature_attributes/infer_feature_attributes.py +++ b/howso/utilities/feature_attributes/infer_feature_attributes.py @@ -14,6 +14,11 @@ TableNameProtocol, ) from howso.utilities.feature_attributes.relational import InferFeatureAttributesSQLDatastore +from howso.utilities.feature_attributes.suggestions import ( + FullPreserveRareValuesConfig, + PreserveRareValuesConfig, + PreserveRareValuesMap, +) from howso.utilities.feature_attributes.time_series import IFATimeSeriesADC, IFATimeSeriesPandas FeatureType: TypeAlias = Literal["continuous", "ordinal", "nominal"] @@ -39,6 +44,9 @@ class InferOptions(TypedDict, total=False): mode_bound_features: Iterable[str] nominal_substitution_config: dict[str, dict[str, Any]] ordinal_feature_values: dict[str, list[Any] | tuple[str]] + preserve_rare_values_config: PreserveRareValuesConfig | FullPreserveRareValuesConfig + preserve_rare_values_map: PreserveRareValuesMap | Literal["all", "off"] + significance_threshold: int tight_bounds: Iterable[str] types: dict[str, FeatureType] | dict[FeatureType, list[str]]