Fix Issue where High-Level API using ECAR with Given Seed Generated all Errors in the Same Columns

docs/requirements.txt

@@ -1,4 +1,4 @@
-Sphinx==8.1.3
-myst-parser==4.0.0
+Sphinx==8.2.3
+myst-parser==4.0.1
 pydata-sphinx-theme==0.16.1
-sphinx-autoapi==3.5.0
+sphinx-autoapi==3.6.1

pyproject.toml

@@ -1,14 +1,14 @@
 [project]
 name = "tab_err"
-version = "0.2.0"
+version = "0.2.1"
 description = "Fully-controllable error generation for tabular data."
 readme = "README.md"
 authors = [
     { name = "Philipp Jung", email = "philippjung@posteo.de" },
     { name = "Sebastian Jäger", email = "tab_err@sebastian-jaeger.me" },
 ]
 license = { text = "Apache License 2.0" }
-keywords = ["tabular-data", "data-quality", "error-modelling", "data-errors"]
+keywords = ["tabular-data", "data-quality", "error-modeling", "data-errors"]
 classifiers = [
     "Intended Audience :: Developers",
     "Intended Audience :: Science/Research",
@@ -25,7 +25,7 @@ classifiers = [
 
 requires-python = ">=3.9,<3.13"
 dependencies = [
-    "pandas>=2.2.3,<3.0.0",
+    "pandas>=2.3.0,<2.4.0",
 ]
 
 [project.urls]
@@ -48,6 +48,7 @@ dev = [
     "myst-parser>=4.0.0 ; python_version >= '3.10'",
     "pydata-sphinx-theme>=0.16.1,<0.17.0",
     "sphinx-autoapi>=3.5.0,<4.0.0",
+    "ty>=0.0.1a29",
 ]
 
 [build-system]
@@ -77,7 +78,7 @@ ignore = [
     "D100", # Do not document public modules
     "D104", # Do not document public packages
     "TD003", # Allow TODOs without links
-    "ISC003", # Do not implicietly concatenate strings
+    "ISC003", # Do not implicitly concatenate strings
     "S101", # Allow testing methods
 ]
 
@@ -87,8 +88,11 @@ ignore = [
 ]
 "docs/source/conf.py" = [
     "A001", # Allow missing __init__.py for docs
-    "INP001", # Allow builtin shadowing for docs
+    "INP001", # Allow built in shadowing for docs
 ]
 
 [tool.ruff.lint.pydocstyle]
 convention = "google"
+
+[tool.ty.environment]
+python-version = "3.9"

tab_err/_utils.py

@@ -38,7 +38,7 @@ def get_column(data: pd.DataFrame, column: int | str) -> pd.Series:
     return data[get_column_str(data, column)]
 
 
-def seed_randomness(seed: int | None) -> np.random.Generator:
+def seed_randomness_and_get_generator(seed: int | None) -> np.random.Generator:
     if seed is not None:
         random.seed(seed)
         random_generator = np.random.default_rng(seed=seed)

tab_err/api/high_level.py

@@ -1,15 +1,18 @@
 from __future__ import annotations
 
-import random
 import warnings
+from typing import TYPE_CHECKING
 
 import pandas as pd
 
 from tab_err import ErrorMechanism, ErrorType, error_mechanism, error_type
 from tab_err._error_model import ErrorModel
-from tab_err._utils import check_data_emptiness, check_error_rate, seed_randomness
+from tab_err._utils import check_data_emptiness, check_error_rate, seed_randomness_and_get_generator
 from tab_err.api import MidLevelConfig, mid_level
 
+if TYPE_CHECKING:
+    from numpy.random import Generator
+
 
 def _are_same_class(obj1: object, obj2: object) -> bool:
     """Checks if two objects are of the same class.
@@ -36,17 +39,20 @@ def _are_same_error_mechanism(error_mechanism1: ErrorMechanism, error_mechanism2
 
 
 def _build_column_type_dictionary(
-    data: pd.DataFrame, error_types_to_include: list[ErrorType] | None = None, error_types_to_exclude: list[ErrorType] | None = None, seed: int | None = None
+    data: pd.DataFrame,
+    random_generator: Generator,
+    error_types_to_include: list[ErrorType] | None = None,
+    error_types_to_exclude: list[ErrorType] | None = None,
 ) -> dict[int | str, list[ErrorType]]:
     """Creates a dictionary mapping from column names to the list of valid error types to apply to that column.
 
     Args:
         data (pd.DataFrame): The pandas DataFrame to create errors in.
+        random_generator (Generator): Random Generator. Defaults to None.
         error_types_to_include (list[ErrorType] | None, optional): A list of the error types to be included when building error models. Defaults to None.
         error_types_to_exclude (list[ErrorType] | None, optional): A list of the error types to be excluded when building error models. Defaults to None.
             When both error_types_to_include and error_types_to_exclude are none, the maximum number of default error types will be used.
             At least one must be None or an error will occur.
-        seed (int | None, optional): Random seed. Defaults to None.
 
     Raises:
         ValueError: If error_types_to_exclude is not None and error_types_to_include is not None, a ValueError is thrown.
@@ -56,15 +62,15 @@ def _build_column_type_dictionary(
         dict[int | str, list[ErrorModel]]: A dictionary mapping from column names to the list of valid error types to apply to that column.
     """
     error_types_applied = [
-        error_type.AddDelta(seed=seed),
-        error_type.CategorySwap(seed=seed),
-        error_type.Extraneous(seed=seed),
-        error_type.Mojibake(seed=seed),
-        error_type.Outlier(seed=seed),
-        error_type.Replace(seed=seed),
-        error_type.Typo(seed=seed),
-        error_type.WrongUnit(seed=seed),
-        error_type.MissingValue(seed=seed),
+        error_type.AddDelta(seed=random_generator.bit_generator.random_raw()),
+        error_type.CategorySwap(seed=random_generator.bit_generator.random_raw()),
+        error_type.Extraneous(seed=random_generator.bit_generator.random_raw()),
+        error_type.Mojibake(seed=random_generator.bit_generator.random_raw()),
+        error_type.Outlier(seed=random_generator.bit_generator.random_raw()),
+        error_type.Replace(seed=random_generator.bit_generator.random_raw()),
+        error_type.Typo(seed=random_generator.bit_generator.random_raw()),
+        error_type.WrongUnit(seed=random_generator.bit_generator.random_raw()),
+        error_type.MissingValue(seed=random_generator.bit_generator.random_raw()),
     ]
 
     if error_types_to_exclude is not None and error_types_to_include is not None:
@@ -96,19 +102,19 @@ def _build_column_type_dictionary(
 
 def _build_column_mechanism_dictionary(
     data: pd.DataFrame,
+    random_generator: Generator,
     error_mechanisms_to_include: list[ErrorMechanism] | None = None,
     error_mechanisms_to_exclude: list[ErrorMechanism] | None = None,
-    seed: int | None = None,
 ) -> dict[int | str, list[ErrorMechanism]]:
     """Builds a dictionary mapping from column names to the list of valid error mechanisms to apply to that column.
 
     Args:
         data (pd.DataFrame): The pandas DataFrame to create errors in.
+        random_generator (Generator): Random Generator. Defaults to None.
         error_mechanisms_to_include (list[ErrorMechanism] | None, optional): The error mechanisms (EAR, ECAR, ENAR) to include from the dictionary.
             Defaults to None.
         error_mechanisms_to_exclude (list[ErrorMechanism] | None, optional): The error mechanisms (EAR, ECAR, ENAR) to exclude from the dictionary.
             Defaults to None.
-        seed (int | None, optional): Random seed. Defaults to None.
 
     Returns:
         dict[int | str, list[ErrorMechanism]]: A dictionary mapping from column names to the list of valid error mechanisms to apply to that column.
@@ -135,8 +141,13 @@ def _build_column_mechanism_dictionary(
             raise ValueError(msg)
 
         for column in data.columns:
-            column_wise_error_mechs = [error_mechanism.ENAR(seed=seed), error_mechanism.ECAR(seed=seed)] + [
-                error_mechanism.EAR(condition_to_column=other_column, seed=seed) for other_column in data.columns if other_column != column
+            column_wise_error_mechs = [
+                error_mechanism.ENAR(seed=random_generator.bit_generator.random_raw()),
+                error_mechanism.ECAR(seed=random_generator.bit_generator.random_raw()),
+            ] + [
+                error_mechanism.EAR(condition_to_column=other_column, seed=random_generator.bit_generator.random_raw())
+                for other_column in data.columns
+                if other_column != column
             ]
             # Prune error mechanisms
             if error_mechanisms_to_exclude is not None:
@@ -208,6 +219,7 @@ def create_errors(  # noqa: PLR0913
             - The first element is a copy of 'data' with errors.
             - The second element is the associated error mask.
     """
+    random_generator = seed_randomness_and_get_generator(seed=seed)
     # Input Checking
     check_error_rate(error_rate)
     check_data_emptiness(data)
@@ -217,14 +229,18 @@ def create_errors(  # noqa: PLR0913
     error_mask = pd.DataFrame(data=False, index=data.index, columns=data.columns)
 
     # Build Dictionaries
-    col_type = _build_column_type_dictionary(data=data, error_types_to_include=error_types_to_include, error_types_to_exclude=error_types_to_exclude, seed=seed)
+    col_type = _build_column_type_dictionary(
+        data=data, random_generator=random_generator, error_types_to_include=error_types_to_include, error_types_to_exclude=error_types_to_exclude
+    )
     col_mechanisms = _build_column_mechanism_dictionary(
-        data=data, error_mechanisms_to_include=error_mechanisms_to_include, error_mechanisms_to_exclude=error_mechanisms_to_exclude, seed=seed
+        data=data,
+        random_generator=random_generator,
+        error_mechanisms_to_include=error_mechanisms_to_include,
+        error_mechanisms_to_exclude=error_mechanisms_to_exclude,
     )
     col_num_models = _build_column_number_of_models_dictionary(data=data, column_types=col_type, column_mechanisms=col_mechanisms)
 
     if n_error_models_per_column > 0:
-        seed_randomness(seed=seed)
         error_rate = error_rate / n_error_models_per_column
         config_dictionary: dict[str | int, list[ErrorModel]] = {
             column: [] for column in data.columns if col_num_models[column] > 0
@@ -237,7 +253,12 @@ def create_errors(  # noqa: PLR0913
         for column, error_model_list in config_dictionary.items():
             for _ in range(n_error_models_per_column):
                 error_model_list.append(
-                    ErrorModel(error_type=random.choice(col_type[column]), error_mechanism=random.choice(col_mechanisms[column]), error_rate=error_rate)
+                    ErrorModel(
+                        # NOTE: in python 3.9 mypy fails here but tests work
+                        error_type=random_generator.choice(col_type[column]),  # type: ignore[arg-type]
+                        error_mechanism=random_generator.choice(col_mechanisms[column]),  # type: ignore[arg-type]
+                        error_rate=error_rate,
+                    )
                 )
         config = MidLevelConfig(config_dictionary)
     else:  # n_error_models_per_column is 0 or less.

tab_err/error_mechanism/_error_mechanism.py

@@ -5,7 +5,7 @@
 
 import pandas as pd
 
-from tab_err._utils import seed_randomness
+from tab_err._utils import seed_randomness_and_get_generator
 
 if TYPE_CHECKING:
     import numpy as np
@@ -86,7 +86,7 @@ def sample(
         if error_mask is None:  # initialize empty error_mask
             error_mask = pd.DataFrame(data=False, index=data.index, columns=data.columns)
 
-        self._random_generator = seed_randomness(self._seed)
+        self._random_generator = seed_randomness_and_get_generator(self._seed)
         return self._sample(data, column, error_rate, error_mask)
 
     @abstractmethod

tab_err/error_type/_category_swap.py

@@ -64,13 +64,13 @@ def _apply(self: CategorySwap, data: pd.DataFrame, error_mask: pd.DataFrame, col
 
         if self.config.mislabel_weighing == "uniform":
 
-            def sample_label(old_label: pd.Any) -> pd.Any:
+            def sample_label(old_label: pd.Series) -> pd.Series:
                 choices = [x for x in series.cat.categories.to_numpy() if x != old_label]
                 return random.choice(choices)
 
         elif self.config.mislabel_weighing == "frequency":
 
-            def sample_label(old_label: pd.Any) -> pd.Any:
+            def sample_label(old_label: pd.Series) -> pd.Series:
                 se_sample = series.loc[series != old_label]
                 return se_sample.sample(1, replace=True).to_numpy()[0]
         else:

tab_err/error_type/_error_type.py

@@ -3,7 +3,7 @@
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, Any
 
-from tab_err._utils import seed_randomness
+from tab_err._utils import seed_randomness_and_get_generator
 
 from ._config import ErrorTypeConfig
 
@@ -63,7 +63,7 @@ def apply(self: ErrorType, data: pd.DataFrame, error_mask: pd.DataFrame, column:
             msg = f"The shape of 'data': {data.shape} was different from the shape of 'error_mask': {error_mask.shape}. They should be the same."
             raise ValueError(msg)
 
-        self._random_generator = seed_randomness(self._seed)
+        self._random_generator = seed_randomness_and_get_generator(self._seed)
         return self._apply(data, error_mask, column)
 
     def get_valid_columns(self: ErrorType, data: pd.DataFrame) -> list[str | int]:

tests/api/test_high_level_api.py

@@ -52,11 +52,9 @@ def test_create_errors_error_rates(self, test_data: dict[str, pd.DataFrame]) ->
         seed = 42
         for i in range(11):
             error_rate = 0.1 * float(i)
-            modified_data_100rows_3columns, data_100rows_3columns_error_mask = create_errors(test_data["data_100rows_3columns"], error_rate, seed=seed)
-            modified_data_10rows_3columns, data_10rows_3columns_error_mask = create_errors(test_data["data_10rows_3columns"], error_rate, seed=seed)
-            modified_data_10rows_3columns_with_datetime, data_10rows_3columns_with_datetime_error_mask = create_errors(
-                test_data["data_10rows_3columns_with_datetime"], error_rate, seed=seed
-            )
+            _, data_100rows_3columns_error_mask = create_errors(test_data["data_100rows_3columns"], error_rate, seed=seed)
+            _, data_10rows_3columns_error_mask = create_errors(test_data["data_10rows_3columns"], error_rate, seed=seed)
+            _, data_10rows_3columns_with_datetime_error_mask = create_errors(test_data["data_10rows_3columns_with_datetime"], error_rate, seed=seed)
 
             # Assert that the error masks have the correct proportion of True to False
             assert pytest.approx(error_rate) == data_100rows_3columns_error_mask.to_numpy().mean()

tests/api/test_low_level_api.py

@@ -12,13 +12,13 @@ def test_create_errors_error_rates(self, test_data: dict[str, pd.DataFrame]) ->
         """Test that create_errors returns two DataFrames with expected properties."""
         for i in range(11):
             error_rate = 0.1 * float(i)
-            modified_data_100rows_3columns, data_100rows_3columns_error_mask = create_errors(
+            _, data_100rows_3columns_error_mask = create_errors(
                 test_data["data_100rows_3columns"], "A", error_rate, error_mechanism.ECAR(), error_type.AddDelta()
             )
-            modified_data_10rows_3columns, data_10rows_3columns_error_mask = create_errors(
+            _, data_10rows_3columns_error_mask = create_errors(
                 test_data["data_10rows_3columns"], "A", error_rate, error_mechanism.ECAR(), error_type.AddDelta()
             )
 
-            # Assert that the error masks have the correct proportion of True to False -- Note only one column is errored
+            # Assert that the error masks have the correct proportion of True to False - Note only one column is errored
             assert pytest.approx(error_rate / 3.0) == data_100rows_3columns_error_mask.to_numpy().mean()
             assert pytest.approx(error_rate / 3.0) == data_10rows_3columns_error_mask.to_numpy().mean()