From 8732dc13d484a5e35c205fb1af62d0d1c5c074e6 Mon Sep 17 00:00:00 2001 From: Philipp Jung Date: Wed, 31 Dec 2025 17:32:39 +0100 Subject: [PATCH 1/3] chore: CI after merging into main, add pyton 3.13 CI is currently only run on PRs. However, it is a good practice to also run CI on `main` after merging to make sure that checks on `main` always pass. I also added Python 3.13 to the `python-version`s in the workflows - I don't see a reason why we wouldn't support 3.13. --- .github/workflows/mypy.yaml | 6 ++++-- .github/workflows/ruff.yaml | 6 ++++-- .github/workflows/tests.yaml | 6 ++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/mypy.yaml b/.github/workflows/mypy.yaml index 7ffe070..511eeff 100644 --- a/.github/workflows/mypy.yaml +++ b/.github/workflows/mypy.yaml @@ -1,7 +1,9 @@ name: mypy on: - - pull_request + pull_request: + push: + branches: [main] jobs: mypy: @@ -12,7 +14,7 @@ jobs: fail-fast: true matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] runs-on: ${{ matrix.os }} steps: #---------------------------------------------- diff --git a/.github/workflows/ruff.yaml b/.github/workflows/ruff.yaml index c2e0793..930bcf9 100644 --- a/.github/workflows/ruff.yaml +++ b/.github/workflows/ruff.yaml @@ -1,7 +1,9 @@ name: Ruff on: - - pull_request + pull_request: + push: + branches: [main] jobs: Ruff: @@ -12,7 +14,7 @@ jobs: fail-fast: true matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] runs-on: ${{ matrix.os }} steps: #---------------------------------------------- diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ca431bc..c3de31c 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -1,7 +1,9 @@ name: Pytest on: - - pull_request + pull_request: + push: + branches: [main] jobs: Test: @@ -12,7 +14,7 @@ jobs: fail-fast: true matrix: os: ["ubuntu-latest", "macos-latest", "windows-latest"] - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13"] runs-on: ${{ matrix.os }} steps: From 2683b4e9e8a00bd4d93c8bb19850168a65d15e4d Mon Sep 17 00:00:00 2001 From: Philipp Jung Date: Wed, 31 Dec 2025 18:51:30 +0100 Subject: [PATCH 2/3] Polish README This adds shields to the README that show supported python versions and the status of our github workflows. The shields are currently grayed out, but once this PR is merged, they will be show how the workflows on the `main` branches did. I gave the README another sturcture, it now starts with a tl;dr section at the very top. I also added a minimal example right to the README - I find that nice, a lot of people won't click any links to example notebooks. I also added a section for citing the paper. --- README.md | 116 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 101 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 798a733..d6bd0f7 100644 --- a/README.md +++ b/README.md @@ -1,34 +1,120 @@ # `tab_err` +

+ Inject Realistic Errors Into Tables +

-`tab_err` is an implementation of a tabular data error model that disentangles error mechanism and error type. -It generalizes the formalization of missing values, implying that missing values are only one of many possible error type implemented here. -`tab_err` gives the user full control over the error generation process and allows to model realistic errors with complex dependency structures. +

+ 📊 🔎 ✅ +

-The building blocks are `ErrorMechanism`s, `ErrorType`s, and `ErrorModel`s. -`ErrorMechanism` defines where the incorrect cells are and model realistic dependency structures and `ErrorType` describes in which way the value is incorrect. -Together they build a `ErrorModel` that can be used to perturb existing data with realistic errors. +

+ Test how your data pipelines and ML models react to tabular data that contains realistic errors. +

-This repository offers (soon) three APIs, low-level, mid-level and high-level. +
+ +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/tab_err)](https://pypi.org/project/tab_err/) +[![mypy](https://github.com/calgo-lab/tab_err/actions/workflows/mypy.yaml/badge.svg?branch=main)](https://github.com/calgo-lab/tab_err/actions/workflows/mypy.yaml) +[![pytest](https://github.com/calgo-lab/tab_err/actions/workflows/tests.yaml/badge.svg?branch=main)](https://github.com/calgo-lab/tab_err/actions/workflows/tests.yaml) +[![Ruff](https://github.com/calgo-lab/tab_err/actions/workflows/ruff.yaml/badge.svg?branch=main)](https://github.com/calgo-lab/tab_err/actions/workflows/ruff.yaml) + + +`tab_err` injects realistic errors into tabular data such as database tables and DataFrames. +The library is developed and maintained by the [Cognitive Algorithms Lab](https://calgo-lab.de/) at BHT Berlin. + +Using error-free tables as input, `tab_err` lets users define an error model that perturbs the table and can be shared as metadata. +Researchers and data practitioners can generate errors in a controlled way, evaluate how their systems behave, and exchange error scenarios reproducibly. + +## How it Works + +The library's building blocks are `ErrorMechanism`s, `ErrorType`s, and `ErrorModel`s. +- An `ErrorMechanism` describes the error's distribution - that's *where* incorrect cells appear in the table. We support *erroneous at random* (EAR), *erroneous not at random* (ENAR) and *erroneous completely at random* (ECAR). +- An `ErrorType` describes *how* the value is wrong: a typo, an outlier, a category swap, and so on. Read the documentation for a [full list of supported error types](https://tab-err.readthedocs.io/latest/api/tab_err/error_type/index.html). +- An `ErrorModel` is a set of mechanisms and types to perturb existing data with realistic errors. It is shareable as metadata. + +`tab_err` is supported by a `pandas` backend. ## Examples +```python +from sklearn.datasets import load_iris -For details and examples please check out our [Getting Started Notebook](https://github.com/calgo-lab/tab_err/blob/main/examples/1-Getting-Started.ipynb). +from tab_err import error_type +from tab_err.api import high_level -## Where to get it +df = load_iris(as_frame=True).frame +corrupted_df, error_mask = high_level.create_errors( + data=df, + error_rate=0.5, + error_types_to_exclude=[error_type.MissingValue()], + seed=42, +) +print("Original:") +print(df.head(2).to_string(index=False)) + +print("\nCorrupted:") +print(corrupted_df.head(2).to_string(index=False)) + +print("\nCorrupted cells:", int(error_mask.to_numpy().sum())) +``` -The source code is currently hosted on GitHub at: - +Example output: -Binary installers for the latest released version are available at the [Python -Package Index (PyPI)](https://pypi.org/project/tab-err). +```text +Original: + sepal length (cm) sepal width (cm) petal length (cm) petal width (cm) target + 5.1 3.5 1.4 0.2 0 + 4.9 3.0 1.4 0.2 0 + +Corrupted: + sepal length (cm) sepal width (cm) petal length (cm) petal width (cm) target + 5.1 35.0 1.400000 -2.775759 0.420326 + 4.9 30.0 1.820326 -3.087558 0.000000 + +Corrupted cells: 375 +``` + +For a detailed guide and more examples, see our [Getting Started Notebook](https://github.com/calgo-lab/tab_err/blob/main/examples/1-Getting-Started.ipynb) and the [documentation](https://tab-err.readthedocs.io/latest/). + +## Where to get it + +The source code is hosted on GitHub at . +Binary installers for the latest releases are available at the Python Package Index (PyPI) . ```sh +# with pip pip install tab-err + +# with uv +uv add tab-err ``` ## Contributing To develop `tab_err`, install the `uv` package manager. Run tests with `uv run pytest`. -Develop features on feature branches and open pull requests once you're ready to contribute. -Make sure that your code is tested, documented, and well described in the pull request. +Develop on feature branches and open pull requests when you're ready. +Make sure that your changes are tested, documented, and clearly described in the pull request. + +## Citation +If you use `tab_err` for a scientific publication, we would appreciate your citation. + +``` +@article{10.1145/3774914, +author = {Jung, Philipp and J\"{a}ger, Sebastian and Chandler, Nicholas and Biessmann, Felix}, +title = {Towards Realistic Error Models for Tabular Data}, +year = {2025}, +issue_date = {December 2025}, +publisher = {Association for Computing Machinery}, +address = {New York, NY, USA}, +volume = {17}, +number = {4}, +issn = {1936-1955}, +url = {https://doi.org/10.1145/3774914}, +doi = {10.1145/3774914}, +journal = {J. Data and Information Quality}, +month = dec, +articleno = {28}, +numpages = {27}, +keywords = {Tabular data, data quality, data errors, data error generation, error model, realistic error model, error type} +} +``` \ No newline at end of file From 93737edc47959819f7dd3b55b71dad8e8fe65a81 Mon Sep 17 00:00:00 2001 From: Philipp Jung Date: Wed, 31 Dec 2025 21:25:24 +0100 Subject: [PATCH 3/3] fix: clarify citation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d6bd0f7..df980a0 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ Develop on feature branches and open pull requests when you're ready. Make sure that your changes are tested, documented, and clearly described in the pull request. ## Citation -If you use `tab_err` for a scientific publication, we would appreciate your citation. +If you use the error model that's underlying `tab_err` for a scientific publication, we would appreciate your citation. ``` @article{10.1145/3774914,