Pandera column checks - problem with parsing errors

[Jarek-Rolski]

I tried to upload data frame with int data type instead of str as in schema. Pandera check generated CHECK_ERROR that contains python error Traceback. This has been picked up by parse_pandera_errors and returned to user. It shouldn't be returned to end user.

Code to replicate error:

import pandas as pd
import rapid
rapid_authentication = rapid.RapidAuth()  
rapid_sdk = rapid.Rapid(auth=rapid_authentication)
raw_data = [{"a": "1", "b": None, "c": 3}, {"a": "1", "b": None, "c": 3}, {"a": "1", "b": 2, "c": 3}]
df = pd.DataFrame(raw_data)
schema = rapid_sdk.generate_schema(
    df=df, layer="default", domain="test", dataset="test_dataset_checks", sensitivity="PUBLIC"
)
schema.columns[0].unique = True
schema.columns[1].unique = True
schema.columns[2].unique = True
schema.columns[0].checks = {"chk1": {"check_type": "str_length", "parameters": {"min_value": 3, "max_value": 6} }}
schema.columns[1].checks = {"chk2": {"check_type": "in_range", "parameters": {"min_value": 3, "max_value": 6} }}
schema.columns[2].checks = {"chk3": {"check_type": "in_range", "parameters": {"min_value": 3, "max_value": 6} }}
schema.metadata.owners[0].name = "Xxx Yyy"
schema.metadata.owners[0].email = "xxx.yyy@gov.uk"
schema.metadata.update_behaviour = "OVERWRITE"

rapid_sdk.create_schema(schema)

raw_data = [{"a": 1, "b": None, "c": 3}, {"a": 1, "b": None, "c": 3}, {"a": 1, "b": 2, "c": 3}]
df = pd.DataFrame(raw_data)

res = rapid_sdk.upload_dataframe("default", "test", "test_dataset_checks", df, wait_to_complete=True)

Error message in console:
JobFailedException: ('Upload failed', {'createdat': 1777628329, 'dataset': 'test_dataset_checks2', 'domain': 'test', 'errors': ['[str_length(3, 6)] Error while executing check function: AttributeError("Can only use .str accessor with string values!")Traceback (most recent call last): File "D:\\\\rapid\\\\backend\\\\.venv\\\\Lib\\\\site-packages\\\\pandera\\\\backends\\\\pandas\\\\components.py", line 240, in run_checks self.run_check( File "D:\\\\rapid\\\\backend\\\\.venv\\\\Lib\\\\site-packages\\\\pandera\\\\backends\\\\pandas\\\\base.py", line 115, in run_check check_result: CheckResult = check(check_obj, *args) ^^^^^^^^^^^^^^^^^^^^^^^ File "D:\\\\rapid\\\\backend\\\\.venv\\\\Lib\\\\site-packages\\\\pandera\\\\api\\\\checks.py", line 230, in __call__ return backend(check_obj, column) ^^^^^^^^^^^^^^^^^^^^^^^^^^ File "D:\\\\rapid\\\\backend\\\\.venv\\\\Lib\\\\site-packages\\\\pandera\\\\backends\\\\pandas\\\\checks.py", line 349, in __call__ check_output = self.apply(check_obj) ^^^^^^^^^^^^^^^^^^^^^ File "D:\\\\rapid\\\\backend\\\\.venv\\\\Lib\\\\site-packages\\\\pandera\\\\backends\\\\pandas\\\\checks.py", line 148, in apply return apply_fn(check_obj) ^^^^^^^^^^^^^^^^^^^ File "D:\\\\rapid\\\\backend\\\\.venv\\\\Lib\\\\site-packages\\\\pandera\\\\backends\\\\pandas\\\\checks.py", line 156, in apply_field return self.check_fn(check_obj) ^^^^^^^^^^^^^^^^^^^^^^^^ File "D:\\\\rapid\\\\backend\\\\.venv\\\\Lib\\\\site-packages\\\\pandera\\\\api\\\\function_dispatch.py", line 25, in __call__ return fn(*args, **kwargs) ^^^^^^^^^^^^^^^^^^^ File "D:\\\\rapid\\\\backend\\\\.venv\\\\Lib\\\\site-packages\\\\pandera\\\\backends\\\\pandas\\\\builtin_checks.py", line 282, in str_length str_len = data.str.len() ^^^^^^^^ File "D:\\\\rapid\\\\backend\\\\.venv\\\\Lib\\\\site-packages\\\\pandas\\\\core\\\\generic.py", line 6321, in __getattr__ return object.__getattribute__(self, name) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "D:\\\\rapid\\\\backend\\\\.venv\\\\Lib\\\\site-packages\\\\pandas\\\\core\\\\accessor.py", line 224, in __get__ accessor_obj = self._accessor(obj) ^^^^^^^^^^^^^^^^^^^ File "D:\\\\rapid\\\\backend\\\\.venv\\\\Lib\\\\site-packages\\\\pandas\\\\core\\\\strings\\\\accessor.py", line 194, in __init__ self._inferred_dtype = self._validate(data) ^^^^^^^^^^^^^^^^^^^^ File "D:\\\\rapid\\\\backend\\\\.venv\\\\Lib\\\\site-packages\\\\pandas\\\\core\\\\strings\\\\accessor.py", line 248, in _validate raise AttributeError("Can only use .str accessor with string values!")AttributeError: Can only use .str accessor with string values!. Did you mean: \'std\'?', 'Column [a] has an incorrect data type. Expected string, received int', "series 'c' contains duplicate values", "[in_range(3, 6)] Column 'b' failed element-wise validator number 0: in_range(3, 6) failure cases: 2.0", "series 'a' contains duplicate values", "series 'b' contains duplicate values"], 'filename': '2bce7dcd-e16f-488c-affb-0e1779798de8-rapid-sdk-1777628327.parquet', 'layer': 'default', 'raw_file_identifier': 'cbbe1854-bb7c-45a7-b9be-d01298f791db', 'sk2': '1vshf8nc814bqhfgarni9obgp7', 'status': 'FAILED', 'step': 'VALIDATION', 'ttl': 1785404329, 'type': 'UPLOAD', 'version': 1, 'job_id': '2bce7dcd-e16f-488c-affb-0e1779798de8'})

This is Pandera Validation Error extracted from API:

{
    "DATA": {
        "SERIES_CONTAINS_DUPLICATES": [
            {
                "schema": null,
                "column": "a",
                "check": "field_uniqueness",
                "error": "series 'a' contains duplicate values:0    11    12    1Name: a, dtype: int64"
            },
            {
                "schema": null,
                "column": "b",
                "check": "field_uniqueness",
                "error": "series 'b' contains duplicate values:0   NaN1   NaNName: b, dtype: float64"
            },
            {
                "schema": null,
                "column": "c",
                "check": "field_uniqueness",
                "error": "series 'c' contains duplicate values:0    31    32    3Name: c, dtype: int64"
            }
        ],
        "CHECK_ERROR": [
            {
                "schema": null,
                "column": "a",
                "check": "str_length(3, 6)",
                "error": "Error while executing check function: AttributeError(\"Can only use .str accessor with string values!\")Traceback (most recent call last):  File \"D:\\rapid\\backend\\.venv\\Lib\\site-packages\\pandera\\backends\\pandas\\components.py\", line 240, in run_checks    self.run_check(  File \"D:\\rapid\\backend\\.venv\\Lib\\site-packages\\pandera\\backends\\pandas\\base.py\", line 115, in run_check    check_result: CheckResult = check(check_obj, *args)                                ^^^^^^^^^^^^^^^^^^^^^^^  File \"D:\\rapid\\backend\\.venv\\Lib\\site-packages\\pandera\\api\\checks.py\", line 230, in __call__    return backend(check_obj, column)           ^^^^^^^^^^^^^^^^^^^^^^^^^^  File \"D:\\rapid\\backend\\.venv\\Lib\\site-packages\\pandera\\backends\\pandas\\checks.py\", line 349, in __call__    check_output = self.apply(check_obj)                   ^^^^^^^^^^^^^^^^^^^^^  File \"D:\\rapid\\backend\\.venv\\Lib\\site-packages\\pandera\\backends\\pandas\\checks.py\", line 148, in apply    return apply_fn(check_obj)           ^^^^^^^^^^^^^^^^^^^  File \"D:\\rapid\\backend\\.venv\\Lib\\site-packages\\pandera\\backends\\pandas\\checks.py\", line 156, in apply_field    return self.check_fn(check_obj)           ^^^^^^^^^^^^^^^^^^^^^^^^  File \"D:\\rapid\\backend\\.venv\\Lib\\site-packages\\pandera\\api\\function_dispatch.py\", line 25, in __call__    return fn(*args, **kwargs)           ^^^^^^^^^^^^^^^^^^^  File \"D:\\rapid\\backend\\.venv\\Lib\\site-packages\\pandera\\backends\\pandas\\builtin_checks.py\", line 282, in str_length    str_len = data.str.len()              ^^^^^^^^  File \"D:\\rapid\\backend\\.venv\\Lib\\site-packages\\pandas\\core\\generic.py\", line 6321, in __getattr__    return object.__getattribute__(self, name)           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^  File \"D:\\rapid\\backend\\.venv\\Lib\\site-packages\\pandas\\core\\accessor.py\", line 224, in __get__    accessor_obj = self._accessor(obj)                   ^^^^^^^^^^^^^^^^^^^  File \"D:\\rapid\\backend\\.venv\\Lib\\site-packages\\pandas\\core\\strings\\accessor.py\", line 194, in __init__    self._inferred_dtype = self._validate(data)                           ^^^^^^^^^^^^^^^^^^^^  File \"D:\\rapid\\backend\\.venv\\Lib\\site-packages\\pandas\\core\\strings\\accessor.py\", line 248, in _validate    raise AttributeError(\"Can only use .str accessor with string values!\")AttributeError: Can only use .str accessor with string values!. Did you mean: 'std'?"
            }
        ],
        "DATAFRAME_CHECK": [
            {
                "schema": null,
                "column": "b",
                "check": "in_range(3, 6)",
                "error": "Column 'b' failed element-wise validator number 0: in_range(3, 6) failure cases: 2.0"
            }
        ]
    }
}

Possible solution:
Inside parse_pandera_errors function there is no need to convert error to string and use regex to extract error messages, it is possible to loop error message with:

def parse_pandera_errors(exc: pandera.errors.SchemaErrors) -> list[str]:
    """
    Parse Pandera SchemaErrors exception to extract the 'error' field from error messages.

    """
    error_messages = []

    for error in exc.message['DATA']['DATAFRAME_CHECK']:
        check_name = error["check"]
        error_msg = error["error"]
        error_msg = f"[{check_name}] {error_msg}"
        error_messages.append(error_msg)

    return error_messages

This solution works for me, but I don't know if it covers all cases.