From 51b763673adf8b77342a3a758d706e765c20c432 Mon Sep 17 00:00:00 2001 From: David Fulford Date: Tue, 26 May 2026 18:35:29 -0500 Subject: [PATCH] fix: hash-based encoding for object-dtype join/groupby keys (v2.2.5) np.unique's internal argsort raises TypeError on object arrays mixing Python types (e.g., str and int) because cross-type `<` is unsupported. The pure-Python encoding paths in _encode_columns and _encode_columns_paired now use a dict-based codebook for object dtype, matching the C accelerator's hash-equality semantics. Fixes joins/groupbys that previously failed when both sides had _dtypes='object' with heterogeneous Python values. Also fixes a pre-existing mypy return-type error in to_pandas via cast(), and documents the three version-bump files in CLAUDE.md. Co-Authored-By: Claude Opus 4.7 (1M context) --- CLAUDE.md | 8 ++++++++ docs/changelog.md | 9 +++++++++ pyproject.toml | 2 +- recipe/meta.yaml | 2 +- tafra/base.py | 11 ++++++++-- tafra/group.py | 21 +++++++++++++++++++ test/test_tafra.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 99 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 507af7a..9e3c6f9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -82,3 +82,11 @@ The library has one core abstraction and a set of aggregation/partitioning opera ## Configuration - `pyproject.toml` — ruff (max-line-length=100), mypy (strict), pytest addopts, coverage + +## Version bump + +Update the version in all three places (they must stay in sync — conda-forge pulls from `meta.yaml`, PyPI from `pyproject.toml`): + +1. `pyproject.toml` — `version = "X.Y.Z"` +2. `recipe/meta.yaml` — `{% set version = "X.Y.Z" %}` +3. `docs/changelog.md` — add a new `## X.Y.Z` section at the top with bulleted `**Fix**:` / `**Feature**:` entries diff --git a/docs/changelog.md b/docs/changelog.md index 917a740..740da2f 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,14 @@ # Version History +## 2.2.5 + +* **Fix**: Join and `group_by` on `object`-dtype key columns containing mixed + Python types (e.g., `str` and `int` across sides, or mixed within one side) + no longer raise `TypeError: '<' not supported between instances of ...`. + The pure-Python encoding path now uses a hash-based codebook for object + arrays, matching the C accelerator's semantics. Previously, `np.unique`'s + internal `argsort` failed on cross-type comparisons. + ## 2.2.4 * **Fix**: Restore `MANIFEST.in` so test CSV fixtures (`test/ex*.csv`) are diff --git a/pyproject.toml b/pyproject.toml index 3615968..9481770 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "tafra" -version = "2.2.4" +version = "2.2.5" description = "Tafra: essence of a dataframe" readme = "README.md" license = "MIT" diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 8df1e67..3a9ec08 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -1,5 +1,5 @@ {% set name = "tafra" %} -{% set version = "2.2.4" %} +{% set version = "2.2.5" %} package: diff --git a/tafra/base.py b/tafra/base.py index ad2b755..a4856a2 100644 --- a/tafra/base.py +++ b/tafra/base.py @@ -2250,8 +2250,15 @@ def to_pandas(self, columns: Iterable[str] | None = None) -> DataFrame: columns = [columns] self._validate_columns(columns) - return pd.DataFrame( - {column: pd.Series(value) for column, value in self._data.items() if column in columns} + return cast( + DataFrame, + pd.DataFrame( + { + column: pd.Series(value) + for column, value in self._data.items() + if column in columns + } + ), ) def to_csv( diff --git a/tafra/group.py b/tafra/group.py index 897c72a..c2f9381 100644 --- a/tafra/group.py +++ b/tafra/group.py @@ -449,6 +449,22 @@ class GroupSet: A `GroupSet` is the set of columns by which we construct our groups. """ + @staticmethod + def _hash_encode_object(arr: np.ndarray[Any, Any]) -> np.ndarray[Any, Any]: + # np.unique sorts via argsort, which raises TypeError on object arrays + # containing mixed Python types (e.g., str and int). Use a dict codebook + # — semantics match _c_encode_strings (hash equality, not ordering). + codebook: dict[Any, int] = {} + codes = np.empty(len(arr), dtype=np.int64) + for i in range(len(arr)): + v = arr[i] + code = codebook.get(v) + if code is None: + code = len(codebook) + codebook[v] = code + codes[i] = code + return codes + @staticmethod def _encode_columns( col_arrays: list[np.ndarray[Any, Any]], @@ -470,6 +486,9 @@ def _encode_columns( codes, _ = _c_encode_strings(obj_arr) encoded.append(codes) codebooks.append(None) + elif c.dtype.kind == "O": + encoded.append(GroupSet._hash_encode_object(c)) + codebooks.append(None) else: uniq, codes = np.unique(c, return_inverse=True) encoded.append(codes) @@ -516,6 +535,8 @@ def _encode_columns_paired( combined = np.concatenate([lc, rc]) if _HAS_ACCEL and len(combined) >= 50_000: codes, _ = _c_encode_strings(combined.astype(object)) + elif combined.dtype.kind == "O": + codes = GroupSet._hash_encode_object(combined) else: _, codes = np.unique(combined, return_inverse=True) left_enc.append(codes[: len(lc)]) diff --git a/test/test_tafra.py b/test/test_tafra.py index 22d005c..b1923a1 100644 --- a/test/test_tafra.py +++ b/test/test_tafra.py @@ -1972,6 +1972,56 @@ def test_join_rejects_int_vs_string_keys(self) -> None: with pytest.raises(TypeError): left.inner_join(right, on=[("k", "k", "==")]) + def test_join_on_object_dtype_mixed_python_types_across_sides(self) -> None: + # Regression: np.unique on the concatenated key array used to call + # argsort, which raises TypeError when object arrays mix str and int. + left = Tafra( + { + "k": np.array(["a", "b", "c"], dtype=object), + "lv": np.array([1, 2, 3]), + } + ) + right = Tafra( + { + "k": np.array([1, 2, 3], dtype=object), + "rv": np.array([10, 20, 30]), + } + ) + t = left.left_join(right, on=[("k", "k", "==")]) + assert len(t) == 3 + # No matches since 'a' != 1 etc.; rv falls back to object/None + assert list(t["rv"]) == [None, None, None] + + def test_join_on_object_dtype_mixed_within_side(self) -> None: + left = Tafra( + { + "k": np.array(["a", 1, "c"], dtype=object), + "lv": np.array([1, 2, 3]), + } + ) + right = Tafra( + { + "k": np.array(["a", 1, "d"], dtype=object), + "rv": np.array([10, 20, 30]), + } + ) + t = left.left_join(right, on=[("k", "k", "==")]) + assert len(t) == 3 + assert list(t["rv"]) == [10, 20, None] + + def test_group_by_object_dtype_mixed_python_types(self) -> None: + t = Tafra( + { + "k": np.array(["a", 1, "a", 1, "b"], dtype=object), + "v": np.array([1.0, 2.0, 3.0, 4.0, 5.0]), + } + ) + g = t.group_by(["k"], {"v_sum": (sum, "v")}) + assert len(g) == 3 + # Hash-based encoding preserves insertion order: 'a', 1, 'b' + result = dict(zip(g["k"].tolist(), g["v_sum"].tolist())) + assert result == {"a": 4.0, 1: 6.0, "b": 5.0} + def test_concat_stringdtype_and_fixed_u(self) -> None: t1 = Tafra( {