From b2991ca10e19d4437aa41673d4303b796b05e923 Mon Sep 17 00:00:00 2001 From: virgesmith Date: Sat, 4 Jul 2026 09:33:23 +0100 Subject: [PATCH 1/2] Fix tee(0) contract, broken apidoc entry point, and doc errors - tee(n) now raises ValueError for n < 1 as its docstring has always promised (previously tee(0) silently discarded the iterator and returned an empty tuple) - remove the apidoc console script from [project.scripts]: the wheel does not package src/scripts, so every pip install shipped a broken apidoc command (ModuleNotFoundError). It is a dev-only tool; AGENTS.md now documents running it directly via uv - introspect.generate_apidoc now uses its cls parameter instead of hardcoding Itr - README: license badge pointed at the xenoform package instead of itrx; fix "accumulated" -> "accumulate" in the method list - docstring corrections (accumulate arg name, collect/next_chunk/skip/ skip_while/take_while return descriptions, map_dict type) and regenerated doc/apidoc.md Co-Authored-By: Claude Fable 5 --- AGENTS.md | 4 ++-- README.md | 4 ++-- doc/apidoc.md | 20 ++++++++++---------- pyproject.toml | 3 --- src/itrx/itr.py | 22 ++++++++++++---------- src/scripts/introspect.py | 6 +++--- src/test/test_combine_split.py | 6 ++++-- 7 files changed, 33 insertions(+), 32 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index dbb54c7..74cfc6c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -51,7 +51,7 @@ When reviewing a PR or diff, check: 5. **Doctests** — run them mentally; verify the output is correct and the example is illuminating. 6. **Types** — return types and generics should be precise. Avoid `Any` unless unavoidable. 7. **Ruff rules** — no rule in the `select` list should be suppressed without justification. The active rules are: `ARG, B, C, D103, E, F, I, N, PERF, PTH, RET, RUF, SIM, UP, W` (E501 is ignored). -8. **README / apidoc** — if a public method is added or its signature changes, update [README.md](README.md) and regenerate [doc/apidoc.md](doc/apidoc.md) with `uv run apidoc`. +8. **README / apidoc** — if a public method is added or its signature changes, update [README.md](README.md) and regenerate [doc/apidoc.md](doc/apidoc.md) with `uv run python src/scripts/introspect.py`. ## QA Rules @@ -99,7 +99,7 @@ pyproject.toml 3. Add or update tests in the matching file under [src/test/](src/test/). 4. Add or update the inline doctest on the method. 5. Run the full gate suite locally. -6. If the public API changed, update [README.md](README.md) and run `uv run apidoc`. +6. If the public API changed, update [README.md](README.md) and run `uv run python src/scripts/introspect.py`. 7. Commit — pre-commit hooks will auto-fix formatting and re-lock `uv.lock`. 8. Open a PR targeting `main`; CI must pass before merging. 9. To release: bump the version in [pyproject.toml](pyproject.toml), update [relnotes.md](relnotes.md), merge to `main`, then push a `vX.Y.Z` tag — PyPI publish triggers automatically. diff --git a/README.md b/README.md index e00b79c..0ab1b81 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ![PyPI - Version](https://img.shields.io/pypi/v/itrx) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/itrx) -![PyPI - License](https://img.shields.io/pypi/l/xenoform) +![PyPI - License](https://img.shields.io/pypi/l/itrx) `itrx` is a Python library that adapts iterators, iterables, and generators, providing a Rust-inspired `Iterator` trait experience with added Pythonic conveniences. It enables developers to build complex data processing pipelines with a fluent, chainable, and lazy API. In most cases, it simply wraps `itertools` and/or builtins in syntactic sugar. @@ -89,7 +89,7 @@ Note: Most `Itr` methods are **lazy transformations**, meaning they return a new `Itr` instance without immediately processing any data. This allows for arbitrary chaining and efficient memory usage, as items are only processed as they are requested. In most cases, `Itr` simply acts as a convenient wrapper around `itertools`, enabling this left-to-right chaining syntax. - **Combining and splitting:** `partition`, `copy`, `batched`, `pairwise`, `rolling`, `chain`, `cycle`, `repeat`, `product`, `inspect`, `intersperse`, `interleave`, `chunk_by` -- **Transformation and filtering:** `accumulated`, `filter`, `map`, `starmap`, `map_while`, `flatten`, `flat_map`, `skip_while`, `take_while` +- **Transformation and filtering:** `accumulate`, `filter`, `map`, `starmap`, `map_while`, `flatten`, `flat_map`, `skip_while`, `take_while` However, some methods are **eager consumers**. These methods iterate over and consume the underlying data, returning concrete values, collections, or aggregates. Examples include: diff --git a/doc/apidoc.md b/doc/apidoc.md index 027daba..43b4afb 100644 --- a/doc/apidoc.md +++ b/doc/apidoc.md @@ -28,7 +28,7 @@ not collapse the iterator like `reduce` or `fold` Args: func (Callable[[T, T], T] | None): A binary function to accumulate results. Defaults to addition. - initial_value: T | None: An optional starting value. If specified, this value will the the first element of + initial (T | None): An optional starting value. If specified, this value will be the first element of the resulting iterator Returns: @@ -117,10 +117,10 @@ Example: ### `collect` -Collect all remaining items from the iterator into a sequence (tuple by default). +Collect all remaining items from the iterator into a container (tuple by default). Returns: - tuple[T]: A list of all remaining items. + _CollectT: The remaining items collected into the given container type. @@ -344,7 +344,7 @@ Returns: Map each item in the iterator using the given dictionary (supports defaultdict). Args: - mapper (dict[T], U]): The lookup to apply. + mapper (dict[T, U]): The lookup to apply. Returns: Itr[U]: An iterator of mapped items. @@ -405,7 +405,7 @@ Returns: ### `next_chunk` -Return a list of the next n items from the iterator. +Return a tuple of the next n items from the iterator. Args: n (int): The number of items to yield. @@ -551,19 +551,19 @@ Args: n (int): The number of items to skip. Returns: - Self: The iterator itself. + Itr[T]: An iterator over the remaining items. ### `skip_while` -Skip items in the iterator as long as the predicate is true, returning self. +Skip items in the iterator as long as the predicate is true. Args: predicate (Callable[[T], bool]): A function to test each element. Returns: - Itr[T]: The iterator itself after skipping items. + Itr[T]: An iterator over the remaining items once the predicate first fails. @@ -610,13 +610,13 @@ Returns: ### `take_while` -Collects and returns items from the iterator as long as the given predicate is true. +Yield items from the iterator as long as the given predicate is true. Args: predicate (Callable[[T], bool]): A function that takes an item and returns True to continue taking items, or False to stop. Returns: - Self: A new Itr instance containing the items taken while the predicate was true. + Itr[T]: A new Itr yielding items while the predicate is true. diff --git a/pyproject.toml b/pyproject.toml index 74e7768..1ddd434 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,9 +38,6 @@ examples = [ "ipykernel", ] -[project.scripts] -apidoc = "scripts.introspect:main" - [tool.pytest.ini_options] testpaths = [ "." diff --git a/src/itrx/itr.py b/src/itrx/itr.py index ce3367d..303f400 100644 --- a/src/itrx/itr.py +++ b/src/itrx/itr.py @@ -38,7 +38,7 @@ def accumulate(self, func: Callable[[T, T], T] | None = None, *, initial: T | No Args: func (Callable[[T, T], T] | None): A binary function to accumulate results. Defaults to addition. - initial_value: T | None: An optional starting value. If specified, this value will the the first element of + initial (T | None): An optional starting value. If specified, this value will be the first element of the resulting iterator Returns: @@ -120,10 +120,10 @@ def collect(self, container: type[set[T]]) -> set[T]: ... def collect[K, V](self, container: type[dict[K, V]]) -> dict[K, V]: ... def collect(self, container: type[_CollectT] = tuple) -> _CollectT: # ty: ignore[invalid-parameter-default] - """Collect all remaining items from the iterator into a sequence (tuple by default). + """Collect all remaining items from the iterator into a container (tuple by default). Returns: - tuple[T]: A list of all remaining items. + _CollectT: The remaining items collected into the given container type. """ return container(self._it) @@ -405,7 +405,7 @@ def map_dict[U](self, mapper: dict[T, U]) -> "Itr[U]": """Map each item in the iterator using the given dictionary (supports defaultdict). Args: - mapper (dict[T], U]): The lookup to apply. + mapper (dict[T, U]): The lookup to apply. Returns: Itr[U]: An iterator of mapped items. @@ -466,7 +466,7 @@ def next(self) -> T: return next(self._it) def next_chunk(self, n: int) -> tuple[T, ...]: - """Return a list of the next n items from the iterator. + """Return a tuple of the next n items from the iterator. Args: n (int): The number of items to yield. @@ -624,19 +624,19 @@ def skip(self, n: int) -> "Itr[T]": n (int): The number of items to skip. Returns: - Self: The iterator itself. + Itr[T]: An iterator over the remaining items. """ return Itr(itertools.islice(self._it, n, None)) def skip_while(self, predicate: Predicate[T]) -> "Itr[T]": - """Skip items in the iterator as long as the predicate is true, returning self. + """Skip items in the iterator as long as the predicate is true. Args: predicate (Callable[[T], bool]): A function to test each element. Returns: - Itr[T]: The iterator itself after skipping items. + Itr[T]: An iterator over the remaining items once the predicate first fails. """ return Itr(itertools.dropwhile(predicate, self._it)) @@ -683,13 +683,13 @@ def take(self, n: int) -> "Itr[T]": return Itr(itertools.islice(self._it, n)) def take_while(self, predicate: Predicate[T]) -> "Itr[T]": - """Collects and returns items from the iterator as long as the given predicate is true. + """Yield items from the iterator as long as the given predicate is true. Args: predicate (Callable[[T], bool]): A function that takes an item and returns True to continue taking items, or False to stop. Returns: - Self: A new Itr instance containing the items taken while the predicate was true. + Itr[T]: A new Itr yielding items while the predicate is true. """ return Itr(itertools.takewhile(predicate, self._it)) @@ -731,6 +731,8 @@ def tee(self, n: int = 2) -> tuple["Itr[T]", ...]: >>> list(b) [0, 1, 2] """ + if n < 1: + raise ValueError(f"tee requires at least 1 iterator, got {n}") return tuple(Itr(t) for t in itertools.tee(self._it, n)) def unzip[U, V](self: "Itr[tuple[U, V]]") -> tuple["Itr[U]", "Itr[V]"]: diff --git a/src/scripts/introspect.py b/src/scripts/introspect.py index e228e69..0e2d1c1 100644 --- a/src/scripts/introspect.py +++ b/src/scripts/introspect.py @@ -18,12 +18,12 @@ def generate_apidoc(cls: type, file: Path) -> None: methods = ( Itr(dir(cls)) .filter(lambda m: m in ("__init__", "__iter__", "__next__") or not m.startswith("_")) - .map(lambda m: (m, getattr(Itr, m).__doc__)) + .map(lambda m: (m, getattr(cls, m).__doc__)) ) with file.open("w") as fd: - fd.write(f"# `Itr` v{itrx_version} class documentation\n") - fd.write(Itr.__doc__ or "") + fd.write(f"# `{cls.__name__}` v{itrx_version} class documentation\n") + fd.write(cls.__doc__ or "") fd.write("## Public methods\n") fd.write(methods.map(lambda m: method_template.format(method_name=m[0], method_doc=m[1])).reduce(add)) diff --git a/src/test/test_combine_split.py b/src/test/test_combine_split.py index 1febe25..f0d5209 100644 --- a/src/test/test_combine_split.py +++ b/src/test/test_combine_split.py @@ -407,8 +407,10 @@ def test_tee_returns_correct_number_of_iterators() -> None: def test_tee_raises_value_error_for_invalid_n() -> None: i = Itr([1, 2, 3]) - # with pytest.raises(ValueError): - assert i.tee(0) == () + with pytest.raises(ValueError): + i.tee(0) + # the underlying iterator is not consumed + assert i.collect() == (1, 2, 3) def test_tee_iterators_are_distinct_objects() -> None: From 78211a93062a41bc6f494e81885d2c301d62fdfb Mon Sep 17 00:00:00 2001 From: virgesmith Date: Sun, 5 Jul 2026 08:08:34 +0100 Subject: [PATCH 2/2] value_counts now behaves like pandas: most common first Reimplemented with Counter.most_common() instead of sort + groupby: results are ordered by descending count (ties by first appearance), items only need to be hashable rather than orderable, and counting is O(n) instead of O(n log n). Adds a doctest and tests for ordering and unorderable (complex) items; relnotes gain an Unreleased section covering this and the other behavioural changes in this PR. Co-Authored-By: Claude Fable 5 --- README.md | 2 +- doc/apidoc.md | 14 ++++++++++---- relnotes.md | 11 +++++++++++ src/itrx/itr.py | 18 ++++++++++++------ src/test/test_aggregation.py | 14 ++++++++++++++ 5 files changed, 48 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 0ab1b81..dc1a770 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ However, some methods are **eager consumers**. These methods iterate over and co * **Collection methods:** `collect`, `last`, `next`, `next_chunk`, `nth`, `position` * **Aggregation methods:** `count`, `reduce`, `max`, `min`, `all`, `any`, `consume`, `find`, `fold` -* **Sorting/grouping:** `groupby` and `value_counts` sort the entire input up front, so they consume the whole iterator immediately and must not be used on infinite sources. Use the lazy `chunk_by` to group consecutive runs without sorting. +* **Sorting/grouping:** `groupby` sorts the entire input up front, and `value_counts` counts it (most common first, like pandas), so both consume the whole iterator immediately and must not be used on infinite sources. Use the lazy `chunk_by` to group consecutive runs without sorting. ### Important Considerations diff --git a/doc/apidoc.md b/doc/apidoc.md index 43b4afb..2401bda 100644 --- a/doc/apidoc.md +++ b/doc/apidoc.md @@ -677,13 +677,19 @@ Note: ### `value_counts` -Returns an iterator over the number of times distinct items appear in the original iterator, which can be -collected into a dict. +Returns an iterator over the number of times distinct items appear in the original iterator, most common +first (like pandas' `value_counts`). Ties are ordered by first appearance. Items must be hashable, and the +result can be collected into a dict. -Do not use on an infinite iterator +This method is **eager**: it consumes the whole iterator immediately, so do not use it on an infinite +iterator. Returns: - Itr[tuple[T, int]]: An iterator of pairs of values and counts. + Itr[tuple[T, int]]: An iterator of (value, count) pairs in descending count order. + +Example: + >>> Itr("abracadabra").value_counts().collect() + (('a', 5), ('b', 2), ('r', 2), ('c', 1), ('d', 1)) ### `zip` diff --git a/relnotes.md b/relnotes.md index 110827d..53a4477 100644 --- a/relnotes.md +++ b/relnotes.md @@ -1,3 +1,14 @@ +## Unreleased + +### Breaking changes + +- `value_counts` now behaves like pandas' `value_counts`: results are ordered most-common-first (ties by first appearance) instead of sorted by key. Items now only need to be hashable rather than orderable, and counting is O(n) rather than O(n log n). +- `tee` now raises `ValueError` when `n < 1`, as its docstring has always stated (previously `tee(0)` silently discarded the iterator and returned an empty tuple). + +### Bug fixes + +- Removed the broken `apidoc` console script from the distribution: the wheel does not package the `scripts` module, so the installed command always failed with `ModuleNotFoundError`. It is a dev-only tool, now run directly from the source tree. + ## 0.3.0 ### Breaking changes diff --git a/src/itrx/itr.py b/src/itrx/itr.py index 303f400..cd1ad2e 100644 --- a/src/itrx/itr.py +++ b/src/itrx/itr.py @@ -1,5 +1,5 @@ import itertools -from collections import deque +from collections import Counter, deque from collections.abc import Callable, Generator, Iterable, Iterator from typing import Any, TypeVar, cast, overload @@ -753,15 +753,21 @@ def unzip[U, V](self: "Itr[tuple[U, V]]") -> tuple["Itr[U]", "Itr[V]"]: def value_counts(self) -> "Itr[tuple[T, int]]": """ - Returns an iterator over the number of times distinct items appear in the original iterator, which can be - collected into a dict. + Returns an iterator over the number of times distinct items appear in the original iterator, most common + first (like pandas' `value_counts`). Ties are ordered by first appearance. Items must be hashable, and the + result can be collected into a dict. - Do not use on an infinite iterator + This method is **eager**: it consumes the whole iterator immediately, so do not use it on an infinite + iterator. Returns: - Itr[tuple[T, int]]: An iterator of pairs of values and counts. + Itr[tuple[T, int]]: An iterator of (value, count) pairs in descending count order. + + Example: + >>> Itr("abracadabra").value_counts().collect() + (('a', 5), ('b', 2), ('r', 2), ('c', 1), ('d', 1)) """ - return self.groupby(lambda x: x).map(lambda x: (x[0], len(x[1]))) + return cast("Itr[tuple[T, int]]", Itr(Counter(self._it).most_common())) def zip[U](self, other: Iterable[U]) -> "Itr[tuple[T, U]]": """Yield pairs of items from this iterator and another iterable. diff --git a/src/test/test_aggregation.py b/src/test/test_aggregation.py index 6ae1f11..ba1ec49 100644 --- a/src/test/test_aggregation.py +++ b/src/test/test_aggregation.py @@ -141,6 +141,20 @@ def test_value_counts_basic() -> None: assert result == {1: 2, 2: 3, 3: 1, 4: 1} +def test_value_counts_most_common_first() -> None: + # like pandas: descending count, ties in first-appearance order + data = [1, 2, 2, 3, 1, 4, 2] + result = Itr(data).value_counts().collect() + assert result == ((2, 3), (1, 2), (3, 1), (4, 1)) + + +def test_value_counts_unorderable_items() -> None: + # items only need to be hashable, not orderable + data = [1j, 2j, 1j] + result = Itr(data).value_counts().collect() + assert result == ((1j, 2), (2j, 1)) + + def test_value_counts_empty() -> None: data: list[int] = [] result = Itr(data).value_counts().collect(dict)