diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 450cdf30..8d5ee162 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -19,6 +19,10 @@ env: PYTHONHASHSEED: "42" BASELINE_RELEASE_TAG: microbenchmarks PR_BENCHMARK_COMPARE_FAIL: min:5% + PR_BENCHMARK_COMMENT_MAX_ROWS: "8" + PR_BENCHMARK_COMMENT_MIN_CHANGE_PCT: "1" + PR_BENCHMARK_COMMENT_MIN_BASELINE_MS: "1" + PR_BENCHMARK_COMMENT_MIN_DELTA_MS: "0.1" jobs: benchmark: @@ -72,6 +76,7 @@ jobs: git checkout ${{ github.sha }} # move aside the '"'"'bidict'"'"' subdirectory to make sure we always import the installed version mv -v bidict src + python -c "from bidict import _native; assert _native.build_bidict_maps is not None; assert _native.build_bidict_maps_from_mapping is not None; assert _native.update_bidict_maps is not None; assert _native.update_bidict_maps_from_mapping is not None" curl -L -s -o baseline.json "${{ steps.metadata.outputs.baseline_url }}" line1=$(head -n1 baseline.json) [ "$line1" = "{" ] @@ -80,6 +85,7 @@ jobs: compare_fail_args+=(--benchmark-compare-fail="$COMPARE_FAIL_EXPR") fi ./cachegrind.py pytest -c /dev/null -n0 \ + --benchmark-json benchmark.json \ --benchmark-autosave \ --benchmark-columns=min,rounds,iterations \ --benchmark-disable-gc \ @@ -130,6 +136,10 @@ jobs: if: always() && github.event_name == 'pull_request' env: BASELINE_ASSET_NAME: ${{ steps.metadata.outputs.baseline_asset_name }} + COMMENT_MAX_ROWS: ${{ env.PR_BENCHMARK_COMMENT_MAX_ROWS }} + COMMENT_MIN_CHANGE_PCT: ${{ env.PR_BENCHMARK_COMMENT_MIN_CHANGE_PCT }} + COMMENT_MIN_BASELINE_MS: ${{ env.PR_BENCHMARK_COMMENT_MIN_BASELINE_MS }} + COMMENT_MIN_DELTA_MS: ${{ env.PR_BENCHMARK_COMMENT_MIN_DELTA_MS }} PR_NUMBER: ${{ github.event.pull_request.number }} RESULT_MESSAGE: ${{ steps.benchmark.outputs.result_message }} RESULT_STATE: ${{ steps.benchmark.outputs.result_state }} @@ -165,6 +175,98 @@ jobs: import os from pathlib import Path + baseline_path = Path('baseline.json') + benchmark_path = Path('benchmark.json') + comment_path = Path('benchmark-pr-comment/comment.md') + max_rows = int(os.environ['COMMENT_MAX_ROWS']) + min_change_pct = float(os.environ['COMMENT_MIN_CHANGE_PCT']) + min_baseline_ms = float(os.environ['COMMENT_MIN_BASELINE_MS']) + min_delta_ms = float(os.environ['COMMENT_MIN_DELTA_MS']) + + def bench_key(bench: dict[str, object]) -> str: + return str(bench['name']) + + def load_benchmark_mins(path: Path) -> dict[str, float]: + data = json.loads(path.read_text()) + return {bench_key(bench): float(bench['stats']['min']) for bench in data['benchmarks']} + + def format_row(name: str, baseline: float, current: float, delta_pct: float) -> str: + return ( + f"| `{name}` | {baseline * 1000:.3f} ms | {current * 1000:.3f} ms | {delta_pct:+.2f}% |" + ) + + if baseline_path.exists() and benchmark_path.exists(): + baseline_mins = load_benchmark_mins(baseline_path) + benchmark_mins = load_benchmark_mins(benchmark_path) + shared = [] + for name, current in benchmark_mins.items(): + baseline = baseline_mins.get(name) + if baseline is None or baseline == 0: + continue + baseline_ms = baseline * 1000 + delta_ms = abs(current - baseline) * 1000 + if baseline_ms < min_baseline_ms or delta_ms < min_delta_ms: + continue + delta_pct = ((current - baseline) / baseline) * 100 + shared.append((name, baseline, current, delta_pct)) + + improvements = [ + entry for entry in shared if entry[3] <= -min_change_pct + ] + regressions = [ + entry for entry in shared if entry[3] >= min_change_pct + ] + improvements.sort(key=lambda entry: entry[3]) + regressions.sort(key=lambda entry: entry[3], reverse=True) + + lines = comment_path.read_text().rstrip().splitlines() + lines.extend([ + '', + ( + f'_Deltas below compare benchmark `min` time against ' + f'`{os.environ["BASELINE_ASSET_NAME"]}` for benchmarks with baseline >= ' + f'{min_baseline_ms:g} ms and absolute delta >= {min_delta_ms:g} ms._' + ), + ]) + + if improvements: + lines.extend([ + '', + '### Notable improvements', + '', + '| Benchmark | Baseline | PR | Delta |', + '| --- | ---: | ---: | ---: |', + *( + format_row(name, baseline, current, delta_pct) + for name, baseline, current, delta_pct in improvements[:max_rows] + ), + ]) + + if regressions: + lines.extend([ + '', + '### Notable regressions', + '', + '| Benchmark | Baseline | PR | Delta |', + '| --- | ---: | ---: | ---: |', + *( + format_row(name, baseline, current, delta_pct) + for name, baseline, current, delta_pct in regressions[:max_rows] + ), + ]) + + if not improvements and not regressions: + lines.extend([ + '', + ( + f'_No benchmark deltas meeting the reporting thresholds ' + f'({min_change_pct:.0f}% change, {min_baseline_ms:g} ms baseline, ' + f'{min_delta_ms:g} ms absolute delta) to report._' + ), + ]) + + comment_path.write_text('\n'.join(lines) + '\n') + payload = { 'pr_number': os.environ['PR_NUMBER'], 'result_state': os.environ['RESULT_STATE'], @@ -189,6 +291,8 @@ jobs: name: microbenchmark results (CPython ${{ steps.metadata.outputs.python_version }}) path: | .benchmarks + baseline.json + benchmark.json benchmark-output.txt include-hidden-files: true if-no-files-found: error diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c417f970..4cfa73c7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,16 +36,21 @@ jobs: - python: "3.14" shell: test314 enable_coverage: true + enable_native: true - python: "3.13" shell: test313 + enable_native: true - python: "3.12" shell: test312 + enable_native: true - python: "3.11" shell: test311 # bidict/_typing.py's `typing`/`typing_extensions` imports branch on python<3.12 enable_coverage: true + enable_native: true - python: "pypy-3.11" shell: testPyPy311 + enable_native: false steps: - name: check out source uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd @@ -69,6 +74,12 @@ jobs: if: matrix.enable_coverage run: | echo RUN_PYTEST_CMD="coverage run -m pytest" >> "$GITHUB_ENV" + - name: verify native helper availability + if: matrix.enable_native + run: | + nix develop .#${{ matrix.shell }} --command bash -c ' + python -c "from bidict import _native; assert _native.build_bidict_maps is not None; assert _native.build_bidict_maps_from_mapping is not None; assert _native.update_bidict_maps is not None; assert _native.update_bidict_maps_from_mapping is not None" + ' - name: run mypy and pytest run: | nix develop .#${{ matrix.shell }} --command bash -c " diff --git a/.github/workflows/update_benchmark_baselines.yml b/.github/workflows/update_benchmark_baselines.yml index 7953af78..31c0cfc7 100644 --- a/.github/workflows/update_benchmark_baselines.yml +++ b/.github/workflows/update_benchmark_baselines.yml @@ -59,6 +59,7 @@ jobs: uv pip install --python "$VIRTUAL_ENV/bin/python" --no-deps . # Move aside the working tree package so we always import the installed version. mv -v bidict src + python -c "from bidict import _native; assert _native.build_bidict_maps is not None; assert _native.build_bidict_maps_from_mapping is not None; assert _native.update_bidict_maps is not None; assert _native.update_bidict_maps_from_mapping is not None" ./cachegrind.py pytest -c /dev/null -n0 \ --benchmark-autosave \ --benchmark-columns=min,rounds,iterations \ diff --git a/.gitignore b/.gitignore index b1190e8e..7d3c3202 100644 --- a/.gitignore +++ b/.gitignore @@ -10,9 +10,13 @@ __pycache__ .hypothesis .idea .mypy_cache +.pytest_cache +.ruff_cache .tox .venv +.venv-* .venv-benchmark +.jump bidict.egg-info build _build @@ -20,3 +24,4 @@ coverage.xml dist htmlcov pip-wheel-metadata +target diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fb8522f3..ec88dee3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -58,3 +58,30 @@ repos: hooks: - id: shellcheck exclude: ^.envrc$ + + - repo: local + hooks: + - id: cargo-fmt + name: cargo fmt + entry: cargo fmt --manifest-path rust/bidict_base_opt_native/Cargo.toml --check + language: system + files: ^rust/bidict_base_opt_native/ + pass_filenames: false + + - id: cargo-check + name: cargo check + entry: cargo check --manifest-path rust/bidict_base_opt_native/Cargo.toml --locked + language: system + files: ^rust/bidict_base_opt_native/ + pass_filenames: false + + - id: cargo-clippy + name: cargo clippy + entry: >- + bash -c 'export RUSTC="$(command -v rustc)"; + export RUSTC_WORKSPACE_WRAPPER="$(command -v clippy-driver)"; + export RUSTFLAGS="-D warnings"; + cargo check --manifest-path rust/bidict_base_opt_native/Cargo.toml --locked --all-targets' + language: system + files: ^rust/bidict_base_opt_native/ + pass_filenames: false diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index f6efe08c..68db550e 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -44,14 +44,18 @@ Making Changes - If you have `Nix `__, run ``nix develop`` from within your clone to start a shell where all supported - Python versions as well as ``prek`` are installed and added - to your PATH. This also pins ``uv`` to the flake-provided - default Python when initializing ``.venv``. + Python versions as well as ``prek``, ``uv``, ``rustc``, + ``cargo``, ``rustfmt``, ``cargo-clippy``, and ``maturin`` + are installed and added to your PATH. + This also pins ``uv`` to the flake-provided default Python + when initializing ``.venv``. - Otherwise, manually ensure you have `uv `__, - `prek `__, and at least the latest - `stable Python version `__ installed - and on your PATH. + `prek `__, + `Rust `__ (including + ``cargo`` and ``rustc``), `maturin `__, + and at least the latest `stable Python version `__ + installed and on your PATH. - Run ``./init_dev_env`` diff --git a/benchmark_linux_arm64.sh b/benchmark_linux_arm64.sh new file mode 100755 index 00000000..e150038c --- /dev/null +++ b/benchmark_linux_arm64.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# +# Copyright 2009-2026 Joshua Bronson. All rights reserved. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +set -euo pipefail + +declare -r container_image='rust:1.95-bookworm' + +if ! command -v container >/dev/null 2>&1; then + >&2 echo "Error: No 'container' command on PATH." + exit 1 +fi + +repo_root=$( + cd -- "$(dirname -- "${BASH_SOURCE[0]}")" + pwd +) + +# shellcheck disable=SC2016 +container run --rm --progress plain \ + -v "${repo_root}:/work" \ + -w /work \ + "${container_image}" \ + bash -c ' + set -euo pipefail + if [ -n "${1:-}" ]; then + export BIDICT_DISABLE_NATIVE="$1" + fi + export DEBIAN_FRONTEND=noninteractive + export PYTHONHASHSEED=42 + export UV_LINK_MODE=copy + export UV_PROJECT_ENVIRONMENT=/tmp/bidict-bench-venv + + apt-get update >/dev/null + apt-get install -y \ + build-essential \ + ca-certificates \ + python3 \ + python3-dev \ + python3-pip \ + python3-venv \ + util-linux \ + valgrind \ + >/dev/null + python3 -m pip install --break-system-packages uv >/dev/null + + rustc --version + cargo --version + + uv sync --all-groups --frozen >/dev/null + . /tmp/bidict-bench-venv/bin/activate + + python - <<"PY" +import os + +import bidict._native as native + +disabled = os.getenv("BIDICT_DISABLE_NATIVE") +if disabled: + assert native.build_bidict_maps is None and native.update_bidict_maps is None + print(f"native helper: disabled via BIDICT_DISABLE_NATIVE={disabled}") +else: + assert native.build_bidict_maps is not None + print("native helper:", native.build_bidict_maps.__module__) +PY + + ./cachegrind.py python -m pytest -c /dev/null -n0 \ + --benchmark-columns=min,rounds,iterations \ + --benchmark-disable-gc \ + --benchmark-group-by=name \ + microbenchmarks.py + ' bash "${BIDICT_DISABLE_NATIVE-}" diff --git a/bidict/_base.py b/bidict/_base.py index 2de3cc58..5c51ca27 100644 --- a/bidict/_base.py +++ b/bidict/_base.py @@ -41,6 +41,10 @@ from ._exc import ValueDuplicationError from ._iter import inverted from ._iter import iteritems +from ._native import build_bidict_maps as _build_bidict_maps +from ._native import build_bidict_maps_from_mapping as _build_bidict_maps_from_mapping +from ._native import update_bidict_maps as _update_bidict_maps +from ._native import update_bidict_maps_from_mapping as _update_bidict_maps_from_mapping from ._typing import KT from ._typing import MISSING from ._typing import OKT @@ -55,6 +59,32 @@ DedupResult: t.TypeAlias = OldKV[KT, VT] | None Unwrites: t.TypeAlias = list[tuple[t.Any, ...]] ReversedIter: t.TypeAlias = t.Callable[['BidictBase[KT, t.Any]'], Iterator[KT]] +_MIN_NATIVE_UPDATE_ITEMS = 8192 +_MIN_NATIVE_FORCEUPDATE_ITEMS = 4096 +_MAX_NATIVE_DUPVAL_FAST_FAIL_ITEMS = 64 + + +def _native_items(arg: MapOrItems[KT, VT], kw: Mapping[str, VT]) -> Iterable[tuple[KT, VT]]: + if not kw and isinstance(arg, Mapping): + return arg.items() + return iteritems(arg, **kw) + + +def _supports_native_mapping(arg: MapOrItems[KT, VT], kw: Mapping[str, VT]) -> bool: + return not kw and isinstance(arg, Mapping) + + +def _prescan_mapping_dupvals(mapping: Mapping[KT, VT], max_items: int | None = None) -> None: + seen_by_val: dict[VT, KT] = {} + seen_get = seen_by_val.get + for index, (key, val) in enumerate(mapping.items()): + if max_items is not None and index >= max_items: + return + prev_key = seen_get(val, MISSING) + if prev_key is MISSING: + seen_by_val[val] = key + elif prev_key != key: + raise ValueDuplicationError(val) class BidictKeysView(KeysView[KT], ValuesView[KT]): @@ -211,6 +241,32 @@ def _make_inverse(self) -> BidictBase[VT, KT]: inv._invm = self._fwdm return inv + def _set_map_data(self, fwdm: MutableMapping[KT, VT], invm: MutableMapping[VT, KT]) -> None: + """Replace our backing maps, preserving any already-materialized inverse instance.""" + if getattr(self, '_inv', None) is None and getattr(self, '_invweak', None) is None: + self._fwdm = fwdm + self._invm = invm + return + self._fwdm.clear() + self._invm.clear() + self._fwdm.update(fwdm) + self._invm.update(invm) + + def _supports_native_map_swap(self) -> bool: + if type(self)._write is not BidictBase._write: + return False + return self._fwdm_cls is dict and self._invm_cls is dict + + def _should_use_native_update(self, incoming_len: int | None, on_dup: OnDup) -> bool: + if incoming_len is None or not self: + return False + if not self._supports_native_map_swap(): + return False + if _update_bidict_maps is None and _update_bidict_maps_from_mapping is None: + return False + min_items = _MIN_NATIVE_FORCEUPDATE_ITEMS if on_dup.val is DROP_OLD else _MIN_NATIVE_UPDATE_ITEMS + return incoming_len >= min(len(self), min_items) + @property def inv(self) -> BidictBase[VT, KT]: """Alias for :attr:`inverse`.""" @@ -443,15 +499,39 @@ def _update( on_dup = self.on_dup if rollback is None: rollback = RAISE in on_dup + incoming_len = len(arg) + len(kw) if isinstance(arg, t.Sized) else None # Fast path when we're empty and updating only from another bidict (i.e. no dup vals in new items). if not self and not kw and isinstance(arg, BidictBase): self._init_from(arg) return + if not self and self._supports_native_map_swap(): + if _supports_native_mapping(arg, kw) and _build_bidict_maps_from_mapping is not None: + mapping_arg = t.cast(Mapping[t.Any, t.Any], arg) + if on_dup.val is RAISE: + _prescan_mapping_dupvals(mapping_arg, _MAX_NATIVE_DUPVAL_FAST_FAIL_ITEMS) + self._set_map_data(*_build_bidict_maps_from_mapping(mapping_arg, on_dup)) + return + if _build_bidict_maps is not None: + self._set_map_data(*_build_bidict_maps(_native_items(arg, kw), on_dup)) + return + + if self._should_use_native_update(incoming_len, on_dup): + fwdm = t.cast(dict[t.Any, t.Any], self._fwdm) + invm = t.cast(dict[t.Any, t.Any], self._invm) + if _supports_native_mapping(arg, kw) and _update_bidict_maps_from_mapping is not None: + mapping_arg = t.cast(Mapping[t.Any, t.Any], arg) + self._set_map_data(*_update_bidict_maps_from_mapping(fwdm, invm, mapping_arg, on_dup)) + return + native_update = _update_bidict_maps + assert native_update is not None + self._set_map_data(*native_update(fwdm, invm, _native_items(arg, kw), on_dup)) + return + # Fast path when we're adding more items than we contain already and rollback is enabled: # Update a copy of self with rollback disabled. Fail if that fails, otherwise become the copy. - if rollback and isinstance(arg, t.Sized) and len(arg) + len(kw) > len(self): + if rollback and incoming_len is not None and incoming_len > len(self): tmp = self.copy() tmp._update(arg, kw, rollback=False, on_dup=on_dup) self._init_from(tmp) diff --git a/bidict/_native.py b/bidict/_native.py new file mode 100644 index 00000000..3fb9128b --- /dev/null +++ b/bidict/_native.py @@ -0,0 +1,140 @@ +# Copyright 2009-2026 Joshua Bronson. All rights reserved. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +"""Optional native helpers for accelerating selected bidict operations.""" + +from __future__ import annotations + +import os +import sys +import typing as t +from collections.abc import Iterable +from collections.abc import Mapping + +from ._dup import OnDup + + +_DISABLE_NATIVE_ENVVAR = 'BIDICT_DISABLE_NATIVE' +_DISABLE_NATIVE_TRUE_VALUES = frozenset({'1', 'true', 'yes'}) +Items = Iterable[tuple[t.Any, t.Any]] +BuildBidictMaps: t.TypeAlias = t.Callable[[Items, OnDup], tuple[dict[t.Any, t.Any], dict[t.Any, t.Any]]] +UpdateBidictMaps: t.TypeAlias = t.Callable[ + [dict[t.Any, t.Any], dict[t.Any, t.Any], Items, OnDup], + tuple[dict[t.Any, t.Any], dict[t.Any, t.Any]], +] +BuildBidictMapsFromMapping: t.TypeAlias = t.Callable[ + [Mapping[t.Any, t.Any], OnDup], tuple[dict[t.Any, t.Any], dict[t.Any, t.Any]] +] +UpdateBidictMapsFromMapping: t.TypeAlias = t.Callable[ + [dict[t.Any, t.Any], dict[t.Any, t.Any], Mapping[t.Any, t.Any], OnDup], + tuple[dict[t.Any, t.Any], dict[t.Any, t.Any]], +] +build_bidict_maps: BuildBidictMaps | None +update_bidict_maps: UpdateBidictMaps | None +build_bidict_maps_from_mapping: BuildBidictMapsFromMapping | None +update_bidict_maps_from_mapping: UpdateBidictMapsFromMapping | None + + +def _native_disabled() -> bool: + value = os.getenv(_DISABLE_NATIVE_ENVVAR) + return value is not None and value.lower() in _DISABLE_NATIVE_TRUE_VALUES + + +def _native_supported_runtime() -> bool: + return sys.implementation.name == 'cpython' + + +if t.TYPE_CHECKING: + build_bidict_maps = None + update_bidict_maps = None + build_bidict_maps_from_mapping = None + update_bidict_maps_from_mapping = None + + def _build_bidict_maps_impl( + items: Items, + on_dup_key: str, + on_dup_val: str, + ) -> tuple[dict[t.Any, t.Any], dict[t.Any, t.Any]]: ... + + def _update_bidict_maps_impl( + fwd: dict[t.Any, t.Any], + inv: dict[t.Any, t.Any], + items: Items, + on_dup_key: str, + on_dup_val: str, + ) -> tuple[dict[t.Any, t.Any], dict[t.Any, t.Any]]: ... + + def _build_bidict_maps_from_mapping_impl( + mapping: Mapping[t.Any, t.Any], + on_dup_key: str, + on_dup_val: str, + ) -> tuple[dict[t.Any, t.Any], dict[t.Any, t.Any]]: ... + + def _update_bidict_maps_from_mapping_impl( + fwd: dict[t.Any, t.Any], + inv: dict[t.Any, t.Any], + mapping: Mapping[t.Any, t.Any], + on_dup_key: str, + on_dup_val: str, + ) -> tuple[dict[t.Any, t.Any], dict[t.Any, t.Any]]: ... + +else: + if _native_disabled() or not _native_supported_runtime(): + build_bidict_maps: BuildBidictMaps | None = None + update_bidict_maps: UpdateBidictMaps | None = None + build_bidict_maps_from_mapping: BuildBidictMapsFromMapping | None = None + update_bidict_maps_from_mapping: UpdateBidictMapsFromMapping | None = None + else: + try: + from bidict_base_opt_native import bidict_base_opt_native as _native_ext + except ModuleNotFoundError as exc: + if exc.name != 'bidict_base_opt_native': + raise + build_bidict_maps = None + update_bidict_maps = None + build_bidict_maps_from_mapping = None + update_bidict_maps_from_mapping = None + else: + _build_bidict_maps_impl = _native_ext.build_bidict_maps + _update_bidict_maps_impl = getattr(_native_ext, 'update_bidict_maps', None) + _build_bidict_maps_from_mapping_impl = getattr(_native_ext, 'build_bidict_maps_from_mapping', None) + _update_bidict_maps_from_mapping_impl = getattr(_native_ext, 'update_bidict_maps_from_mapping', None) + + def build_bidict_maps(items: Items, on_dup: OnDup) -> tuple[dict[t.Any, t.Any], dict[t.Any, t.Any]]: + return _build_bidict_maps_impl(items, on_dup.key.name, on_dup.val.name) + + if _build_bidict_maps_from_mapping_impl is None: + build_bidict_maps_from_mapping = None + else: + + def build_bidict_maps_from_mapping( + mapping: Mapping[t.Any, t.Any], on_dup: OnDup + ) -> tuple[dict[t.Any, t.Any], dict[t.Any, t.Any]]: + return _build_bidict_maps_from_mapping_impl(mapping, on_dup.key.name, on_dup.val.name) + + if _update_bidict_maps_impl is None: + update_bidict_maps = None + else: + + def update_bidict_maps( + fwd: dict[t.Any, t.Any], + inv: dict[t.Any, t.Any], + items: Items, + on_dup: OnDup, + ) -> tuple[dict[t.Any, t.Any], dict[t.Any, t.Any]]: + return _update_bidict_maps_impl(fwd, inv, items, on_dup.key.name, on_dup.val.name) + + if _update_bidict_maps_from_mapping_impl is None: + update_bidict_maps_from_mapping = None + else: + + def update_bidict_maps_from_mapping( + fwd: dict[t.Any, t.Any], + inv: dict[t.Any, t.Any], + mapping: Mapping[t.Any, t.Any], + on_dup: OnDup, + ) -> tuple[dict[t.Any, t.Any], dict[t.Any, t.Any]]: + return _update_bidict_maps_from_mapping_impl(fwd, inv, mapping, on_dup.key.name, on_dup.val.name) diff --git a/cachegrind.py b/cachegrind.py index 9524b427..db82e7aa 100755 --- a/cachegrind.py +++ b/cachegrind.py @@ -48,7 +48,7 @@ DISABLE_ASLR_CMD = ['setarch', ARCH, '-R'] -def run_with_cachegrind(args_list: list[str]) -> dict[str, int]: +def run_with_cachegrind(args_list: list[str]) -> tuple[dict[str, int], int]: """ Run the the given program and arguments under Cachegrind, parse the Cachegrind specs. @@ -57,21 +57,24 @@ def run_with_cachegrind(args_list: list[str]) -> dict[str, int]: """ temp_file = NamedTemporaryFile('r+') # noqa: SIM115 # Don't fail if the program fails (to support e.g. `pytest --benchmark-compare-fail=...`) - sp.call([ - *DISABLE_ASLR_CMD, - 'valgrind', - '--tool=cachegrind', - '--cache-sim=yes', - # Set some reasonable L1 and LL values, based on Haswell. - # Feel free to update, important part is that they are consistent across runs, - # instead of the default of copying from the current machine. - '--I1=32768,8,64', - '--D1=32768,8,64', - '--LL=8388608,16,64', - '--cachegrind-out-file=' + temp_file.name, - *args_list, - ]) - return parse_cachegrind_output(temp_file) + completed = sp.run( + [ + *DISABLE_ASLR_CMD, + 'valgrind', + '--tool=cachegrind', + '--cache-sim=yes', + # Set some reasonable L1 and LL values, based on Haswell. + # Feel free to update, important part is that they are consistent across runs, + # instead of the default of copying from the current machine. + '--I1=32768,8,64', + '--D1=32768,8,64', + '--LL=8388608,16,64', + '--cachegrind-out-file=' + temp_file.name, + *args_list, + ], + check=False, + ) + return parse_cachegrind_output(temp_file), completed.returncode def parse_cachegrind_output(temp_file: t.IO[str]) -> dict[str, int]: @@ -128,10 +131,11 @@ def combined_instruction_estimate(counts: dict[str, int]) -> int: def main() -> None: - results = run_with_cachegrind(sys.argv[1:]) + results, exit_code = run_with_cachegrind(sys.argv[1:]) counts = get_counts(results) estimate = combined_instruction_estimate(counts) print(f'{"*" * 80}\nCombined instruction estimate: {estimate:,}') # noqa: T201 + raise SystemExit(exit_code) if __name__ == '__main__': diff --git a/flake.nix b/flake.nix index 0d4ddcd3..50c90d07 100644 --- a/flake.nix +++ b/flake.nix @@ -12,7 +12,11 @@ pkgs = import nixpkgs { inherit system; }; lib = pkgs.lib; latestPython = pkgs.python314; - commonTools = with pkgs; [prek uv]; + baseDevTools = with pkgs; [prek uv]; + rustDevTools = with pkgs; [cargo rustc rustfmt clippy maturin]; + allDevTools = baseDevTools ++ rustDevTools; + nativePackageName = "bidict-base-opt-native"; + nativeReinstallArg = "--reinstall-package=${nativePackageName}"; supportedPythons = with pkgs; [ python314 python313 @@ -52,7 +56,7 @@ extraShellHook ? "", }: let - packages = commonTools ++ [python] ++ extraPackages; + packages = baseDevTools ++ [python] ++ extraPackages; in pkgs.mkShell { inherit packages; @@ -61,11 +65,19 @@ }; }; - mkTestShell = { python, projectEnv }: + mkTestShell = { + python, + projectEnv, + enableNative ? false, + }: mkUvShell { inherit python projectEnv; - syncArgs = "--only-group=test"; + syncArgs = + if enableNative + then "--only-group=test --only-group=native ${nativeReinstallArg}" + else "--only-group=test"; activate = true; + extraPackages = lib.optionals enableNative rustDevTools; extraShellHook = '' uv pip install --python "$UV_PROJECT_ENVIRONMENT/bin/python" --no-deps -e . ''; @@ -74,7 +86,7 @@ devShells = { default = let - packages = commonTools ++ supportedPythons; + packages = allDevTools ++ supportedPythons; in pkgs.mkShell { inherit packages; @@ -91,31 +103,37 @@ benchmark = mkUvShell { python = latestPython; projectEnv = ".venv-benchmark"; - syncArgs = "--only-group=test"; + syncArgs = "--only-group=test --only-group=native ${nativeReinstallArg}"; activate = true; + extraPackages = rustDevTools; }; build = mkUvShell { python = pkgs.python313; + extraPackages = rustDevTools; }; lint = pkgs.mkShell { - packages = with pkgs; [prek]; - shellHook = mkPathPrefix [pkgs.prek]; + packages = allDevTools; + shellHook = mkPathPrefix allDevTools; }; test311 = mkTestShell { python = pkgs.python311; projectEnv = ".venv-test-3.11"; + enableNative = true; }; test312 = mkTestShell { python = pkgs.python312; projectEnv = ".venv-test-3.12"; + enableNative = true; }; test313 = mkTestShell { python = pkgs.python313; projectEnv = ".venv-test-3.13"; + enableNative = true; }; test314 = mkTestShell { python = pkgs.python314; projectEnv = ".venv-test-3.14"; + enableNative = true; }; testPyPy311 = mkTestShell { python = pkgs.pypy3; @@ -123,6 +141,7 @@ }; update_deps = mkUvShell { python = latestPython; + extraPackages = rustDevTools; }; }; }); diff --git a/init_dev_env b/init_dev_env index ea31df74..7134ac23 100755 --- a/init_dev_env +++ b/init_dev_env @@ -12,7 +12,7 @@ set -euo pipefail declare -r hint="Hint: Use 'nix develop' to bootstrap a development environment" -for cmd in uv prek; do +for cmd in uv prek rustc cargo rustfmt cargo-clippy clippy-driver maturin; do if ! command -v "$cmd" >/dev/null 2>&1; then >&2 echo "Error: No '$cmd' on PATH. $hint" exit 1 @@ -20,5 +20,5 @@ for cmd in uv prek; do done prek install -f -uv sync --all-groups +uv sync --all-groups --reinstall-package bidict-base-opt-native echo "Development virtualenv initialized" diff --git a/microbenchmarks.py b/microbenchmarks.py index 8f908165..6a0f04c4 100644 --- a/microbenchmarks.py +++ b/microbenchmarks.py @@ -77,6 +77,10 @@ PARTIAL_OVERLAP_RESULTS_BY_LEN: dict[int, dict[int, int]] = { n: (INT_DICTS_BY_LEN[n] | PARTIAL_OVERLAP_UPDATES_BY_LEN[n]) for n in LENS } +FORCEUPDATE_EXISTING_VALUES_UPDATES_BY_LEN: dict[int, dict[int, int]] = {n: {n + i: i for i in range(n)} for n in LENS} +FORCEUPDATE_EXISTING_VALUES_RESULTS_BY_LEN: dict[int, dict[int, int]] = { + n: FORCEUPDATE_EXISTING_VALUES_UPDATES_BY_LEN[n].copy() for n in LENS +} BIDICT_AND_DICT_LAST_TWO_ITEMS_DIFFERENT_ORDER: dict[int, tuple[bidict.bidict[int, int], dict[int, int]]] = {} ORDERED_BIDICT_AND_DICT_LAST_TWO_ITEMS_DIFFERENT_ORDER: dict[ @@ -107,6 +111,10 @@ def _update(bi: bidict.bidict[int, int], other: dict[int, int], _expected: dict[ bi.update(other) +def _forceupdate(bi: bidict.bidict[int, int], other: dict[int, int], _expected: dict[int, int]) -> None: + bi.forceupdate(other) + + def _failing_update(bi: bidict.bidict[int, int], other: dict[int, int], _expected: dict[int, int]) -> None: with pytest.raises(bidict.DuplicationError): bi.update(other) @@ -173,6 +181,28 @@ def _setup_update_partial_overlap(n: int) -> tuple[tuple[t.Any, ...], dict[str, ) +def _setup_forceupdate_partial_overlap(n: int) -> tuple[tuple[t.Any, ...], dict[str, t.Any]]: + return ( + ( + INT_BIDICTS_BY_LEN[n].copy(), + PARTIAL_OVERLAP_UPDATES_BY_LEN[n], + PARTIAL_OVERLAP_RESULTS_BY_LEN[n], + ), + {}, + ) + + +def _setup_forceupdate_existing_values(n: int) -> tuple[tuple[t.Any, ...], dict[str, t.Any]]: + return ( + ( + INT_BIDICTS_BY_LEN[n].copy(), + FORCEUPDATE_EXISTING_VALUES_UPDATES_BY_LEN[n], + FORCEUPDATE_EXISTING_VALUES_RESULTS_BY_LEN[n], + ), + {}, + ) + + def _setup_failing_update_early(n: int) -> tuple[tuple[t.Any, ...], dict[str, t.Any]]: return ( ( @@ -305,6 +335,26 @@ def test_bi_update_partial_overlap(n: int, benchmark: t.Any) -> None: ) +@pytest.mark.parametrize('n', LENS) +def test_bi_forceupdate_partial_overlap(n: int, benchmark: t.Any) -> None: + """Benchmark forceupdating from a mapping with a mix of overlapping and new items.""" + benchmark.pedantic( + _forceupdate, + setup=lambda n=n: _setup_forceupdate_partial_overlap(n), + teardown=_assert_mapping_matches, + ) + + +@pytest.mark.parametrize('n', LENS) +def test_bi_forceupdate_existing_values(n: int, benchmark: t.Any) -> None: + """Benchmark forceupdating from a mapping whose values replace all existing ones.""" + benchmark.pedantic( + _forceupdate, + setup=lambda n=n: _setup_forceupdate_existing_values(n), + teardown=_assert_mapping_matches, + ) + + @pytest.mark.parametrize('n', LENS) def test_bi_update_fail_early_dupval(n: int, benchmark: t.Any) -> None: """Benchmark a bulk update that fails near the start and rolls back.""" diff --git a/pyproject.toml b/pyproject.toml index de267948..fcdd5969 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,9 +60,15 @@ docs = [ "sphinx-copybutton", "furo", ] +native = [ + "bidict-base-opt-native", +] [tool.uv] -default-groups = ["test", "dev", "docs"] +default-groups = ["test", "dev", "docs", "native"] + +[tool.uv.sources] +bidict-base-opt-native = { path = "rust/bidict_base_opt_native", editable = true } [tool.uv.build-backend] module-root = "" diff --git a/rust/bidict_base_opt_native/Cargo.lock b/rust/bidict_base_opt_native/Cargo.lock new file mode 100644 index 00000000..3d5cdcc2 --- /dev/null +++ b/rust/bidict_base_opt_native/Cargo.lock @@ -0,0 +1,180 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bidict-base-opt-native" +version = "0.23.2-dev0" +dependencies = [ + "pyo3", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unindent" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" diff --git a/rust/bidict_base_opt_native/Cargo.toml b/rust/bidict_base_opt_native/Cargo.toml new file mode 100644 index 00000000..db098e6c --- /dev/null +++ b/rust/bidict_base_opt_native/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "bidict-base-opt-native" +version = "0.23.2-dev0" +edition = "2021" +license = "MPL-2.0" +publish = false + +[lib] +name = "bidict_base_opt_native" +crate-type = ["cdylib"] + +[dependencies] +pyo3 = { version = "0.22.6", features = ["abi3-py311", "extension-module"] } diff --git a/rust/bidict_base_opt_native/pyproject.toml b/rust/bidict_base_opt_native/pyproject.toml new file mode 100644 index 00000000..4158e9e0 --- /dev/null +++ b/rust/bidict_base_opt_native/pyproject.toml @@ -0,0 +1,14 @@ +[project] +name = "bidict-base-opt-native" +version = "0.23.2.dev0" +description = "Optional native helpers for bidict." +authors = [{ name = "Joshua Bronson", email = "jabronson@gmail.com" }] +license = "MPL-2.0" +requires-python = ">=3.11" + +[build-system] +requires = ["maturin>=1.8,<2.0"] +build-backend = "maturin" + +[tool.maturin] +manifest-path = "Cargo.toml" diff --git a/rust/bidict_base_opt_native/src/lib.rs b/rust/bidict_base_opt_native/src/lib.rs new file mode 100644 index 00000000..80d9f356 --- /dev/null +++ b/rust/bidict_base_opt_native/src/lib.rs @@ -0,0 +1,275 @@ +#![allow( + clippy::useless_conversion, + reason = "PyO3 #[pyfunction] expansion triggers this false positive on PyResult returns" +)] + +use pyo3::prelude::*; +use pyo3::types::PyDict; +use pyo3::types::PyType; + +#[derive(Clone, Copy)] +enum OnDupAction { + Raise, + DropOld, + DropNew, +} + +impl OnDupAction { + fn parse(value: &str) -> PyResult { + match value { + "RAISE" => Ok(Self::Raise), + "DROP_OLD" => Ok(Self::DropOld), + "DROP_NEW" => Ok(Self::DropNew), + _ => Err(pyo3::exceptions::PyValueError::new_err(format!( + "unknown OnDupAction: {value}" + ))), + } + } +} + +fn bidict_err(py: Python<'_>, name: &str, args: A) -> PyErr +where + A: pyo3::PyErrArguments + Send + Sync + 'static, +{ + let bidict = py + .import_bound("bidict") + .expect("bidict should already be importable"); + let err_type = bidict + .getattr(name) + .expect("bidict exception should already be importable") + .downcast_into::() + .expect("bidict exception should be a Python type"); + PyErr::from_type_bound(err_type, args) +} + +struct ExistingItems { + oldval: Option>, + oldkey: Option>, +} + +fn handle_dup_item( + py: Python<'_>, + fwd: &Bound<'_, PyDict>, + inv: &Bound<'_, PyDict>, + key: &Bound<'_, PyAny>, + val: &Bound<'_, PyAny>, + existing: ExistingItems, + on_dup: (OnDupAction, OnDupAction), +) -> PyResult<()> { + let ExistingItems { oldval, oldkey } = existing; + let (on_dup_key, on_dup_val) = on_dup; + let isdupkey = oldval.is_some(); + let isdupval = oldkey.is_some(); + + if isdupkey && isdupval { + let oldkey = oldkey.as_ref().expect("checked is_some above"); + let oldval = oldval.as_ref().expect("checked is_some above"); + if key.eq(oldkey.bind(py))? { + assert!(val.eq(oldval.bind(py))?); + return Ok(()); + } + match on_dup_val { + OnDupAction::Raise => { + return Err(bidict_err( + py, + "KeyAndValueDuplicationError", + (key.clone().unbind(), val.clone().unbind()), + )); + } + OnDupAction::DropNew => return Ok(()), + OnDupAction::DropOld => {} + } + } else if isdupkey { + match on_dup_key { + OnDupAction::Raise => { + return Err(bidict_err( + py, + "KeyDuplicationError", + (key.clone().unbind(),), + )); + } + OnDupAction::DropNew => return Ok(()), + OnDupAction::DropOld => {} + } + } else if isdupval { + match on_dup_val { + OnDupAction::Raise => { + return Err(bidict_err( + py, + "ValueDuplicationError", + (val.clone().unbind(),), + )); + } + OnDupAction::DropNew => return Ok(()), + OnDupAction::DropOld => {} + } + } + + fwd.set_item(key, val)?; + inv.set_item(val, key)?; + + if isdupkey && isdupval { + let oldkey = oldkey.as_ref().expect("checked is_some above"); + let oldval = oldval.as_ref().expect("checked is_some above"); + fwd.del_item(oldkey.bind(py))?; + inv.del_item(oldval.bind(py))?; + } else if isdupkey { + let oldval = oldval.as_ref().expect("checked is_some above"); + inv.del_item(oldval.bind(py))?; + } else if isdupval { + let oldkey = oldkey.as_ref().expect("checked is_some above"); + fwd.del_item(oldkey.bind(py))?; + } + + Ok(()) +} + +fn apply_items( + py: Python<'_>, + fwd: &Bound<'_, PyDict>, + inv: &Bound<'_, PyDict>, + items: Bound<'_, PyAny>, + on_dup_key: OnDupAction, + on_dup_val: OnDupAction, +) -> PyResult<()> { + for item in items.iter()? { + let item = item?; + let (key, val): (Py, Py) = item.extract()?; + let key = key.bind(py); + let val = val.bind(py); + let oldval = fwd.get_item(key)?.map(Bound::unbind); + let oldkey = inv.get_item(val)?.map(Bound::unbind); + if oldval.is_none() && oldkey.is_none() { + fwd.set_item(key, val)?; + inv.set_item(val, key)?; + continue; + } + handle_dup_item( + py, + fwd, + inv, + key, + val, + ExistingItems { oldval, oldkey }, + (on_dup_key, on_dup_val), + )?; + } + + Ok(()) +} + +fn apply_mapping( + py: Python<'_>, + fwd: &Bound<'_, PyDict>, + inv: &Bound<'_, PyDict>, + mapping: Bound<'_, PyAny>, + on_dup_key: OnDupAction, + on_dup_val: OnDupAction, +) -> PyResult<()> { + if let Ok(dict) = mapping.downcast::() { + for (key, val) in dict.iter() { + let oldval = fwd.get_item(&key)?.map(Bound::unbind); + let oldkey = inv.get_item(&val)?.map(Bound::unbind); + if oldval.is_none() && oldkey.is_none() { + fwd.set_item(&key, &val)?; + inv.set_item(&val, &key)?; + continue; + } + handle_dup_item( + py, + fwd, + inv, + &key, + &val, + ExistingItems { oldval, oldkey }, + (on_dup_key, on_dup_val), + )?; + } + } else { + let items = mapping.call_method0("items")?; + apply_items(py, fwd, inv, items, on_dup_key, on_dup_val)?; + } + + Ok(()) +} + +#[pyfunction] +fn build_bidict_maps( + py: Python<'_>, + items: Bound<'_, PyAny>, + on_dup_key: &str, + on_dup_val: &str, +) -> PyResult<(Py, Py)> { + let on_dup_key = OnDupAction::parse(on_dup_key)?; + let on_dup_val = OnDupAction::parse(on_dup_val)?; + let fwd = PyDict::new_bound(py); + let inv = PyDict::new_bound(py); + + apply_items(py, &fwd, &inv, items, on_dup_key, on_dup_val)?; + + Ok((fwd.unbind(), inv.unbind())) +} + +#[pyfunction] +fn build_bidict_maps_from_mapping( + py: Python<'_>, + mapping: Bound<'_, PyAny>, + on_dup_key: &str, + on_dup_val: &str, +) -> PyResult<(Py, Py)> { + let on_dup_key = OnDupAction::parse(on_dup_key)?; + let on_dup_val = OnDupAction::parse(on_dup_val)?; + let fwd = PyDict::new_bound(py); + let inv = PyDict::new_bound(py); + + apply_mapping(py, &fwd, &inv, mapping, on_dup_key, on_dup_val)?; + + Ok((fwd.unbind(), inv.unbind())) +} + +#[pyfunction] +fn update_bidict_maps( + py: Python<'_>, + fwd: Bound<'_, PyDict>, + inv: Bound<'_, PyDict>, + items: Bound<'_, PyAny>, + on_dup_key: &str, + on_dup_val: &str, +) -> PyResult<(Py, Py)> { + let on_dup_key = OnDupAction::parse(on_dup_key)?; + let on_dup_val = OnDupAction::parse(on_dup_val)?; + let new_fwd = fwd.copy()?; + let new_inv = inv.copy()?; + + apply_items(py, &new_fwd, &new_inv, items, on_dup_key, on_dup_val)?; + + Ok((new_fwd.unbind(), new_inv.unbind())) +} + +#[pyfunction] +fn update_bidict_maps_from_mapping( + py: Python<'_>, + fwd: Bound<'_, PyDict>, + inv: Bound<'_, PyDict>, + mapping: Bound<'_, PyAny>, + on_dup_key: &str, + on_dup_val: &str, +) -> PyResult<(Py, Py)> { + let on_dup_key = OnDupAction::parse(on_dup_key)?; + let on_dup_val = OnDupAction::parse(on_dup_val)?; + let new_fwd = fwd.copy()?; + let new_inv = inv.copy()?; + + apply_mapping(py, &new_fwd, &new_inv, mapping, on_dup_key, on_dup_val)?; + + Ok((new_fwd.unbind(), new_inv.unbind())) +} + +#[pymodule] +fn bidict_base_opt_native(_py: Python<'_>, module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(build_bidict_maps, module)?)?; + module.add_function(wrap_pyfunction!(build_bidict_maps_from_mapping, module)?)?; + module.add_function(wrap_pyfunction!(update_bidict_maps, module)?)?; + module.add_function(wrap_pyfunction!(update_bidict_maps_from_mapping, module)?)?; + Ok(()) +} diff --git a/tests/test_native.py b/tests/test_native.py new file mode 100644 index 00000000..d61264ce --- /dev/null +++ b/tests/test_native.py @@ -0,0 +1,411 @@ +# Copyright 2009-2026 Joshua Bronson. All rights reserved. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import annotations + +import importlib +import sys +from collections.abc import Iterable +from types import SimpleNamespace + +import pytest + +from bidict import DROP_NEW +from bidict import ON_DUP_DROP_OLD +from bidict import RAISE +from bidict import KeyAndValueDuplicationError +from bidict import KeyDuplicationError +from bidict import OnDup +from bidict import OrderedBidict +from bidict import ValueDuplicationError +from bidict import _base as base_mod +from bidict import _native as native_mod +from bidict import bidict + + +def test_native_env_var_disables_helpers(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv('BIDICT_DISABLE_NATIVE', '1') + reloaded = importlib.reload(native_mod) + try: + assert reloaded.build_bidict_maps is None + assert reloaded.build_bidict_maps_from_mapping is None + assert reloaded.update_bidict_maps is None + assert reloaded.update_bidict_maps_from_mapping is None + finally: + monkeypatch.delenv('BIDICT_DISABLE_NATIVE', raising=False) + importlib.reload(reloaded) + + +def test_non_cpython_runtime_disables_helpers(monkeypatch: pytest.MonkeyPatch) -> None: + original_implementation = sys.implementation + implementation_attrs = vars(original_implementation).copy() + implementation_attrs['name'] = 'pypy' + monkeypatch.setattr(sys, 'implementation', SimpleNamespace(**implementation_attrs)) + reloaded = importlib.reload(native_mod) + try: + assert reloaded.build_bidict_maps is None + assert reloaded.build_bidict_maps_from_mapping is None + assert reloaded.update_bidict_maps is None + assert reloaded.update_bidict_maps_from_mapping is None + finally: + monkeypatch.setattr(sys, 'implementation', original_implementation) + importlib.reload(reloaded) + + +def test_empty_update_uses_native_builder_when_available(monkeypatch: pytest.MonkeyPatch) -> None: + items_seen: list[tuple[int, int]] = [] + + def fake_build(items: Iterable[tuple[int, int]], _on_dup: object) -> tuple[dict[int, int], dict[int, int]]: + items_seen.extend(items) + return {1: 2}, {2: 1} + + monkeypatch.setattr(base_mod, '_build_bidict_maps', fake_build) + bi = bidict[int, int]() + + bi.update([(1, 2)]) + + assert items_seen == [(1, 2)] + assert dict(bi.items()) == {1: 2} + assert dict(bi.inverse.items()) == {2: 1} + + +def test_empty_mapping_update_uses_mapping_native_builder_when_available(monkeypatch: pytest.MonkeyPatch) -> None: + seen_mapping: dict[int, int] | None = None + + def fail_build(_items: object, _on_dup: object) -> tuple[dict[int, int], dict[int, int]]: + msg = 'generic native builder should not run for mapping updates' + raise AssertionError(msg) + + def fake_build_from_mapping(mapping: dict[int, int], _on_dup: object) -> tuple[dict[int, int], dict[int, int]]: + nonlocal seen_mapping + seen_mapping = mapping + return {1: 2}, {2: 1} + + monkeypatch.setattr(base_mod, '_build_bidict_maps', fail_build) + monkeypatch.setattr(base_mod, '_build_bidict_maps_from_mapping', fake_build_from_mapping) + bi = bidict[int, int]() + + mapping = {1: 2} + bi.update(mapping) + + assert seen_mapping is mapping + + +def test_empty_mapping_update_prescans_early_dupvals_before_native_builder(monkeypatch: pytest.MonkeyPatch) -> None: + def fail_build_from_mapping(_mapping: object, _on_dup: object) -> tuple[dict[int, int], dict[int, int]]: + msg = 'native builder should not run after early duplicate-value prescan fails' + raise AssertionError(msg) + + monkeypatch.setattr(base_mod, '_build_bidict_maps_from_mapping', fail_build_from_mapping) + monkeypatch.setattr(base_mod, '_MAX_NATIVE_DUPVAL_FAST_FAIL_ITEMS', 2) + bi = bidict[int, int]() + + with pytest.raises(ValueDuplicationError): + bi.update({1: 0, 2: 0, 3: 3}) + + assert not bi + + +def test_empty_update_preserves_materialized_inverse(monkeypatch: pytest.MonkeyPatch) -> None: + items_seen: list[tuple[int, int]] = [] + bi = bidict[int, int]() + inv = bi.inverse + + def fake_build(items: Iterable[tuple[int, int]], _on_dup: object) -> tuple[dict[int, int], dict[int, int]]: + items_seen.extend(items) + return {1: 2}, {2: 1} + + monkeypatch.setattr(base_mod, '_build_bidict_maps', fake_build) + + bi.update([(1, 2)]) + + assert items_seen == [(1, 2)] + assert dict(inv.items()) == {2: 1} + assert inv.inverse is bi + + +def test_nonempty_update_skips_native_builder(monkeypatch: pytest.MonkeyPatch) -> None: + bi = bidict({1: 2}) + + def fail_build(_items: object, _on_dup: object) -> tuple[dict[int, int], dict[int, int]]: + msg = 'native builder should not run for non-empty updates' + raise AssertionError(msg) + + monkeypatch.setattr(base_mod, '_build_bidict_maps', fail_build) + + bi.update([(3, 4)]) + + assert dict(bi.items()) == {1: 2, 3: 4} + + +def test_orderedbidict_skips_native_builder() -> None: + bi = OrderedBidict([(1, 2), (3, 4)]) + + assert not bi._supports_native_map_swap() + assert tuple(bi.items()) == ((1, 2), (3, 4)) + + +def test_nonempty_bulk_update_uses_native_updater_when_available(monkeypatch: pytest.MonkeyPatch) -> None: + items_seen: list[tuple[int, int]] = [] + + def fake_update( + fwd: dict[int, int], + inv: dict[int, int], + items: Iterable[tuple[int, int]], + _on_dup: object, + ) -> tuple[dict[int, int], dict[int, int]]: + items_seen.extend(items) + assert fwd == {1: 2, 3: 4} + assert inv == {2: 1, 4: 3} + return {1: 2, 3: 4, 5: 6, 7: 8}, {2: 1, 4: 3, 6: 5, 8: 7} + + monkeypatch.setattr(base_mod, '_update_bidict_maps', fake_update) + monkeypatch.setattr(base_mod, '_MIN_NATIVE_FORCEUPDATE_ITEMS', 2) + bi = bidict({1: 2, 3: 4}) + + bi.update([(5, 6), (7, 8)]) + + assert items_seen == [(5, 6), (7, 8)] + assert dict(bi.items()) == {1: 2, 3: 4, 5: 6, 7: 8} + + +def test_nonempty_mapping_update_uses_mapping_native_updater(monkeypatch: pytest.MonkeyPatch) -> None: + seen_mapping: dict[int, int] | None = None + + def fail_update( + _fwd: dict[int, int], + _inv: dict[int, int], + _items: Iterable[tuple[int, int]], + _on_dup: object, + ) -> tuple[dict[int, int], dict[int, int]]: + msg = 'generic native updater should not run for mapping updates' + raise AssertionError(msg) + + def fake_update_from_mapping( + _fwd: dict[int, int], + _inv: dict[int, int], + mapping: dict[int, int], + _on_dup: object, + ) -> tuple[dict[int, int], dict[int, int]]: + nonlocal seen_mapping + seen_mapping = mapping + return {1: 2, 3: 4, 5: 6, 7: 8}, {2: 1, 4: 3, 6: 5, 8: 7} + + monkeypatch.setattr(base_mod, '_update_bidict_maps', fail_update) + monkeypatch.setattr(base_mod, '_update_bidict_maps_from_mapping', fake_update_from_mapping) + monkeypatch.setattr(base_mod, '_MIN_NATIVE_UPDATE_ITEMS', 2) + bi = bidict({1: 2, 3: 4}) + + mapping = {5: 6, 7: 8} + bi.update(mapping) + + assert seen_mapping is mapping + + +def test_nonempty_small_update_skips_native_updater(monkeypatch: pytest.MonkeyPatch) -> None: + bi = bidict({1: 2, 3: 4}) + + def fail_update( + _fwd: object, _inv: object, _items: object, _on_dup: object + ) -> tuple[dict[int, int], dict[int, int]]: + msg = 'native updater should not run for small updates' + raise AssertionError(msg) + + monkeypatch.setattr(base_mod, '_update_bidict_maps', fail_update) + + bi.update([(5, 6)]) + + assert dict(bi.items()) == {1: 2, 3: 4, 5: 6} + + +def test_orderedbidict_skips_native_updater() -> None: + bi = OrderedBidict({1: 2, 3: 4}) + assert not bi._supports_native_map_swap() + + +def test_forceupdate_uses_native_updater_when_available(monkeypatch: pytest.MonkeyPatch) -> None: + items_seen: list[tuple[int, int]] = [] + + def fake_update( + fwd: dict[int, int], + inv: dict[int, int], + items: Iterable[tuple[int, int]], + on_dup: object, + ) -> tuple[dict[int, int], dict[int, int]]: + items_seen.extend(items) + assert fwd == {1: 2, 3: 4} + assert inv == {2: 1, 4: 3} + assert on_dup == ON_DUP_DROP_OLD + return {1: 2, 5: 4, 6: 7}, {2: 1, 4: 5, 7: 6} + + monkeypatch.setattr(base_mod, '_update_bidict_maps', fake_update) + monkeypatch.setattr(base_mod, '_MIN_NATIVE_UPDATE_ITEMS', 2) + bi = bidict({1: 2, 3: 4}) + + bi.forceupdate([(5, 4), (6, 7)]) + + assert items_seen == [(5, 4), (6, 7)] + assert dict(bi.items()) == {1: 2, 5: 4, 6: 7} + + +def test_nonempty_mapping_dupval_failure_uses_native_updater(monkeypatch: pytest.MonkeyPatch) -> None: + called = False + + def fail_update( + _fwd: object, _inv: object, _mapping: object, _on_dup: object + ) -> tuple[dict[int, int], dict[int, int]]: + nonlocal called + called = True + raise ValueDuplicationError(0) + + monkeypatch.setattr(base_mod, '_update_bidict_maps_from_mapping', fail_update) + monkeypatch.setattr(base_mod, '_MIN_NATIVE_UPDATE_ITEMS', 1) + bi = bidict({10: 10}) + + with pytest.raises(ValueDuplicationError): + bi.update({1: 0, 2: 0}) + + assert called + assert dict(bi.items()) == {10: 10} + + +def test_nonempty_mapping_update_preserves_duplication_precedence(monkeypatch: pytest.MonkeyPatch) -> None: + called = False + + def fail_update( + _fwd: object, _inv: object, _mapping: object, _on_dup: object + ) -> tuple[dict[int, int], dict[int, int]]: + nonlocal called + called = True + raise KeyDuplicationError(1) + + monkeypatch.setattr(base_mod, '_update_bidict_maps_from_mapping', fail_update) + monkeypatch.setattr(base_mod, '_MIN_NATIVE_UPDATE_ITEMS', 1) + bi = bidict({1: 10}) + + with pytest.raises(KeyDuplicationError): + bi.putall({1: 0, 2: 0, 3: 3}, OnDup(RAISE, RAISE)) + + assert called + assert dict(bi.items()) == {1: 10} + + +def test_nonempty_native_update_preserves_materialized_inverse(monkeypatch: pytest.MonkeyPatch) -> None: + bi = bidict({1: 2, 3: 4}) + inv = bi.inverse + + def fake_update( + _fwd: dict[int, int], + _inv: dict[int, int], + items: Iterable[tuple[int, int]], + _on_dup: object, + ) -> tuple[dict[int, int], dict[int, int]]: + assert list(items) == [(3, 5), (6, 7)] + return {1: 2, 3: 5, 6: 7}, {2: 1, 5: 3, 7: 6} + + monkeypatch.setattr(base_mod, '_update_bidict_maps', fake_update) + monkeypatch.setattr(base_mod, '_MIN_NATIVE_UPDATE_ITEMS', 2) + + bi.update([(3, 5), (6, 7)]) + + assert dict(inv.items()) == {2: 1, 5: 3, 7: 6} + assert inv.inverse is bi + + +native_build = native_mod.build_bidict_maps +native_build_from_mapping = native_mod.build_bidict_maps_from_mapping +native_update = native_mod.update_bidict_maps +native_update_from_mapping = native_mod.update_bidict_maps_from_mapping +if ( + native_build is not None + or native_build_from_mapping is not None + or native_update is not None + or native_update_from_mapping is not None +): + pytest.importorskip('bidict_base_opt_native') + + +@pytest.mark.skipif(native_build is None, reason='optional native helper is not installed') +def test_native_build_bidict_maps_matches_drop_old_behavior() -> None: + assert native_build is not None + fwd, inv = native_build([(1, 2), (1, 3), (4, 3)], ON_DUP_DROP_OLD) + + assert fwd == {4: 3} + assert inv == {3: 4} + + +@pytest.mark.skipif(native_build is None, reason='optional native helper is not installed') +def test_native_build_bidict_maps_honors_drop_new() -> None: + assert native_build is not None + on_dup = OnDup(key=DROP_NEW, val=DROP_NEW) + fwd, inv = native_build([(1, 2), (1, 3), (4, 2)], on_dup) + + assert fwd == {1: 2} + assert inv == {2: 1} + + +@pytest.mark.skipif(native_build is None, reason='optional native helper is not installed') +def test_native_build_bidict_maps_raises_value_duplication_error() -> None: + assert native_build is not None + with pytest.raises(ValueDuplicationError): + native_build([(1, 2), (3, 2)], bidict.on_dup) + + +@pytest.mark.skipif(native_build is None, reason='optional native helper is not installed') +def test_native_build_bidict_maps_raises_key_and_value_duplication_error() -> None: + assert native_build is not None + with pytest.raises(KeyAndValueDuplicationError): + native_build([(1, 2), (3, 4), (1, 4)], bidict.on_dup) + + +@pytest.mark.skipif(native_build_from_mapping is None, reason='optional native mapping helper is not installed') +def test_native_build_bidict_maps_from_mapping_matches_drop_old_behavior() -> None: + assert native_build_from_mapping is not None + fwd, inv = native_build_from_mapping({1: 3, 4: 3}, ON_DUP_DROP_OLD) + + assert fwd == {4: 3} + assert inv == {3: 4} + + +@pytest.mark.skipif(native_update is None, reason='optional native helper is not installed') +def test_native_update_bidict_maps_matches_drop_old_behavior() -> None: + assert native_update is not None + fwd = {1: 2, 3: 4} + inv = {2: 1, 4: 3} + + new_fwd, new_inv = native_update(fwd, inv, [(3, 5), (6, 5)], ON_DUP_DROP_OLD) + + assert new_fwd == {1: 2, 6: 5} + assert new_inv == {2: 1, 5: 6} + assert fwd == {1: 2, 3: 4} + assert inv == {2: 1, 4: 3} + + +@pytest.mark.skipif(native_update is None, reason='optional native helper is not installed') +def test_native_update_bidict_maps_raises_without_mutating_inputs() -> None: + assert native_update is not None + fwd = {1: 2, 3: 4} + inv = {2: 1, 4: 3} + + with pytest.raises(ValueDuplicationError): + native_update(fwd, inv, [(5, 6), (7, 4)], bidict.on_dup) + + assert fwd == {1: 2, 3: 4} + assert inv == {2: 1, 4: 3} + + +@pytest.mark.skipif(native_update_from_mapping is None, reason='optional native mapping helper is not installed') +def test_native_update_bidict_maps_from_mapping_matches_drop_old_behavior() -> None: + assert native_update_from_mapping is not None + fwd = {1: 2, 3: 4} + inv = {2: 1, 4: 3} + + new_fwd, new_inv = native_update_from_mapping(fwd, inv, {3: 5, 6: 5}, ON_DUP_DROP_OLD) + + assert new_fwd == {1: 2, 6: 5} + assert new_inv == {2: 1, 5: 6} + assert fwd == {1: 2, 3: 4} + assert inv == {2: 1, 4: 3} diff --git a/uv.lock b/uv.lock index 8ce9cdfe..66872d5a 100644 --- a/uv.lock +++ b/uv.lock @@ -112,6 +112,9 @@ docs = [ { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "sphinx-copybutton" }, ] +native = [ + { name = "bidict-base-opt-native" }, +] test = [ { name = "coverage" }, { name = "hypothesis" }, @@ -141,6 +144,7 @@ docs = [ { name = "sphinx" }, { name = "sphinx-copybutton" }, ] +native = [{ name = "bidict-base-opt-native", editable = "rust/bidict_base_opt_native" }] test = [ { name = "coverage" }, { name = "hypothesis" }, @@ -154,6 +158,11 @@ test = [ { name = "typing-extensions" }, ] +[[package]] +name = "bidict-base-opt-native" +version = "0.23.2.dev0" +source = { editable = "rust/bidict_base_opt_native" } + [[package]] name = "cachetools" version = "7.1.2"