From b9bbfef323ac14c07296a216bf4535b0d2003d75 Mon Sep 17 00:00:00 2001 From: Aniruddha Adak Date: Thu, 1 Oct 2026 23:21:26 +0530 Subject: [PATCH 1/3] fix(inference): validate tuning holdout frac before inverting it for folds `tuning_holdout_frac` is passed through verbatim, so any user value was fed straight into `round(holdout_frac, 2)` before `1 / rounded_holdout_frac`. Pre-rounding to 2 decimals collapsed values below 0.005 to zero and inflated values just above it, so `tuning_holdout_frac=0.004` raised ZeroDivisionError out of `fit()` and `0.005` asked for 100 folds, which StratifiedKFold rejects on most real datasets. Out-of-range and non-finite fracs were silently accepted. Derive the fold count from the exact fraction instead, and reject fractions that are outside (0, 1) or that would need more folds than we support. --- src/tabpfn/inference_tuning.py | 54 +++++++++++++++-- tests/test_inference_tuning.py | 102 +++++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+), 4 deletions(-) diff --git a/src/tabpfn/inference_tuning.py b/src/tabpfn/inference_tuning.py index c20278a57..3fb509f96 100644 --- a/src/tabpfn/inference_tuning.py +++ b/src/tabpfn/inference_tuning.py @@ -6,6 +6,7 @@ import copy import dataclasses +import math import warnings from collections.abc import Callable from enum import Enum @@ -34,6 +35,15 @@ MIN_NUM_SAMPLES_RECOMMENDED_FOR_TUNING = 500 +MAX_TUNING_FOLDS = 100 +"""Upper bound on the (stratified) k-fold count implied by a holdout fraction. + +A smaller holdout fraction means more folds, and folds are what make each tuning +split expensive, so we refuse fractions that would ask for more than this many. +""" +MIN_TUNING_HOLDOUT_FRAC = 1 / MAX_TUNING_FOLDS +"""The smallest holdout fraction that stays within `MAX_TUNING_FOLDS`.""" + @dataclasses.dataclass class TuningConfig: @@ -235,6 +245,40 @@ def compute_regression_metric_to_minimize( ) +def _folds_for_holdout_frac(holdout_frac: float) -> int: + """Derives the number of (stratified) k-folds from a holdout fraction. + + Each k-fold split holds out `1 / n_folds` of the data, so the requested + fraction has to be inverted to get the fold count. + + Args: + holdout_frac: The fraction of the data to hold out for tuning. + + Returns: + The number of folds to use, in `[2, MAX_TUNING_FOLDS]`. + + Raises: + ValueError: If `holdout_frac` is not a finite number in the open + interval `(0, 1)`, or if it would need more than + `MAX_TUNING_FOLDS` folds. + """ + if not math.isfinite(holdout_frac) or not 0.0 < holdout_frac < 1.0: + raise ValueError( + "tuning holdout fraction must be a finite number strictly between " + f"0 and 1, got {holdout_frac!r}." + ) + if holdout_frac < MIN_TUNING_HOLDOUT_FRAC: + raise ValueError( + f"tuning holdout fraction {holdout_frac!r} is too small: it would " + f"need more than {MAX_TUNING_FOLDS} tuning folds. Use a value of at " + f"least {MIN_TUNING_HOLDOUT_FRAC}." + ) + # Inverse the fraction directly rather than rounding it first: pre-rounding + # to 2 decimals collapses values below 0.005 to zero (dividing by zero) and + # inflates values just above it (0.005 -> 0.01 -> 100 folds). + return min(max(round(1 / holdout_frac), 2), MAX_TUNING_FOLDS) + + def get_tuning_splits( X: np.ndarray, y: np.ndarray, @@ -260,13 +304,15 @@ def get_tuning_splits( Returns a list of splits as tuples of (X_train_NtF, X_holdout_NhF, y_train_Nt, y_holdout_Nh). Shape suffixes: Nt=num train samples, F=num features, Nh=num holdout samples. + + Raises: + ValueError: If `holdout_frac` is not a finite number in the open + interval `(0, 1)`. """ # We want to use (Stratified)KFold to ensure that no train samples are used twice. # Therefore, we have to invert the holdout_frac to get the number of folds to - # use for (Stratified)KFold. Round holdout_frac to 2 digits to avoid needing - # more than 100 folds - rounded_holdout_frac = round(holdout_frac, 2) - n_folds = max(2, round(1 / rounded_holdout_frac)) + # use for (Stratified)KFold. + n_folds = _folds_for_holdout_frac(holdout_frac) if isinstance(random_state, np.random.Generator): # (Stratified)KFold does not accept np.random.Generator. diff --git a/tests/test_inference_tuning.py b/tests/test_inference_tuning.py index 7ebaae20e..0d52ef0fd 100644 --- a/tests/test_inference_tuning.py +++ b/tests/test_inference_tuning.py @@ -560,6 +560,108 @@ def test__get_tuning_splits__classification_is_unchanged_by_default() -> None: assert (y_holdout == 0).sum() == 4 +@pytest.mark.parametrize( + "holdout_frac", + [ + pytest.param(0.0, id="zero"), + pytest.param(-0.1, id="negative"), + pytest.param(1.0, id="one"), + pytest.param(1.5, id="above_one"), + pytest.param(float("nan"), id="nan"), + pytest.param(float("inf"), id="inf"), + ], +) +def test__get_tuning_splits__rejects_holdout_frac_outside_open_unit_interval( + holdout_frac: float, +) -> None: + y = np.random.default_rng(0).normal(size=200) + + # A frac that rounds to zero previously raised ZeroDivisionError, and NaN or + # out-of-range fracs silently produced a nonsense fold count. + with pytest.raises(ValueError, match="strictly between"): + get_tuning_splits( + X=_identity_X(200), + y=y, + holdout_frac=holdout_frac, + n_splits=1, + task_type="regressor", + ) + + +@pytest.mark.parametrize( + ("holdout_frac", "expected_holdout_size"), + [ + pytest.param(0.15, 143, id="fractional_frac"), + pytest.param(0.01, 10, id="at_the_supported_lower_bound"), + ], +) +def test__get_tuning_splits__derives_sane_fold_count_from_awkward_frac( + holdout_frac: float, + expected_holdout_size: int, +) -> None: + n_samples = 1_000 + y = np.random.default_rng(0).normal(size=n_samples) + + splits = get_tuning_splits( + X=_identity_X(n_samples), + y=y, + holdout_frac=holdout_frac, + n_splits=1, + task_type="regressor", + ) + + # The fold count is derived from the exact frac, so 0.15 maps to the nearest + # usable fold count (7) instead of the 6 that pre-rounding implied. + assert len(splits[0][1]) == expected_holdout_size + + +@pytest.mark.parametrize("holdout_frac", [0.004, 0.005, 0.009]) +def test__get_tuning_splits__rejects_frac_needing_more_folds_than_supported( + holdout_frac: float, +) -> None: + y = np.random.default_rng(0).normal(size=200) + + # These all need more than the supported fold count. Previously 0.004 divided + # by zero outright, and 0.005 asked for 100 folds, which StratifiedKFold + # rejects on most real datasets. + with pytest.raises(ValueError, match="too small"): + get_tuning_splits( + X=_identity_X(200), + y=y, + holdout_frac=holdout_frac, + n_splits=1, + task_type="regressor", + ) + + +@pytest.mark.parametrize( + ("holdout_frac", "expected_holdout_size"), + [ + pytest.param(0.1, 100, id="auto_default_small_dataset"), + pytest.param(0.2, 200, id="auto_default_mid_dataset"), + pytest.param(0.3, 334, id="auto_default_large_dataset"), + ], +) +def test__get_tuning_splits__auto_default_fracs_keep_their_fold_counts( + holdout_frac: float, + expected_holdout_size: int, +) -> None: + # `get_default_tuning_holdout_frac` only ever yields 0.1/0.2/0.3. Those must + # keep implying 10/5/3 folds, so existing users see no behaviour change. + n_samples = 1_000 + y = np.random.default_rng(0).normal(size=n_samples) + + splits = get_tuning_splits( + X=_identity_X(n_samples), + y=y, + holdout_frac=holdout_frac, + n_splits=1, + task_type="regressor", + ) + + assert len(splits[0][1]) == expected_holdout_size + + @pytest.mark.parametrize( ("num_samples", "expected_holdout_frac", "expected_n_folds"), [ From 0be1d69ef3787586368be977b0828a014458a887 Mon Sep 17 00:00:00 2001 From: Aniruddha Adak Date: Fri, 2 Oct 2026 11:57:14 +0530 Subject: [PATCH 2/3] changelog: add the changelog fragment for #1350 --- changelog/1350.fixed.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/1350.fixed.md diff --git a/changelog/1350.fixed.md b/changelog/1350.fixed.md new file mode 100644 index 000000000..a3e8f6114 --- /dev/null +++ b/changelog/1350.fixed.md @@ -0,0 +1 @@ +Validate uning_holdout_frac before inverting it into a fold count, so a fraction below 0.005 raises a clear ValueError instead of ZeroDivisionError and a small fraction no longer yields an unusable number of folds. \ No newline at end of file From a4e3dfd44ba9b303cbf55c14933524ed2eb01f59 Mon Sep 17 00:00:00 2001 From: Aniruddha Adak Date: Fri, 2 Oct 2026 13:48:03 +0530 Subject: [PATCH 3/3] changelog: describe the change from the user's perspective --- changelog/1350.fixed.md | 2 +- src/tabpfn/inference_tuning.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/changelog/1350.fixed.md b/changelog/1350.fixed.md index a3e8f6114..d3207241b 100644 --- a/changelog/1350.fixed.md +++ b/changelog/1350.fixed.md @@ -1 +1 @@ -Validate uning_holdout_frac before inverting it into a fold count, so a fraction below 0.005 raises a clear ValueError instead of ZeroDivisionError and a small fraction no longer yields an unusable number of folds. \ No newline at end of file +Validate uning_holdout_frac and derive the tuning fold count from the requested fraction, so an unsupported value reports a clear error instead of an unusable fold count. \ No newline at end of file diff --git a/src/tabpfn/inference_tuning.py b/src/tabpfn/inference_tuning.py index 3fb509f96..ae265a357 100644 --- a/src/tabpfn/inference_tuning.py +++ b/src/tabpfn/inference_tuning.py @@ -273,9 +273,6 @@ def _folds_for_holdout_frac(holdout_frac: float) -> int: f"need more than {MAX_TUNING_FOLDS} tuning folds. Use a value of at " f"least {MIN_TUNING_HOLDOUT_FRAC}." ) - # Inverse the fraction directly rather than rounding it first: pre-rounding - # to 2 decimals collapses values below 0.005 to zero (dividing by zero) and - # inflates values just above it (0.005 -> 0.01 -> 100 folds). return min(max(round(1 / holdout_frac), 2), MAX_TUNING_FOLDS)