diff --git a/changelog/1199.changed.md b/changelog/1199.changed.md new file mode 100644 index 000000000..6b223be04 --- /dev/null +++ b/changelog/1199.changed.md @@ -0,0 +1 @@ +Regression target transforms now act on raw targets before final standardization. Both `power` and `safepower` retain input standardization; `None` and `"none"` retain standardization only. Invalid `log` and `1_plus_log` target domains now raise clear errors during fitting, and `exp` warns when transformed targets would overflow in float32. diff --git a/src/tabpfn/preprocessing/target_transform.py b/src/tabpfn/preprocessing/target_transform.py index 53fc2d669..e7cdba200 100644 --- a/src/tabpfn/preprocessing/target_transform.py +++ b/src/tabpfn/preprocessing/target_transform.py @@ -49,7 +49,7 @@ def inverse_transform(self, X: np.ndarray) -> np.ndarray: def make_target_transform(transform: Transformer | Pipeline | None) -> Pipeline: - """Standardize raw targets, then apply the optional preset. + """Apply the optional preset to raw targets, then standardize. The inverse returns values in the target's original units. """ @@ -57,9 +57,9 @@ def make_target_transform(transform: Transformer | Pipeline | None) -> Pipeline: return Pipeline(steps=[(STANDARDIZE_STEP, StandardizeTarget())]) return Pipeline( steps=[ - # The preset reshapes the standardized target. - (STANDARDIZE_STEP, StandardizeTarget()), + # Reshape the raw target before standardizing it for the model. (TARGET_TRANSFORM_STEP, transform), + (STANDARDIZE_STEP, StandardizeTarget()), ], ) diff --git a/src/tabpfn/preprocessing/transform.py b/src/tabpfn/preprocessing/transform.py index d593ef423..2e3b91299 100644 --- a/src/tabpfn/preprocessing/transform.py +++ b/src/tabpfn/preprocessing/transform.py @@ -95,7 +95,8 @@ def _transform_labels_one( Args: config: Ensemble config. - y_train: The unprocessed labels. + y_train: Encoded class indices or regression targets. A configured + regression target pipeline expects targets in their original units. Return: The processed labels. """ diff --git a/src/tabpfn/regressor.py b/src/tabpfn/regressor.py index a888b944f..bfbdd43d9 100644 --- a/src/tabpfn/regressor.py +++ b/src/tabpfn/regressor.py @@ -38,6 +38,7 @@ check_is_fitted, clone, ) +from sklearn.pipeline import Pipeline from tqdm.auto import tqdm from tabpfn.architectures.shared.bar_distribution import FullSupportBarDistribution @@ -124,7 +125,6 @@ if TYPE_CHECKING: import numpy.typing as npt - from sklearn.pipeline import Pipeline from torch.types import _dtype from tabpfn.architectures.interface import ( @@ -1020,11 +1020,44 @@ def _initialize_dataset_preprocessing( num_examples=y.shape[0], # Use length of validated y random_state=random_state, # Use the provided rng ) + for preset in self.inference_config_.REGRESSION_Y_PREPROCESS_TRANSFORMS: + if preset == "log" and np.any(y <= 0): + raise ValueError( + "The 'log' regression target transform requires strictly positive " + "raw targets (y > 0). Use 'safepower' or None for " + "nonpositive targets." + ) + if preset == "1_plus_log" and np.any(y <= -1): + raise ValueError( + "The '1_plus_log' regression target transform requires raw targets " + "greater than -1. Use 'safepower' or None for targets <= -1." + ) + if preset == "exp": + overflow_threshold = np.log(np.finfo(np.float32).max) + overflow_count = np.count_nonzero(y >= overflow_threshold) + if overflow_count: + warnings.warn( + f"The 'exp' regression target transform would make " + f"{overflow_count} of {y.size} raw targets overflow " + "in float32. " + "Even one infinite value makes target standardization invalid. " + "Rescale the targets or use 'safepower' or None.", + UserWarning, + stacklevel=2, + ) + # Yeo-Johnson depends on units and offset; fit it on standardized targets. + for preset in ("power", "safepower"): + possible_target_transforms[preset] = Pipeline( + steps=[ + ("standardize_input", StandardizeTarget()), + (preset, possible_target_transforms[preset]), + ] + ) # Each member fits its target pipeline on raw targets. target_preprocessors: list[Pipeline] = [ make_target_transform( None - if y_target_preprocessor is None + if y_target_preprocessor in (None, "none") else possible_target_transforms[y_target_preprocessor] ) for y_target_preprocessor in ( diff --git a/tests/test_preprocessing/test_target_transform.py b/tests/test_preprocessing/test_target_transform.py index 3ffdbb84d..aa17d4e80 100644 --- a/tests/test_preprocessing/test_target_transform.py +++ b/tests/test_preprocessing/test_target_transform.py @@ -71,18 +71,43 @@ def test__make_target_transform__without_a_transform_only_standardizes() -> None @pytest.mark.parametrize("name", TRANSFORM_NAMES) -def test__make_target_transform__transform_sees_the_standardized_target( +def test__make_target_transform__transform_sees_the_unnormalized_target( name: str, ) -> None: - """The preset receives standardized targets.""" + """The preset reshapes the raw target before standardization.""" y = _target() transform = _get_transform(name, len(y)) got = make_target_transform(transform).fit_transform(y.reshape(-1, 1)).astype(float) - standardized = (y - np.mean(y)) / (np.std(y) + StandardizeTarget.EPSILON) - expected = _get_transform(name, len(y)).fit_transform(standardized.reshape(-1, 1)) - np.testing.assert_allclose(got, np.asarray(expected, dtype=float), rtol=1e-12) + reshaped = np.asarray( + _get_transform(name, len(y)).fit_transform(y.reshape(-1, 1)), dtype=float + ) + expected = (reshaped - reshaped.mean()) / ( + reshaped.std() + StandardizeTarget.EPSILON + ) + np.testing.assert_allclose(got, expected, rtol=1e-10) + + +def test__make_target_transform__log_of_the_target_is_defined() -> None: + """Positive raw targets stay finite even when their z-scores fall below -1.""" + rng = np.random.default_rng(0) + y = rng.normal(100.0, 10.0, size=500) # symmetric, strictly positive + + standardized = (y - np.mean(y)) / np.std(y) + assert not np.isfinite(np.log1p(standardized)).all() + + got = ( + make_target_transform(_get_transform("1_plus_log", len(y))) + .fit_transform(y.reshape(-1, 1)) + .ravel() + ) + + assert np.isfinite(got).all() + expected = np.log1p(y) + np.testing.assert_allclose( + got, (expected - expected.mean()) / expected.std(), rtol=1e-10 + ) @pytest.mark.parametrize("name", TRANSFORM_NAMES) diff --git a/tests/test_regressor_interface.py b/tests/test_regressor_interface.py index 10056d39f..2ba4a1090 100644 --- a/tests/test_regressor_interface.py +++ b/tests/test_regressor_interface.py @@ -6,6 +6,7 @@ import itertools import os import typing +import warnings from collections.abc import Callable from fractions import Fraction from functools import partial @@ -1500,6 +1501,111 @@ def test__fit__member_target_is_the_znormalized_target() -> None: assert np.array_equal(member_y, (y - np.mean(y)) / (np.std(y) + 1e-20)) +def _mk_skewed_reg_dataset( + seed: int, n: int = 200, f: int = 5 +) -> tuple[np.ndarray, np.ndarray]: + """A strictly positive, right-skewed target -- the log-transform case.""" + r = np.random.RandomState(seed) + X = r.randn(n, f) + y = np.exp(1.5 + X @ np.array([1.0, -0.5, 0.3, 0.0, 0.2]) + 0.3 * r.randn(n)) + return X, y + + +def test__fit__target_transform__applied_to_the_unnormalized_target() -> None: + """Apply log1p to raw targets before standardizing model inputs.""" + X, y = _mk_skewed_reg_dataset(0) + reg = TabPFNRegressor( + n_estimators=1, + device="cpu", + random_state=42, + inference_config={"REGRESSION_Y_PREPROCESS_TRANSFORMS": ("1_plus_log",)}, + ) + reg.fit(X, y) + + expected = np.log1p(y) + expected = (expected - expected.mean()) / expected.std() + member_y = np.asarray(reg.executor_.ensemble_members[0].y_train) + + np.testing.assert_allclose(member_y, expected, rtol=1e-8) + + +@pytest.mark.parametrize("dtype", [np.float32, np.float64]) +@pytest.mark.parametrize("offset", [0.0, 1e6]) +@pytest.mark.parametrize("transform", ["power", "safepower"]) +def test__fit__power_standardizes_before_fitting( + dtype: type[np.floating], offset: float, transform: str +) -> None: + X, y = _mk_skewed_reg_dataset(0) + y = (y + offset).astype(dtype) + reg = TabPFNRegressor( + n_estimators=1, + device="cpu", + random_state=42, + inference_config={"REGRESSION_Y_PREPROCESS_TRANSFORMS": (transform,)}, + ) + reg.fit(X, y) + + preset = reg.executor_.ensemble_members[0].config.target_transform.named_steps[ + "target_transform" + ] + standardized = (y - float(np.mean(y))) / (float(np.std(y)) + 1e-20) + np.testing.assert_array_equal( + preset.named_steps["standardize_input"].transform(y.reshape(-1, 1)).ravel(), + standardized, + ) + expected = regressor_module.get_all_reshape_feature_distribution_preprocessors( + num_examples=len(y), random_state=42 + )[transform].fit_transform(standardized.reshape(-1, 1)) + np.testing.assert_array_equal(preset.transform(y.reshape(-1, 1)), expected) + pipeline = reg.executor_.ensemble_members[0].config.target_transform + np.testing.assert_allclose( + pipeline.inverse_transform(pipeline.transform(y.reshape(-1, 1))).ravel(), + y, + rtol=1e-5, + ) + + +def test__fit__none_preset_is_resolved_to_no_transform() -> None: + """`"none"` is the identity, so it must not cost an extra pass.""" + X, y = _mk_skewed_reg_dataset(0) + reg = TabPFNRegressor( + n_estimators=1, + device="cpu", + random_state=42, + inference_config={"REGRESSION_Y_PREPROCESS_TRANSFORMS": ("none",)}, + ) + reg.fit(X, y) + + pipeline = reg.executor_.ensemble_members[0].config.target_transform + assert list(pipeline.named_steps) == ["standardize_target"] + assert np.array_equal( + np.asarray(reg.executor_.ensemble_members[0].y_train), + (y - np.mean(y)) / (np.std(y) + 1e-20), + ) + + +@pytest.mark.parametrize("transform", ["1_plus_log", "log", "safepower"]) +def test__predict__target_transform__predictions_are_finite_and_accurate( + transform: str, +) -> None: + """Target transformation and inverse border mapping preserve prediction quality.""" + X, y = _mk_skewed_reg_dataset(0) + X_train, X_test, y_train, y_test = X[:150], X[150:], y[:150], y[150:] + reg = TabPFNRegressor( + n_estimators=2, + device="cpu", + # Keep transform accuracy independent of CPU bfloat16 support. + inference_precision=torch.float32, + random_state=42, + inference_config={"REGRESSION_Y_PREPROCESS_TRANSFORMS": (transform,)}, + ) + reg.fit(X_train, y_train) + pred = reg.predict(X_test) + + assert np.isfinite(pred).all() + assert r2_score(y_test, pred) > 0.7 + + @pytest.mark.parametrize("dtype", ["float32", "float64"]) def test__fit__target_frame_is_independent_of_the_target_units(dtype: str) -> None: """Rescaling the target changes its frame but preserves standardized inputs.""" @@ -2194,3 +2300,56 @@ def test__eval_metric_and_tuning_config__round_trip_through_sklearn_clone( round_tripped = TabPFNRegressor().set_params(**params) assert round_tripped.eval_metric == eval_metric assert round_tripped.tuning_config == tuning_config + + +@pytest.mark.parametrize( + ("transform", "minimum", "domain"), + [ + ("log", 0.0, "strictly positive"), + ("log", -1.0, "strictly positive"), + ("1_plus_log", -1.0, "greater than -1"), + ("1_plus_log", -2.0, "greater than -1"), + ], +) +def test__fit__invalid_log_target__raises_clear_error( + transform: str, minimum: float, domain: str +) -> None: + X, y = _mk_skewed_reg_dataset(0) + y[0] = minimum + reg = TabPFNRegressor( + n_estimators=1, + device="cpu", + inference_config={"REGRESSION_Y_PREPROCESS_TRANSFORMS": (transform,)}, + ) + with pytest.raises(ValueError, match=f"{transform}.*{domain}"): + reg.fit(X, y) + + +@pytest.mark.parametrize( + ("maximum", "warns"), + [ + (1.0, False), + (88.0, False), + (np.nextafter(np.log(np.finfo(np.float32).max), np.float32(-np.inf)), False), + (np.log(np.finfo(np.float32).max), True), + (90.0, True), + ], +) +def test__fit__exp_target__warns_only_for_float32_overflow( + maximum: float, warns: bool +) -> None: + X, y = _mk_skewed_reg_dataset(0) + reg = TabPFNRegressor( + n_estimators=1, + device="cpu", + inference_config={"REGRESSION_Y_PREPROCESS_TRANSFORMS": ("exp",)}, + ) + y = y / y.max() + y[-1] = maximum + if warns: + with pytest.warns(UserWarning, match="exp.*1 of .*overflow in float32"): + reg.fit(X, y) + else: + with warnings.catch_warnings(): + warnings.filterwarnings("error", message=".*exp.*", category=UserWarning) + reg.fit(X, y)