Apply regression target transforms to the unnormalized target [RES-2639]

changelog/1199.changed.md

@@ -0,0 +1 @@
+Regression target transforms now act on raw targets before final standardization. Both `power` and `safepower` retain input standardization; `None` and `"none"` retain standardization only. Invalid `log` and `1_plus_log` target domains now raise clear errors during fitting, and `exp` warns when transformed targets would overflow in float32.

src/tabpfn/preprocessing/target_transform.py

@@ -49,17 +49,17 @@ def inverse_transform(self, X: np.ndarray) -> np.ndarray:
 
 
 def make_target_transform(transform: Transformer | Pipeline | None) -> Pipeline:
-    """Standardize raw targets, then apply the optional preset.
+    """Apply the optional preset to raw targets, then standardize.
 
     The inverse returns values in the target's original units.
     """
     if transform is None:
         return Pipeline(steps=[(STANDARDIZE_STEP, StandardizeTarget())])
     return Pipeline(
         steps=[
-            # The preset reshapes the standardized target.
-            (STANDARDIZE_STEP, StandardizeTarget()),
+            # Reshape the raw target before standardizing it for the model.
             (TARGET_TRANSFORM_STEP, transform),
+            (STANDARDIZE_STEP, StandardizeTarget()),
         ],
     )
 

src/tabpfn/preprocessing/transform.py

@@ -95,7 +95,8 @@ def _transform_labels_one(
 
     Args:
         config: Ensemble config.
-        y_train: The unprocessed labels.
+        y_train: Encoded class indices or regression targets. A configured
+            regression target pipeline expects targets in their original units.
 
     Return: The processed labels.
     """

src/tabpfn/regressor.py

@@ -38,6 +38,7 @@
     check_is_fitted,
     clone,
 )
+from sklearn.pipeline import Pipeline
 from tqdm.auto import tqdm
 
 from tabpfn.architectures.shared.bar_distribution import FullSupportBarDistribution
@@ -124,7 +125,6 @@
 
 if TYPE_CHECKING:
     import numpy.typing as npt
-    from sklearn.pipeline import Pipeline
     from torch.types import _dtype
 
     from tabpfn.architectures.interface import (
@@ -1020,11 +1020,44 @@ def _initialize_dataset_preprocessing(
             num_examples=y.shape[0],  # Use length of validated y
             random_state=random_state,  # Use the provided rng
         )
+        for preset in self.inference_config_.REGRESSION_Y_PREPROCESS_TRANSFORMS:
+            if preset == "log" and np.any(y <= 0):
+                raise ValueError(
+                    "The 'log' regression target transform requires strictly positive "
+                    "raw targets (y > 0). Use 'safepower' or None for "
+                    "nonpositive targets."
+                )
+            if preset == "1_plus_log" and np.any(y <= -1):
+                raise ValueError(
+                    "The '1_plus_log' regression target transform requires raw targets "
+                    "greater than -1. Use 'safepower' or None for targets <= -1."
+                )
+            if preset == "exp":
+                overflow_threshold = np.log(np.finfo(np.float32).max)
+                overflow_count = np.count_nonzero(y >= overflow_threshold)
+                if overflow_count:
+                    warnings.warn(
+                        f"The 'exp' regression target transform would make "
+                        f"{overflow_count} of {y.size} raw targets overflow "
+                        "in float32. "
+                        "Even one infinite value makes target standardization invalid. "
+                        "Rescale the targets or use 'safepower' or None.",
+                        UserWarning,
+                        stacklevel=2,
+                    )
+        # Yeo-Johnson depends on units and offset; fit it on standardized targets.
+        for preset in ("power", "safepower"):
+            possible_target_transforms[preset] = Pipeline(
+                steps=[
+                    ("standardize_input", StandardizeTarget()),
+                    (preset, possible_target_transforms[preset]),
+                ]
+            )
         # Each member fits its target pipeline on raw targets.
         target_preprocessors: list[Pipeline] = [
             make_target_transform(
                 None
-                if y_target_preprocessor is None
+                if y_target_preprocessor in (None, "none")
                 else possible_target_transforms[y_target_preprocessor]
             )
             for y_target_preprocessor in (

tests/test_preprocessing/test_target_transform.py

@@ -71,18 +71,43 @@ def test__make_target_transform__without_a_transform_only_standardizes() -> None
 
 
 @pytest.mark.parametrize("name", TRANSFORM_NAMES)
-def test__make_target_transform__transform_sees_the_standardized_target(
+def test__make_target_transform__transform_sees_the_unnormalized_target(
     name: str,
 ) -> None:
-    """The preset receives standardized targets."""
+    """The preset reshapes the raw target before standardization."""
     y = _target()
     transform = _get_transform(name, len(y))
 
     got = make_target_transform(transform).fit_transform(y.reshape(-1, 1)).astype(float)
 
-    standardized = (y - np.mean(y)) / (np.std(y) + StandardizeTarget.EPSILON)
-    expected = _get_transform(name, len(y)).fit_transform(standardized.reshape(-1, 1))
-    np.testing.assert_allclose(got, np.asarray(expected, dtype=float), rtol=1e-12)
+    reshaped = np.asarray(
+        _get_transform(name, len(y)).fit_transform(y.reshape(-1, 1)), dtype=float
+    )
+    expected = (reshaped - reshaped.mean()) / (
+        reshaped.std() + StandardizeTarget.EPSILON
+    )
+    np.testing.assert_allclose(got, expected, rtol=1e-10)
+
+
+def test__make_target_transform__log_of_the_target_is_defined() -> None:
+    """Positive raw targets stay finite even when their z-scores fall below -1."""
+    rng = np.random.default_rng(0)
+    y = rng.normal(100.0, 10.0, size=500)  # symmetric, strictly positive
+
+    standardized = (y - np.mean(y)) / np.std(y)
+    assert not np.isfinite(np.log1p(standardized)).all()
+
+    got = (
+        make_target_transform(_get_transform("1_plus_log", len(y)))
+        .fit_transform(y.reshape(-1, 1))
+        .ravel()
+    )
+
+    assert np.isfinite(got).all()
+    expected = np.log1p(y)
+    np.testing.assert_allclose(
+        got, (expected - expected.mean()) / expected.std(), rtol=1e-10
+    )
 
 
 @pytest.mark.parametrize("name", TRANSFORM_NAMES)

tests/test_regressor_interface.py

@@ -6,6 +6,7 @@
 import itertools
 import os
 import typing
+import warnings
 from collections.abc import Callable
 from fractions import Fraction
 from functools import partial
@@ -1500,6 +1501,111 @@ def test__fit__member_target_is_the_znormalized_target() -> None:
     assert np.array_equal(member_y, (y - np.mean(y)) / (np.std(y) + 1e-20))
 
 
+def _mk_skewed_reg_dataset(
+    seed: int, n: int = 200, f: int = 5
+) -> tuple[np.ndarray, np.ndarray]:
+    """A strictly positive, right-skewed target -- the log-transform case."""
+    r = np.random.RandomState(seed)
+    X = r.randn(n, f)
+    y = np.exp(1.5 + X @ np.array([1.0, -0.5, 0.3, 0.0, 0.2]) + 0.3 * r.randn(n))
+    return X, y
+
+
+def test__fit__target_transform__applied_to_the_unnormalized_target() -> None:
+    """Apply log1p to raw targets before standardizing model inputs."""
+    X, y = _mk_skewed_reg_dataset(0)
+    reg = TabPFNRegressor(
+        n_estimators=1,
+        device="cpu",
+        random_state=42,
+        inference_config={"REGRESSION_Y_PREPROCESS_TRANSFORMS": ("1_plus_log",)},
+    )
+    reg.fit(X, y)
+
+    expected = np.log1p(y)
+    expected = (expected - expected.mean()) / expected.std()
+    member_y = np.asarray(reg.executor_.ensemble_members[0].y_train)
+
+    np.testing.assert_allclose(member_y, expected, rtol=1e-8)
+
+
+@pytest.mark.parametrize("dtype", [np.float32, np.float64])
+@pytest.mark.parametrize("offset", [0.0, 1e6])
+@pytest.mark.parametrize("transform", ["power", "safepower"])
+def test__fit__power_standardizes_before_fitting(
+    dtype: type[np.floating], offset: float, transform: str
+) -> None:
+    X, y = _mk_skewed_reg_dataset(0)
+    y = (y + offset).astype(dtype)
+    reg = TabPFNRegressor(
+        n_estimators=1,
+        device="cpu",
+        random_state=42,
+        inference_config={"REGRESSION_Y_PREPROCESS_TRANSFORMS": (transform,)},
+    )
+    reg.fit(X, y)
+
+    preset = reg.executor_.ensemble_members[0].config.target_transform.named_steps[
+        "target_transform"
+    ]
+    standardized = (y - float(np.mean(y))) / (float(np.std(y)) + 1e-20)
+    np.testing.assert_array_equal(
+        preset.named_steps["standardize_input"].transform(y.reshape(-1, 1)).ravel(),
+        standardized,
+    )
+    expected = regressor_module.get_all_reshape_feature_distribution_preprocessors(
+        num_examples=len(y), random_state=42
+    )[transform].fit_transform(standardized.reshape(-1, 1))
+    np.testing.assert_array_equal(preset.transform(y.reshape(-1, 1)), expected)
+    pipeline = reg.executor_.ensemble_members[0].config.target_transform
+    np.testing.assert_allclose(
+        pipeline.inverse_transform(pipeline.transform(y.reshape(-1, 1))).ravel(),
+        y,
+        rtol=1e-5,
+    )
+
+
+def test__fit__none_preset_is_resolved_to_no_transform() -> None:
+    """`"none"` is the identity, so it must not cost an extra pass."""
+    X, y = _mk_skewed_reg_dataset(0)
+    reg = TabPFNRegressor(
+        n_estimators=1,
+        device="cpu",
+        random_state=42,
+        inference_config={"REGRESSION_Y_PREPROCESS_TRANSFORMS": ("none",)},
+    )
+    reg.fit(X, y)
+
+    pipeline = reg.executor_.ensemble_members[0].config.target_transform
+    assert list(pipeline.named_steps) == ["standardize_target"]
+    assert np.array_equal(
+        np.asarray(reg.executor_.ensemble_members[0].y_train),
+        (y - np.mean(y)) / (np.std(y) + 1e-20),
+    )
+
+
+@pytest.mark.parametrize("transform", ["1_plus_log", "log", "safepower"])
+def test__predict__target_transform__predictions_are_finite_and_accurate(
+    transform: str,
+) -> None:
+    """Target transformation and inverse border mapping preserve prediction quality."""
+    X, y = _mk_skewed_reg_dataset(0)
+    X_train, X_test, y_train, y_test = X[:150], X[150:], y[:150], y[150:]
+    reg = TabPFNRegressor(
+        n_estimators=2,
+        device="cpu",
+        # Keep transform accuracy independent of CPU bfloat16 support.
+        inference_precision=torch.float32,
+        random_state=42,
+        inference_config={"REGRESSION_Y_PREPROCESS_TRANSFORMS": (transform,)},
+    )
+    reg.fit(X_train, y_train)
+    pred = reg.predict(X_test)
+
+    assert np.isfinite(pred).all()
+    assert r2_score(y_test, pred) > 0.7
+
+
 @pytest.mark.parametrize("dtype", ["float32", "float64"])
 def test__fit__target_frame_is_independent_of_the_target_units(dtype: str) -> None:
     """Rescaling the target changes its frame but preserves standardized inputs."""
@@ -2194,3 +2300,56 @@ def test__eval_metric_and_tuning_config__round_trip_through_sklearn_clone(
     round_tripped = TabPFNRegressor().set_params(**params)
     assert round_tripped.eval_metric == eval_metric
     assert round_tripped.tuning_config == tuning_config
+
+
+@pytest.mark.parametrize(
+    ("transform", "minimum", "domain"),
+    [
+        ("log", 0.0, "strictly positive"),
+        ("log", -1.0, "strictly positive"),
+        ("1_plus_log", -1.0, "greater than -1"),
+        ("1_plus_log", -2.0, "greater than -1"),
+    ],
+)
+def test__fit__invalid_log_target__raises_clear_error(
+    transform: str, minimum: float, domain: str
+) -> None:
+    X, y = _mk_skewed_reg_dataset(0)
+    y[0] = minimum
+    reg = TabPFNRegressor(
+        n_estimators=1,
+        device="cpu",
+        inference_config={"REGRESSION_Y_PREPROCESS_TRANSFORMS": (transform,)},
+    )
+    with pytest.raises(ValueError, match=f"{transform}.*{domain}"):
+        reg.fit(X, y)
+
+
+@pytest.mark.parametrize(
+    ("maximum", "warns"),
+    [
+        (1.0, False),
+        (88.0, False),
+        (np.nextafter(np.log(np.finfo(np.float32).max), np.float32(-np.inf)), False),
+        (np.log(np.finfo(np.float32).max), True),
+        (90.0, True),
+    ],
+)
+def test__fit__exp_target__warns_only_for_float32_overflow(
+    maximum: float, warns: bool
+) -> None:
+    X, y = _mk_skewed_reg_dataset(0)
+    reg = TabPFNRegressor(
+        n_estimators=1,
+        device="cpu",
+        inference_config={"REGRESSION_Y_PREPROCESS_TRANSFORMS": ("exp",)},
+    )
+    y = y / y.max()
+    y[-1] = maximum
+    if warns:
+        with pytest.warns(UserWarning, match="exp.*1 of .*overflow in float32"):
+            reg.fit(X, y)
+    else:
+        with warnings.catch_warnings():
+            warnings.filterwarnings("error", message=".*exp.*", category=UserWarning)
+            reg.fit(X, y)