From 67987e8aa44b9768b3fc19531fbc2ee7a3df4939 Mon Sep 17 00:00:00 2001 From: Georg Grab Date: Thu, 1 Oct 2026 18:15:51 +0000 Subject: [PATCH 1/2] test: compare same-device regression round-trips at float32 precision Inference runs in float32, and on MPS the regression border remap sums in a nondeterministic order, so two predicts can differ by one float32 step. Six fixed decimals demand bit-identical float32 for predictions in the hundreds; compare against the predictions' own scale instead. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_018qH4DKrmQVSS3qbzViv5nf --- tests/test_save_load_fitted_model.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_save_load_fitted_model.py b/tests/test_save_load_fitted_model.py index 0b8797747..3e20929e5 100644 --- a/tests/test_save_load_fitted_model.py +++ b/tests/test_save_load_fitted_model.py @@ -60,8 +60,8 @@ def _assert_roundtrip_predictions( ) -> None: """Check that a save/load round-trip produced an equivalent model. - Same-device round-trips must reproduce predictions near-exactly, so we - assert numerical equivalence. Cross-device round-trips (e.g. + Same-device round-trips must reproduce predictions to float32 precision, so + we assert numerical equivalence. Cross-device round-trips (e.g. ``cpu``<->``cuda``) cannot: CPU and GPU use different default inference precisions and different matmul/attention kernels whose summation order differs, so bit-identity is unattainable. For those we only verify that the @@ -85,9 +85,14 @@ def _assert_roundtrip_predictions( return # Same device: the round-trip must be numerically faithful. - np.testing.assert_array_almost_equal(original_preds, loaded_preds) if isinstance(original, TabPFNClassifier): + np.testing.assert_array_almost_equal(original_preds, loaded_preds) np.testing.assert_array_almost_equal(original_probas, loaded_probas) + else: + # Inference runs in float32, so regression outputs agree to float32 + # precision of their own scale rather than to a fixed number of decimals. + atol = 1e-6 * max(float(np.max(np.abs(original_preds))), 1.0) + np.testing.assert_allclose(loaded_preds, original_preds, rtol=0, atol=atol) @pytest.mark.parametrize( From 88a387aab0972dcfeae88f09325f6345034c219d Mon Sep 17 00:00:00 2001 From: Oscar Key Date: Fri, 2 Oct 2026 11:17:55 +0200 Subject: [PATCH 2/2] Scale tolerance by error in z, excluding shift. --- tests/test_save_load_fitted_model.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_save_load_fitted_model.py b/tests/test_save_load_fitted_model.py index 3e20929e5..e311bc4bd 100644 --- a/tests/test_save_load_fitted_model.py +++ b/tests/test_save_load_fitted_model.py @@ -89,9 +89,14 @@ def _assert_roundtrip_predictions( np.testing.assert_array_almost_equal(original_preds, loaded_preds) np.testing.assert_array_almost_equal(original_probas, loaded_probas) else: - # Inference runs in float32, so regression outputs agree to float32 - # precision of their own scale rather than to a fixed number of decimals. - atol = 1e-6 * max(float(np.max(np.abs(original_preds))), 1.0) + # original_preds = (y_train_std * z) + y_train_mean + # z is computed in float32, so its error grows with |z|, and the + # predictions must agree to float32 error scaled by y_train_std * |z|. + y_train_std_abs_z = np.abs(original_preds - original.y_train_mean_) + min_abs_z = 1.0 + atol = 1e-6 * max( + float(np.max(y_train_std_abs_z)), original.y_train_std_ * min_abs_z + ) np.testing.assert_allclose(loaded_preds, original_preds, rtol=0, atol=atol)