Skip to content

benchs #54

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
gianlucadetommaso wants to merge 26 commits into
base: main
Choose a base branch
from
Open

benchs #54

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
0bba6ca
benchs
Aug 10, 2024
0653cd7
missing files
Aug 10, 2024
4ca1abe
add missing
Aug 10, 2024
a0fda60
remove pycache
Aug 10, 2024
273bb3a
add tqdm
Aug 10, 2024
4f4c248
fix if no groups
Aug 10, 2024
a9b40ed
fix
Aug 10, 2024
4a5149a
sort datasets
Aug 10, 2024
7898828
gcu return normal probs if no groups
Aug 10, 2024
f8aa5ea
group may be none
Aug 10, 2024
d625c42
calib groups to None
Aug 10, 2024
73799e5
fix kernelized
Aug 10, 2024
96210da
change asces
Aug 14, 2024
52f3e48
changes to one shot kernelized
Aug 14, 2024
1847408
pass groups
Aug 14, 2024
ef71a8a
change gasce
Aug 14, 2024
8534562
edit
Aug 14, 2024
f0a2442
uncomment
Aug 14, 2024
786c901
change osk
Aug 14, 2024
ffa2a8f
remove models
Aug 14, 2024
e283471
edit loss
Aug 14, 2024
00103f6
edit asces
Aug 15, 2024
c5160ab
make changes
Aug 15, 2024
eb9605b
adjust asces and perm
Aug 16, 2024
8c55c02
further changes
Aug 18, 2024
86046bc
precommit
Aug 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 64 additions & 60 deletions benchmarks/binary_classification/binary_classification.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,17 @@
IsotonicRegressionBinaryClassificationModel,
IterativeBinningBinaryClassificationModel,
IterativeFittingBinaryClassificationModel,
IterativeSmoothHistogramBinningBinaryClassificationModel,
IterativeKernelizedBinningBinaryClassificationModel,
KneePointLinearScalingBinaryClassificationModel,
ModelBiasConstantShiftBinaryClassificationModel,
NegativeF1LinearScalingBinaryClassificationModel,
OneShotKernelizedBinaryClassificationModel,
PositiveF1LinearScalingBinaryClassificationModel,
PositiveNegativeRatesLinearScalingBinaryClassificationModel,
PrecisionRecallLinearScalingBinaryClassificationModel,
PredictiveValuesLinearScalingBinaryClassificationModel,
RighteousnessLinearScalingBinaryClassificationModel,
SmoothLinearScalingBinaryClassificationModel,
)
from caliber.binary_classification.metrics import (
average_smooth_squared_calibration_error,
Expand Down Expand Up @@ -77,75 +79,77 @@
test_preds = (test_probs >= THRESHOLD).astype(int)

posthoc_models = {
"balanced_accuracy_linear_scaling": BalancedAccuracyLinearScalingBinaryClassificationModel(
threshold=THRESHOLD
),
"balanced_accuracy_temperature_scaling": BalancedAccuracyLinearScalingBinaryClassificationModel(
threshold=THRESHOLD, has_intercept=False
),
"positive_f1_linear_scaling": PositiveF1LinearScalingBinaryClassificationModel(
threshold=THRESHOLD
),
"positive_f1_temperature_scaling": PositiveF1LinearScalingBinaryClassificationModel(
threshold=THRESHOLD, has_intercept=False
),
"negative_f1_linear_scaling": NegativeF1LinearScalingBinaryClassificationModel(
threshold=THRESHOLD
),
"negative_f1_temperature_scaling": NegativeF1LinearScalingBinaryClassificationModel(
threshold=THRESHOLD, has_intercept=False
),
"predictive_values_linear_scaling": PredictiveValuesLinearScalingBinaryClassificationModel(
threshold=THRESHOLD
),
"predictive_values_temperature_scaling": PredictiveValuesLinearScalingBinaryClassificationModel(
threshold=THRESHOLD, has_intercept=False
),
"positive_negative_rates_linear_scaling": PositiveNegativeRatesLinearScalingBinaryClassificationModel(
threshold=THRESHOLD
),
"positive_negative_rates_temperature_scaling": PositiveNegativeRatesLinearScalingBinaryClassificationModel(
threshold=THRESHOLD, has_intercept=False
),
"righteousness_linear_scaling": RighteousnessLinearScalingBinaryClassificationModel(
threshold=THRESHOLD
),
"righteousness_temperature_scaling": RighteousnessLinearScalingBinaryClassificationModel(
threshold=THRESHOLD, has_intercept=False
),
"precrec_linear_scaling": PrecisionRecallLinearScalingBinaryClassificationModel(
threshold=THRESHOLD
),
# "balanced_accuracy_linear_scaling": BalancedAccuracyLinearScalingBinaryClassificationModel(
# threshold=THRESHOLD
# ),
# "balanced_accuracy_temperature_scaling": BalancedAccuracyLinearScalingBinaryClassificationModel(
# threshold=THRESHOLD, has_intercept=False
# ),
# "positive_f1_linear_scaling": PositiveF1LinearScalingBinaryClassificationModel(
# threshold=THRESHOLD
# ),
# "positive_f1_temperature_scaling": PositiveF1LinearScalingBinaryClassificationModel(
# threshold=THRESHOLD, has_intercept=False
# ),
# "negative_f1_linear_scaling": NegativeF1LinearScalingBinaryClassificationModel(
# threshold=THRESHOLD
# ),
# "negative_f1_temperature_scaling": NegativeF1LinearScalingBinaryClassificationModel(
# threshold=THRESHOLD, has_intercept=False
# ),
# "predictive_values_linear_scaling": PredictiveValuesLinearScalingBinaryClassificationModel(
# threshold=THRESHOLD
# ),
# "predictive_values_temperature_scaling": PredictiveValuesLinearScalingBinaryClassificationModel(
# threshold=THRESHOLD, has_intercept=False
# ),
# "positive_negative_rates_linear_scaling": PositiveNegativeRatesLinearScalingBinaryClassificationModel(
# threshold=THRESHOLD
# ),
# "positive_negative_rates_temperature_scaling": PositiveNegativeRatesLinearScalingBinaryClassificationModel(
# threshold=THRESHOLD, has_intercept=False
# ),
# "righteousness_linear_scaling": RighteousnessLinearScalingBinaryClassificationModel(
# threshold=THRESHOLD
# ),
# "righteousness_temperature_scaling": RighteousnessLinearScalingBinaryClassificationModel(
# threshold=THRESHOLD, has_intercept=False
# ),
# "precrec_linear_scaling": PrecisionRecallLinearScalingBinaryClassificationModel(
# threshold=THRESHOLD
# ),
"brier_linear_scaling": BrierLinearScalingBinaryClassificationModel(),
"brier_temperature_scaling": BrierLinearScalingBinaryClassificationModel(
has_intercept=False
),
# "brier_temperature_scaling": BrierLinearScalingBinaryClassificationModel(
# has_intercept=False
# ),
"cross_entropy_linear_scaling": CrossEntropyLinearScalingBinaryClassificationModel(),
"cross_entropy_temperature_scaling": CrossEntropyLinearScalingBinaryClassificationModel(
has_intercept=False
),
# "cross_entropy_temperature_scaling": CrossEntropyLinearScalingBinaryClassificationModel(
# has_intercept=False
# ),
"focal_linear_scaling": FocalLinearScalingBinaryClassificationModel(),
"focal_temperature_scaling": FocalLinearScalingBinaryClassificationModel(
has_intercept=False
),
"knee_linear_scaling": KneePointLinearScalingBinaryClassificationModel(),
"knee_temperature_scaling": KneePointLinearScalingBinaryClassificationModel(
has_intercept=False
),
"constant_shift": ModelBiasConstantShiftBinaryClassificationModel(),
# "focal_temperature_scaling": FocalLinearScalingBinaryClassificationModel(
# has_intercept=False
# ),
# "knee_linear_scaling": KneePointLinearScalingBinaryClassificationModel(),
# "knee_temperature_scaling": KneePointLinearScalingBinaryClassificationModel(
# has_intercept=False
# ),
# "constant_shift": ModelBiasConstantShiftBinaryClassificationModel(),
"histogram_binning": HistogramBinningBinaryClassificationModel(),
"isotonic_regression": IsotonicRegressionBinaryClassificationModel(),
"iterative_smooth_histogram_binning": IterativeSmoothHistogramBinningBinaryClassificationModel(),
"iterative_smooth_grouped_histogram_binning": IterativeSmoothHistogramBinningBinaryClassificationModel(),
"kernelized": OneShotKernelizedBinaryClassificationModel(),
"smooth_linear_scaling": SmoothLinearScalingBinaryClassificationModel(),
"iterative_smooth_histogram_binning": IterativeKernelizedBinningBinaryClassificationModel(),
# "iterative_smooth_grouped_histogram_binning": IterativeSmoothHistogramBinningBinaryClassificationModel(),
"iterative_histogram_binning": IterativeBinningBinaryClassificationModel(),
"iterative_linear_binning": IterativeBinningBinaryClassificationModel(
bin_model=BrierLinearScalingBinaryClassificationModel(),
),
"iterative_grouped_linear_binning": IterativeBinningBinaryClassificationModel(
bin_model=BrierLinearScalingBinaryClassificationModel(),
),
# "iterative_grouped_linear_binning": IterativeBinningBinaryClassificationModel(
# bin_model=BrierLinearScalingBinaryClassificationModel(),
# ),
"grouped_conditional_unbiased": GroupConditionalUnbiasedBinaryClassificationModel(),
"iterative_grouped_fitting": IterativeFittingBinaryClassificationModel(),
# "iterative_grouped_fitting": IterativeFittingBinaryClassificationModel(),
}
performance_metrics = {
"accuracy": accuracy_score,
Expand Down
Loading