From 48cb5ab49bcc5d9db44eb6c1226705280e4d31be Mon Sep 17 00:00:00 2001
From: Dilyara Bareeva
Date: Mon, 4 May 2026 14:51:41 +0200
Subject: [PATCH 01/39] chore: remove plotting scripts
---
.gitignore | 5 +-
scripts/PLOT_ALL.sh | 9 -
scripts/awa2_resnet50_bench/plot_awa2.sh | 12 -
scripts/bert_qnli_bench/plot_qnli.sh | 12 -
scripts/cifar_resnet9_bench/plot_cifar.sh | 12 -
scripts/gpt2_trex_bench/plot_gpt2.sh | 12 -
scripts/mnsit_lenet_bench/plot_mnist.sh | 12 -
scripts/plot_results.py | 624 ----------------------
8 files changed, 4 insertions(+), 694 deletions(-)
delete mode 100644 scripts/PLOT_ALL.sh
delete mode 100755 scripts/awa2_resnet50_bench/plot_awa2.sh
delete mode 100755 scripts/bert_qnli_bench/plot_qnli.sh
delete mode 100755 scripts/cifar_resnet9_bench/plot_cifar.sh
delete mode 100755 scripts/gpt2_trex_bench/plot_gpt2.sh
delete mode 100755 scripts/mnsit_lenet_bench/plot_mnist.sh
delete mode 100644 scripts/plot_results.py
diff --git a/.gitignore b/.gitignore
index 94787884..300414ee 100644
--- a/.gitignore
+++ b/.gitignore
@@ -88,4 +88,7 @@ tmp_*/*
scripts/paper_tex/**
scratch/*
slurm/get_logs.sh
-scripts/delete_hf.py
\ No newline at end of file
+scripts/delete_hf.py
+scripts/plot_results_LOCAL.py
+scripts/**/plot_*_LOCAL.sh
+scripts/PLOT_ALL_LOCAL.sh
\ No newline at end of file
diff --git a/scripts/PLOT_ALL.sh b/scripts/PLOT_ALL.sh
deleted file mode 100644
index d51d484c..00000000
--- a/scripts/PLOT_ALL.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-
-DIR="$(dirname "$0")"
-
-bash "$DIR/mnsit_lenet_bench/plot_mnist.sh"
-bash "$DIR/cifar_resnet9_bench/plot_cifar.sh"
-bash "$DIR/bert_qnli_bench/plot_qnli.sh"
-bash "$DIR/gpt2_trex_bench/plot_gpt2.sh"
-bash "$DIR/awa2_resnet50_bench/plot_awa2.sh"
diff --git a/scripts/awa2_resnet50_bench/plot_awa2.sh b/scripts/awa2_resnet50_bench/plot_awa2.sh
deleted file mode 100755
index 1edc4b6e..00000000
--- a/scripts/awa2_resnet50_bench/plot_awa2.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-
-DIR="$(dirname "$0")"
-ROOT="$(realpath "$DIR/../..")"
-RESULTS_DIR="/data2/bareeva/Projects/quanda/cluster_output_new2/eval_results/awa2"
-OUT="${OUT:-$DIR/bar_rank.png}"
-
-python "$DIR/../plot_results.py" \
- --results-dir "$RESULTS_DIR" \
- --config "$DIR/awa2_plot_config.json" \
- --out "$OUT" \
- "$@"
diff --git a/scripts/bert_qnli_bench/plot_qnli.sh b/scripts/bert_qnli_bench/plot_qnli.sh
deleted file mode 100755
index 22ef0b3d..00000000
--- a/scripts/bert_qnli_bench/plot_qnli.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-
-DIR="$(dirname "$0")"
-ROOT="$(realpath "$DIR/../..")"
-RESULTS_DIR="/data2/bareeva/Projects/quanda/cluster_output_new2/eval_results/qnli"
-OUT="${OUT:-$DIR/bar_rank.png}"
-
-python "$DIR/../plot_results.py" \
- --results-dir "$RESULTS_DIR" \
- --config "$DIR/qnli_plot_config.json" \
- --out "$OUT" \
- "$@"
diff --git a/scripts/cifar_resnet9_bench/plot_cifar.sh b/scripts/cifar_resnet9_bench/plot_cifar.sh
deleted file mode 100755
index fc9d8cf6..00000000
--- a/scripts/cifar_resnet9_bench/plot_cifar.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-
-DIR="$(dirname "$0")"
-ROOT="$(realpath "$DIR/../..")"
-RESULTS_DIR="/data2/bareeva/Projects/quanda/cluster_output_new2/eval_results/cifar"
-OUT="${OUT:-$DIR/bar_rank.png}"
-
-python "$DIR/../plot_results.py" \
- --results-dir "$RESULTS_DIR" \
- --config "$DIR/cifar_plot_config.json" \
- --out "$OUT" \
- "$@"
diff --git a/scripts/gpt2_trex_bench/plot_gpt2.sh b/scripts/gpt2_trex_bench/plot_gpt2.sh
deleted file mode 100755
index c0034371..00000000
--- a/scripts/gpt2_trex_bench/plot_gpt2.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-
-DIR="$(dirname "$0")"
-ROOT="$(realpath "$DIR/../..")"
-RESULTS_DIR="/data2/bareeva/Projects/quanda/cluster_output_new2/eval_results/gpt2_trex"
-OUT="${OUT:-$DIR/bar_rank.png}"
-
-python "$DIR/../plot_results.py" \
- --results-dir "$RESULTS_DIR" \
- --config "$DIR/gpt2_plot_config.json" \
- --out "$OUT" \
- "$@"
diff --git a/scripts/mnsit_lenet_bench/plot_mnist.sh b/scripts/mnsit_lenet_bench/plot_mnist.sh
deleted file mode 100755
index e5e68091..00000000
--- a/scripts/mnsit_lenet_bench/plot_mnist.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-
-DIR="$(dirname "$0")"
-ROOT="$(realpath "$DIR/../..")"
-RESULTS_DIR="/data2/bareeva/Projects/quanda/cluster_output_new2/eval_results/mnist"
-OUT="${OUT:-$DIR/bar_rank.png}"
-
-python "$DIR/../plot_results.py" \
- --results-dir "$RESULTS_DIR" \
- --config "$DIR/mnist_plot_config.json" \
- --out "$OUT" \
- "$@"
diff --git a/scripts/plot_results.py b/scripts/plot_results.py
deleted file mode 100644
index ee4b6c86..00000000
--- a/scripts/plot_results.py
+++ /dev/null
@@ -1,624 +0,0 @@
-"""Render bar-rank plot from local eval JSON results."""
-
-from __future__ import annotations
-
-import argparse
-import glob
-import json
-import os
-import re
-
-import matplotlib.pyplot as plt
-import numpy as np
-import pandas as pd
-from matplotlib import rcParams
-
-METHOD_COLORS = {
- "representer_points": "#EB9C38",
- "arnoldi": "#E41517",
- "tracincpfast": "#7EAF6E",
- "trak": "#7D53BA",
- "similarity": "#6F97B1",
- "random": "#90918B",
- "kronfluence": "#FDAEB9",
- "kronfluence_gpt2": "#FDAEB9",
- "dattri_trak": "#7D53BA",
- "dattri_if_datainf": "#204541",
-}
-_FALLBACK_COLOR = "#90918B"
-
-METHOD_LABELS = {
- "representer_points": "ReprPoints",
- "arnoldi": "ArnoldiInf",
- "tracincpfast": "TracInCP",
- "trak": "TRAK-1",
- "similarity": "Similarity",
- "random": "Random",
- "kronfluence": "Kronfluence",
- "kronfluence_gpt2": "Kronfluence",
- "dattri_trak": "TRAK-1",
- "dattri_if_datainf": "DataInf",
-}
-
-BENCH_LABEL_SUFFIXES = {
- "class_detection": "Class\nDetection",
- "subclass_detection": "Subclass\nDetection",
- "mislabeling_detection": "Mislabeling\nDetection",
- "shortcut_detection": "Shortcut\nDetection",
- "mixed_datasets": "Mixed Dataset\nSeparation",
- "top_k_cardinality": "Top-K\nCardinality",
- "model_randomization": "Model\nRandomization",
- "linear_datamodeling": "LDS",
-}
-
-BENCH_ORDER = (
- "class_detection",
- "subclass_detection",
- "mislabeling_detection",
- "shortcut_detection",
- "mixed_datasets",
- "top_k_cardinality",
- "linear_datamodeling",
- "model_randomization",
- "mrr",
- "recall_at_k",
- "tail_patch",
-)
-
-
-def _bench_rank(bench_id: str) -> int:
- for i, suffix in enumerate(BENCH_ORDER):
- if bench_id == suffix or bench_id.endswith("_" + suffix):
- return i
- return len(BENCH_ORDER)
-
-
-MIN_ABS_BENCH_SUBSTRINGS = ("model_randomization",)
-SIDE_PANEL_BENCH_SUBSTRINGS = (
- "tail_patch",
- "model_randomization",
- "linear_datamodeling",
-)
-
-NO_CI_EXEMPT_SUBSTRINGS = ("mislabeling_detection", "top_k_cardinality")
-
-
-def _ci_exempt(bench_id: str) -> bool:
- return any(s in bench_id for s in NO_CI_EXEMPT_SUBSTRINGS)
-
-
-def _is_min_abs(bench_id: str) -> bool:
- return any(s in bench_id for s in MIN_ABS_BENCH_SUBSTRINGS)
-
-
-def _is_side_panel(bench_id: str) -> bool:
- return any(s in bench_id for s in SIDE_PANEL_BENCH_SUBSTRINGS)
-
-
-def _detect_setting(benches: list[str]) -> str | None:
- """Common dataset prefix from bench ids (e.g. 'cifar' from
- 'cifar_class_detection'); None if benches don't share a prefix."""
- if not benches:
- return None
- prefix = benches[0].split("_", 1)[0]
- if all(b.startswith(prefix + "_") for b in benches):
- return prefix
- return None
-
-
-def _scalar(score):
- if isinstance(score, (int, float)):
- return float(score)
- if isinstance(score, dict):
- v = next(iter(score.values()), None)
- return float(v) if isinstance(v, (int, float)) else None
- return None
-
-
-def _bench_version_from_path(path: str) -> str | None:
- """Third `__`-separated segment of the filename, e.g.
- 'bdb919e-default_ClassDetection'. Identifies the bench config the
- result was produced under."""
- parts = os.path.basename(path).split("__")
- return parts[2] if len(parts) >= 3 else None
-
-
-def _canonical_bench_versions() -> dict[str, str]:
- """Map bench_id → canonical version (yaml stem) from config_map.py."""
- from quanda.benchmarks.resources.config_map import config_map
-
- out = {}
- for bench_id, path in config_map.items():
- stem, _ = os.path.splitext(os.path.basename(str(path)))
- out[bench_id] = stem
- return out
-
-
-def _prefer_canonical_versions(
- df: pd.DataFrame, canonical: dict[str, str]
-) -> pd.DataFrame:
- """Within each (bench, method, kwargs) group with multiple versions,
- drop rows whose bench_version doesn't match the canonical one if
- canonical is present in the group."""
- drop_mask = pd.Series(False, index=df.index)
- for (bench, method, kw), grp in df.groupby(
- ["bench", "method", "kwargs_key"]
- ):
- if grp["bench_version"].nunique() <= 1:
- continue
- canon = canonical.get(bench)
- if canon is None or canon not in set(grp["bench_version"]):
- continue
- drop_mask.loc[grp.index] = grp["bench_version"] != canon
- return df[~drop_mask]
-
-
-def _warn_multiple_bench_versions(
- df: pd.DataFrame, canonical: dict[str, str]
-) -> None:
- for (bench, method, _), grp in df.groupby(
- ["bench", "method", "kwargs_key"]
- ):
- versions = sorted(set(grp["bench_version"].dropna()))
- if len(versions) <= 1:
- continue
- canon = canonical.get(bench)
- suffix = (
- f"; canonical {canon!r} not found"
- if canon and canon not in versions
- else ""
- )
- print(
- f"warning: multiple bench versions for "
- f"bench={bench!r} method={method!r}: {versions}{suffix}"
- )
-
-
-def load_scores(
- results_dir: str, methods: list[str], benches: list[str]
-) -> pd.DataFrame:
- rows = []
- for path in glob.glob(os.path.join(results_dir, "*.json")):
- with open(path) as f:
- d = json.load(f)
- score = _scalar(d.get("score"))
- if score is None:
- continue
- rows.append(
- {
- "method": d.get("method"),
- "bench": d.get("bench_id"),
- "score": score,
- "ci_low": _scalar(d.get("ci_low")),
- "ci_high": _scalar(d.get("ci_high")),
- "mtime": os.path.getmtime(path),
- "bench_version": _bench_version_from_path(path),
- "kwargs_key": json.dumps(
- d.get("expl_kwargs") or {}, sort_keys=True
- ),
- }
- )
- df = pd.DataFrame(rows)
- df = df[df["method"].isin(methods) & df["bench"].isin(benches)]
- df = df.dropna(subset=["score"])
- canonical = _canonical_bench_versions()
- df = _prefer_canonical_versions(df, canonical)
- _warn_multiple_bench_versions(df, canonical)
-
- is_random = df["method"] == "random"
- random_stats = (
- df[is_random].groupby("bench")["score"].agg(["mean", "std", "count"])
- )
-
- non_random = df[~is_random].copy()
- non_random["__rank"] = non_random.apply(
- lambda r: abs(r.score) if _is_min_abs(r.bench) else -r.score,
- axis=1,
- )
- best = non_random.loc[
- non_random.groupby(["method", "bench"])["__rank"].idxmin()
- ].drop(columns="__rank")
- bars_df = best.pivot(index="method", columns="bench", values="score")
- ci_low_df = best.pivot(index="method", columns="bench", values="ci_low")
- ci_high_df = best.pivot(index="method", columns="bench", values="ci_high")
- return bars_df, ci_low_df, ci_high_df, random_stats
-
-
-def _discover(results_dir: str) -> tuple[list[str], list[str]]:
- methods, benches = set(), set()
- for path in glob.glob(os.path.join(results_dir, "*.json")):
- with open(path) as f:
- d = json.load(f)
- if d.get("method"):
- methods.add(d["method"])
- if d.get("bench_id"):
- benches.add(d["bench_id"])
- return sorted(methods), sorted(benches)
-
-
-def _parse_args() -> argparse.Namespace:
- ap = argparse.ArgumentParser()
- ap.add_argument(
- "--results-dir",
- default="/data2/bareeva/Projects/quanda/cluster_output_new/eval_results/cifar",
- )
- ap.add_argument(
- "--config",
- default=os.path.join(
- os.path.dirname(__file__),
- "cifar_resnet9_bench",
- "cifar_plot_config.json",
- ),
- help=(
- "JSON config with keys: methods, benches, method_labels, "
- "bench_labels. If omitted, methods/benches are discovered "
- "from results-dir and labels default to ids."
- ),
- )
- ap.add_argument(
- "--out",
- default=os.path.join(os.path.dirname(__file__), "bar_rank.png"),
- )
- return ap.parse_args()
-
-
-def _load_config(path: str | None) -> dict:
- if not path:
- return {}
- with open(path) as f:
- text = re.sub(r"(?m)^\s*//.*$|//[^\n\"]*$", "", f.read())
- return json.loads(text)
-
-
-def _resolve_methods_benches(
- cfg: dict, results_dir: str
-) -> tuple[list[str], list[str]]:
- methods = cfg.get("methods")
- benches = cfg.get("benches")
- if methods is None or benches is None:
- disc_methods, disc_benches = _discover(results_dir)
- methods = methods or disc_methods
- benches = benches or disc_benches
- return methods, sorted(benches, key=_bench_rank)
-
-
-def _apply_setting_to_outpath(out: str, setting: str | None) -> str:
- if not setting:
- return out
- out_dir = os.path.dirname(out)
- out_base, out_ext = os.path.splitext(os.path.basename(out))
- if setting in out_base:
- return out
- return os.path.join(out_dir, f"{out_base}_{setting}{out_ext}")
-
-
-def _default_bench_labels(
- setting: str | None, benches: list[str]
-) -> dict[str, str]:
- if not setting:
- return {}
- out = {}
- for b in benches:
- suffix = b[len(setting) + 1 :] if b.startswith(setting + "_") else b
- if suffix in BENCH_LABEL_SUFFIXES:
- out[b] = BENCH_LABEL_SUFFIXES[suffix]
- return out
-
-
-def _warn_missing_ci(
- df: pd.DataFrame,
- ci_low_df: pd.DataFrame,
- ci_high_df: pd.DataFrame,
- benches: list[str],
- bar_methods: list[str],
-) -> None:
- for b in benches:
- if _ci_exempt(b) or b not in ci_low_df.columns:
- continue
- for m in bar_methods:
- if m not in ci_low_df.index or m not in df.index:
- continue
- if pd.isna(df.loc[m, b]):
- continue
- if pd.isna(ci_low_df.loc[m, b]) and pd.isna(ci_high_df.loc[m, b]):
- print(
- f"warning: benchmark {b!r} is missing error bars "
- f"for explainer {m!r}"
- )
-
-
-def _draw_bench_bars(
- ax,
- df: pd.DataFrame,
- ci_low_df: pd.DataFrame,
- ci_high_df: pd.DataFrame,
- random_stats: pd.DataFrame,
- metric: str,
- bench_id: str,
- is_min_abs: bool,
- group_start_px: float,
- bar_px: int,
- inner_pad_px: int,
- group_w_px: float,
- colors: list[str],
- random_color: str,
-) -> float:
- values = df[metric].values
- valid = ~np.isnan(values)
- if is_min_abs:
- # Closest-to-zero first.
- sorted_idx = np.argsort(np.abs(values[valid]))
- else:
- sorted_idx = np.argsort(values[valid])[::-1]
- sorted_values = values[valid][sorted_idx]
- orig_idx = np.flatnonzero(valid)[sorted_idx]
- n_bars = len(sorted_values)
-
- x_positions = (
- group_start_px
- + bar_px / 2
- + np.arange(n_bars) * (bar_px + inner_pad_px)
- )
- ax.bar(
- x_positions,
- sorted_values,
- width=bar_px,
- color=[colors[i % len(colors)] for i in orig_idx],
- edgecolor="none",
- label=metric,
- )
-
- lows = ci_low_df[metric].values[orig_idx]
- highs = ci_high_df[metric].values[orig_idx]
- err_mask = ~np.isnan(lows) & ~np.isnan(highs)
- if err_mask.any():
- yerr = np.vstack(
- [
- np.maximum(sorted_values[err_mask] - lows[err_mask], 0),
- np.maximum(highs[err_mask] - sorted_values[err_mask], 0),
- ]
- )
- ax.errorbar(
- x_positions[err_mask],
- sorted_values[err_mask],
- yerr=yerr,
- fmt="none",
- ecolor="black",
- elinewidth=0.7,
- capsize=1,
- capthick=0.7,
- zorder=5,
- )
-
- if bench_id in random_stats.index:
- mu = random_stats.loc[bench_id, "mean"]
- line_x = (group_start_px, group_start_px + group_w_px)
- ax.hlines(
- mu,
- *line_x,
- colors=random_color,
- linewidth=1.1,
- linestyles=(0, (2, 1)),
- zorder=6,
- )
-
- return group_start_px + group_w_px / 2
-
-
-def _style_panel(
- ax,
- panel_w: float,
- xtick_positions: list[float],
- xtick_labels: list[str],
- tick_fontsize_pt: int,
-) -> None:
- ax.set_xlim(0, panel_w)
- ax.set_facecolor("#FFFFFF")
- ax.yaxis.grid(
- True,
- linewidth=0.3,
- zorder=0,
- color="gray",
- linestyle="dashed",
- )
- ax.set_axisbelow(True)
- ax.set_xticks(xtick_positions)
- ax.set_xticklabels(
- xtick_labels,
- rotation=0,
- ha="center",
- fontsize=tick_fontsize_pt,
- )
- ax.xaxis.tick_top()
- ax.xaxis.set_label_position("top")
- ax.tick_params(axis="x", pad=1, size=0, width=0.5)
- ax.tick_params(
- axis="y",
- labelsize=tick_fontsize_pt,
- pad=1,
- size=3,
- width=0.5,
- )
- for spine in ax.spines.values():
- spine.set_linewidth(0.3)
- spine.set_color("black")
-
-
-def _append_random_rows(
- df: pd.DataFrame,
- random_stats: pd.DataFrame,
- benches: list[str],
- bench_labels: dict[str, str],
- method_labels: dict[str, str],
-) -> pd.DataFrame:
- if random_stats.empty:
- return df
- random_label = method_labels.get("random", "random")
- for stat in ("mean", "std"):
- row = {"explainer": f"{random_label} ({stat})"}
- for b in benches:
- row[bench_labels.get(b, b)] = (
- random_stats.loc[b, stat]
- if b in random_stats.index
- else np.nan
- )
- df = pd.concat([df, pd.DataFrame([row])], ignore_index=True)
- return df
-
-
-def main():
- args = _parse_args()
- cfg = _load_config(args.config)
- methods, benches = _resolve_methods_benches(cfg, args.results_dir)
-
- setting = _detect_setting(benches)
- args.out = _apply_setting_to_outpath(args.out, setting)
-
- bar_methods = [m for m in methods if m != "random"]
-
- method_labels = {**METHOD_LABELS, **cfg.get("method_labels", {})}
- bench_labels = {
- **_default_bench_labels(setting, benches),
- **cfg.get("bench_labels", {}),
- }
- colors = [METHOD_COLORS.get(m, _FALLBACK_COLOR) for m in bar_methods]
- random_color = "#000000"
-
- df, ci_low_df, ci_high_df, random_stats = load_scores(
- args.results_dir, methods, benches
- )
- _warn_missing_ci(df, ci_low_df, ci_high_df, benches, bar_methods)
- rename_idx = {m: method_labels.get(m, m) for m in bar_methods}
- rename_cols = {b: bench_labels.get(b, b) for b in benches}
- df = df.reindex(index=bar_methods, columns=benches)
- df = df.rename(index=rename_idx, columns=rename_cols)
- ci_low_df = ci_low_df.reindex(index=bar_methods, columns=benches).rename(
- index=rename_idx, columns=rename_cols
- )
- ci_high_df = ci_high_df.reindex(index=bar_methods, columns=benches).rename(
- index=rename_idx, columns=rename_cols
- )
- df.index.name = "explainer"
- df.reset_index(inplace=True)
- ci_low_df = ci_low_df.reset_index(drop=True)
- ci_high_df = ci_high_df.reset_index(drop=True)
-
- metric_pairs = [
- (b, bench_labels.get(b, b), _is_min_abs(b)) for b in benches
- ]
- metric_pairs = [p for p in metric_pairs if not df[p[1]].isna().all()]
-
- rcParams["font.family"] = "DejaVu Sans"
- rcParams["font.weight"] = "normal"
-
- main_pairs = [p for p in metric_pairs if not _is_side_panel(p[0])]
- side_pairs = [p for p in metric_pairs if _is_side_panel(p[0])]
- groups = [g for g in (main_pairs, side_pairs) if g]
-
- bar_px = 7
- inner_pad_px = 1
- axes_pad_px = 4
- group_gap_px = 20
- panel_gap_px = 31
- left_margin_px = 35
- right_margin_px = 16
- top_margin_px = 22
- bottom_margin_px = 8
- out_height_px = 100
- dpi = 96
- save_dpi = 4 * dpi
-
- n_explainers = len(df)
- group_w_px = n_explainers * bar_px + (n_explainers - 1) * inner_pad_px
- panels_px = [
- 2 * axes_pad_px + len(g) * group_w_px + (len(g) - 1) * group_gap_px
- for g in groups
- ]
- total_w_px = (
- left_margin_px
- + sum(panels_px)
- + (len(groups) - 1) * panel_gap_px
- + right_margin_px
- )
-
- width_in = total_w_px / dpi
- height_in = out_height_px / dpi
-
- tick_fontsize_pt = 6
- rcParams["font.size"] = tick_fontsize_pt
- rcParams["axes.labelsize"] = tick_fontsize_pt
- rcParams["xtick.labelsize"] = tick_fontsize_pt
- rcParams["ytick.labelsize"] = tick_fontsize_pt
-
- fig = plt.figure(figsize=(width_in, height_in), dpi=dpi)
- fig.patch.set_facecolor("#FAFAF2")
-
- plot_y_frac = bottom_margin_px / out_height_px
- plot_h_frac = (
- out_height_px - top_margin_px - bottom_margin_px
- ) / out_height_px
-
- axes = []
- x_off_px = left_margin_px
- for panel_w in panels_px:
- ax = fig.add_axes(
- [
- x_off_px / total_w_px,
- plot_y_frac,
- panel_w / total_w_px,
- plot_h_frac,
- ]
- )
- axes.append(ax)
- x_off_px += panel_w + panel_gap_px
-
- for ax, group, panel_w in zip(axes, groups, panels_px):
- g_metrics = [p[1] for p in group]
- xtick_positions = []
- for j, (bench_id, metric, is_min_abs) in enumerate(group):
- group_start_px = axes_pad_px + j * (group_w_px + group_gap_px)
- xtick_positions.append(
- _draw_bench_bars(
- ax,
- df,
- ci_low_df,
- ci_high_df,
- random_stats,
- metric,
- bench_id,
- is_min_abs,
- group_start_px,
- bar_px,
- inner_pad_px,
- group_w_px,
- colors,
- random_color,
- )
- )
- _style_panel(ax, panel_w, xtick_positions, g_metrics, tick_fontsize_pt)
-
- axes[0].set_ylabel("Metric score", fontsize=tick_fontsize_pt)
-
- plt.savefig(args.out, bbox_inches=None, pad_inches=0, dpi=save_dpi)
- print(
- f"wrote {args.out} "
- f"({total_w_px}x{out_height_px} px @ {dpi} dpi layout)"
- )
- n_per_bench = ", ".join(
- f"{b}={int(c)}" for b, c in random_stats["count"].items()
- )
- print(f"random runs: {n_per_bench}")
-
- csv_path = os.path.join(
- os.path.dirname(args.out) or ".",
- os.path.splitext(os.path.basename(args.out))[0] + ".csv",
- )
- df = _append_random_rows(
- df, random_stats, benches, bench_labels, method_labels
- )
- df.to_csv(csv_path, index=False)
- print(f"wrote {csv_path}")
-
-
-if __name__ == "__main__":
- main()
From c3478f42cb6c345299be6535f165fb0ca9c7b15c Mon Sep 17 00:00:00 2001
From: Dilyara Bareeva
Date: Mon, 4 May 2026 14:53:22 +0200
Subject: [PATCH 02/39] chore: full reproduction scripts
---
scripts/REPRODUCE_ALL.sh | 39 +++++++++++++++
.../compute_lds_subset_logits_awa2_all.sh | 22 ---------
scripts/awa2_resnet50_bench/eval_all_awa2.sh | 7 ---
.../awa2_resnet50_bench/eval_awa2_arnoldi.sh | 49 -------------------
.../eval_awa2_local_arnoldi.sh | 23 ---------
.../awa2_resnet50_bench/eval_awa2_tracin.sh | 28 -----------
scripts/awa2_resnet50_bench/eval_awa2_trak.sh | 28 -----------
scripts/awa2_resnet50_bench/train_awa2.sh | 10 ++--
scripts/awa2_resnet50_bench/train_awa2_lds.sh | 3 +-
.../awa2_resnet50_bench/train_awa2_lds_all.sh | 11 -----
.../train_awa2_per_bench.sh | 30 ------------
.../train_awa2_per_bench_all.sh | 16 ------
.../train_awa2_pipeline_all.sh | 48 ------------------
.../compute_lds_subset_logits_qnli_all.sh | 22 ---------
scripts/bert_qnli_bench/train_qnli.sh | 3 ++
scripts/bert_qnli_bench/train_qnli_lds.sh | 4 +-
scripts/cifar_resnet9_bench/eval_all_cifar.sh | 7 ---
.../cifar_resnet9_bench/train_cifar_lds.sh | 4 +-
scripts/gpt2_trex_bench/eval_all_gpt2_trex.sh | 32 ------------
scripts/gpt2_trex_bench/eval_mrr_all.sh | 15 ------
.../gpt2_trex_bench/eval_recall_at_k_all.sh | 15 ------
.../gpt2_trex_bench/eval_tail_patch_all.sh | 15 ------
scripts/mnsit_lenet_bench/eval_all_mnist.sh | 6 ---
scripts/mnsit_lenet_bench/train_mnist_lds.sh | 4 +-
24 files changed, 57 insertions(+), 384 deletions(-)
create mode 100755 scripts/REPRODUCE_ALL.sh
delete mode 100755 scripts/awa2_resnet50_bench/compute_lds_subset_logits_awa2_all.sh
delete mode 100644 scripts/awa2_resnet50_bench/eval_all_awa2.sh
delete mode 100755 scripts/awa2_resnet50_bench/eval_awa2_arnoldi.sh
delete mode 100755 scripts/awa2_resnet50_bench/eval_awa2_local_arnoldi.sh
delete mode 100755 scripts/awa2_resnet50_bench/eval_awa2_tracin.sh
delete mode 100755 scripts/awa2_resnet50_bench/eval_awa2_trak.sh
delete mode 100755 scripts/awa2_resnet50_bench/train_awa2_lds_all.sh
delete mode 100755 scripts/awa2_resnet50_bench/train_awa2_per_bench.sh
delete mode 100755 scripts/awa2_resnet50_bench/train_awa2_per_bench_all.sh
delete mode 100755 scripts/awa2_resnet50_bench/train_awa2_pipeline_all.sh
delete mode 100755 scripts/bert_qnli_bench/compute_lds_subset_logits_qnli_all.sh
delete mode 100644 scripts/cifar_resnet9_bench/eval_all_cifar.sh
delete mode 100755 scripts/gpt2_trex_bench/eval_all_gpt2_trex.sh
delete mode 100755 scripts/gpt2_trex_bench/eval_mrr_all.sh
delete mode 100755 scripts/gpt2_trex_bench/eval_recall_at_k_all.sh
delete mode 100755 scripts/gpt2_trex_bench/eval_tail_patch_all.sh
delete mode 100644 scripts/mnsit_lenet_bench/eval_all_mnist.sh
diff --git a/scripts/REPRODUCE_ALL.sh b/scripts/REPRODUCE_ALL.sh
new file mode 100755
index 00000000..0f9031c7
--- /dev/null
+++ b/scripts/REPRODUCE_ALL.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+# The following script is not meant to be run and only serves as a representation of the sequence of steps performed to train and evaluate all benchmarks.
+set -e
+
+DIR="$(dirname "$0")"
+
+# 1) Train benchmarks (with hyperparam sweep)
+bash "$DIR/mnsit_lenet_bench/train_mnist.sh"
+bash "$DIR/cifar_resnet9_bench/train_cifar.sh"
+bash "$DIR/awa2_resnet50_bench/train_awa2.sh"
+bash "$DIR/bert_qnli_bench/train_qnli.sh"
+
+# 2) Train LDS models
+bash "$DIR/mnsit_lenet_bench/train_mnist_lds.sh"
+bash "$DIR/cifar_resnet9_bench/train_cifar_lds.sh"
+bash "$DIR/awa2_resnet50_bench/train_awa2_lds.sh"
+bash "$DIR/bert_qnli_bench/train_qnli_lds.sh"
+
+# 3) Collect LDS submodel logits
+bash "$DIR/awa2_resnet50_bench/compute_lds_subset_logits_awa2.sh" \
+ --start 0 --end 100 \
+ --batch-size 64 --max-eval-n 1000 --eval-seed 42 \
+ --inference-batch-size 64 --device cuda:0
+bash "$DIR/bert_qnli_bench/compute_lds_subset_logits_qnli.sh" \
+ --start 0 --end 100 \
+ --batch-size 8 --max-eval-n 1000 --eval-seed 42 \
+ --inference-batch-size 32 --device cuda:0
+
+# 4) Run eval
+bash "$DIR/mnsit_lenet_bench/eval_mnist_pt1.sh"
+bash "$DIR/mnsit_lenet_bench/eval_mnist_pt2.sh"
+bash "$DIR/cifar_resnet9_bench/eval_cifar_pt1.sh"
+bash "$DIR/cifar_resnet9_bench/eval_cifar_pt2.sh"
+bash "$DIR/awa2_resnet50_bench/eval_awa2_pt1.sh"
+bash "$DIR/awa2_resnet50_bench/eval_awa2_pt2.sh"
+bash "$DIR/bert_qnli_bench/eval_qnli.sh"
+bash "$DIR/gpt2_trex_bench/eval_mrr.sh"
+bash "$DIR/gpt2_trex_bench/eval_recall_at_k.sh"
+bash "$DIR/gpt2_trex_bench/eval_tail_patch.sh"
diff --git a/scripts/awa2_resnet50_bench/compute_lds_subset_logits_awa2_all.sh b/scripts/awa2_resnet50_bench/compute_lds_subset_logits_awa2_all.sh
deleted file mode 100755
index 95f21377..00000000
--- a/scripts/awa2_resnet50_bench/compute_lds_subset_logits_awa2_all.sh
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/bin/bash
-
-M=100
-STRIDE=10
-
-BATCH_SIZE=64
-MAX_EVAL_N=1000
-EVAL_SEED=42
-INFERENCE_BATCH_SIZE=64
-DEVICE=cuda:0
-
-for start in $(seq 0 "$STRIDE" "$((M - STRIDE))"); do
- end=$((start + STRIDE))
- sbatch slurm/slurm_job.sbatch \
- scripts/awa2_resnet50_bench/compute_lds_subset_logits_awa2.sh \
- --start "$start" --end "$end" \
- --batch-size "$BATCH_SIZE" \
- --max-eval-n "$MAX_EVAL_N" \
- --eval-seed "$EVAL_SEED" \
- --inference-batch-size "$INFERENCE_BATCH_SIZE" \
- --device "$DEVICE"
-done
diff --git a/scripts/awa2_resnet50_bench/eval_all_awa2.sh b/scripts/awa2_resnet50_bench/eval_all_awa2.sh
deleted file mode 100644
index 309d5691..00000000
--- a/scripts/awa2_resnet50_bench/eval_all_awa2.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-set -euo pipefail
-
-jid1=$(sbatch --parsable slurm/slurm_job.sbatch scripts/awa2_resnet50_bench/eval_awa2_pt1.sh)
-[[ -n $jid1 ]] || { echo "pt1 submission failed"; exit 1; }
-
-sbatch --dependency=afterok:$jid1 slurm/slurm_job.sbatch scripts/awa2_resnet50_bench/eval_awa2_pt2.sh
-
diff --git a/scripts/awa2_resnet50_bench/eval_awa2_arnoldi.sh b/scripts/awa2_resnet50_bench/eval_awa2_arnoldi.sh
deleted file mode 100755
index 809e6d7f..00000000
--- a/scripts/awa2_resnet50_bench/eval_awa2_arnoldi.sh
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-# Worker mode: run a single (method, benchmark) pair.
-if [ "${1:-}" = "--run" ]; then
- source "$(dirname "$0")/eval_defs.sh"
- EVAL_CONFIG_NAME="awa2_resnet50"
- PARALLEL=false
- methods=("$2")
- benchmarks=("$3")
- source "$(dirname "$0")/../eval.sh"
- exit
-fi
-
-# Submitter mode: one sbatch job per (method, benchmark);
-# every pt2 job waits on all pt1 jobs.
-methods=(
- arnoldi
-)
-
-bench_pt1=(
- awa2_class_detection
- #awa2_subclass_detection
- #awa2_shortcut_detection
- #awa2_mixed_datasets
- #awa2_mislabeling_detection
-)
-
-bench_pt2=(
- awa2_linear_datamodeling
- awa2_top_k_cardinality
- awa2_model_randomization
-)
-
-pt1_jids=()
-for method in "${methods[@]}"; do
- for bench in "${bench_pt1[@]}"; do
- jid=$(sbatch --parsable slurm/slurm_job.sbatch "$0" --run "$method" "$bench")
- [[ -n $jid ]] || { echo "$method $bench pt1 submission failed"; exit 1; }
- pt1_jids+=("$jid")
- done
-done
-
-dep=$(IFS=:; echo "${pt1_jids[*]}")
-for method in "${methods[@]}"; do
- for bench in "${bench_pt2[@]}"; do
- sbatch --dependency=afterok:$dep slurm/slurm_job.sbatch "$0" --run "$method" "$bench"
- done
-done
diff --git a/scripts/awa2_resnet50_bench/eval_awa2_local_arnoldi.sh b/scripts/awa2_resnet50_bench/eval_awa2_local_arnoldi.sh
deleted file mode 100755
index 2a184d19..00000000
--- a/scripts/awa2_resnet50_bench/eval_awa2_local_arnoldi.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/bash
-
-source "$(dirname "$0")/eval_defs.sh"
-
-EVAL_CONFIG_NAME="awa2_resnet50"
-
-benchmarks=(
- awa2_linear_datamodeling
- awa2_top_k_cardinality
- awa2_model_randomization
-)
-
-methods=(
- #similarity
- #representer_points
- #tracincpfast
- arnoldi
- #trak
- #random
-)
-PARALLEL=false
-
-source "$(dirname "$0")/../eval.sh" "$@"
diff --git a/scripts/awa2_resnet50_bench/eval_awa2_tracin.sh b/scripts/awa2_resnet50_bench/eval_awa2_tracin.sh
deleted file mode 100755
index 9083384c..00000000
--- a/scripts/awa2_resnet50_bench/eval_awa2_tracin.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/bin/bash
-
-source "$(dirname "$0")/eval_defs.sh"
-
-EVAL_CONFIG_NAME="awa2_resnet50"
-
-benchmarks=(
- #awa2_class_detection
- #awa2_subclass_detection
- #awa2_shortcut_detection
- awa2_mixed_datasets
- awa2_linear_datamodeling
- awa2_top_k_cardinality
- awa2_model_randomization
- awa2_mislabeling_detection
-)
-
-methods=(
- #similarity
- #representer_points
- tracincpfast
- #arnoldi
- #trak
- #random
-)
-PARALLEL=false
-
-source "$(dirname "$0")/../eval.sh" "$@"
diff --git a/scripts/awa2_resnet50_bench/eval_awa2_trak.sh b/scripts/awa2_resnet50_bench/eval_awa2_trak.sh
deleted file mode 100755
index 32780636..00000000
--- a/scripts/awa2_resnet50_bench/eval_awa2_trak.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/bin/bash
-
-source "$(dirname "$0")/eval_defs.sh"
-
-EVAL_CONFIG_NAME="awa2_resnet50"
-
-benchmarks=(
- awa2_class_detection
- awa2_subclass_detection
- awa2_shortcut_detection
- awa2_mixed_datasets
- awa2_linear_datamodeling
- awa2_top_k_cardinality
- awa2_model_randomization
- awa2_mislabeling_detection
-)
-
-methods=(
- #similarity
- #representer_points
- trak
- #arnoldi
- #trak
- #random
-)
-PARALLEL=false
-
-source "$(dirname "$0")/../eval.sh" "$@"
diff --git a/scripts/awa2_resnet50_bench/train_awa2.sh b/scripts/awa2_resnet50_bench/train_awa2.sh
index a5a93140..a780e8e3 100755
--- a/scripts/awa2_resnet50_bench/train_awa2.sh
+++ b/scripts/awa2_resnet50_bench/train_awa2.sh
@@ -6,11 +6,13 @@ CONFIG_NAME="awa2_resnet50"
CONFIG_MAP_PREFIX="awa2"
benchmarks=(
- #ClassDetection
- #SubclassDetection
+ ClassDetection
+ SubclassDetection
MixedDatasets
- #ShortcutDetection
- #MislabelingDetection
+ ShortcutDetection
+ MislabelingDetection
+ LDS
)
+PARALLEL=false
source "$(dirname "$0")/../train.sh" "$@"
diff --git a/scripts/awa2_resnet50_bench/train_awa2_lds.sh b/scripts/awa2_resnet50_bench/train_awa2_lds.sh
index 26e1ceb3..481c3f82 100755
--- a/scripts/awa2_resnet50_bench/train_awa2_lds.sh
+++ b/scripts/awa2_resnet50_bench/train_awa2_lds.sh
@@ -9,4 +9,5 @@ source "$(dirname "$0")/../train_lds.sh" \
--n-lds-parallel 1 \
--hf-push-sleep 10 \
--gpu-split false \
- "$@"
+ --start 0 \
+ --end 100
diff --git a/scripts/awa2_resnet50_bench/train_awa2_lds_all.sh b/scripts/awa2_resnet50_bench/train_awa2_lds_all.sh
deleted file mode 100755
index f9faa7c9..00000000
--- a/scripts/awa2_resnet50_bench/train_awa2_lds_all.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-
-M=100
-STRIDE=10
-
-for start in $(seq 0 "$STRIDE" "$((M - STRIDE))"); do
- end=$((start + STRIDE - 1))
- sbatch slurm/slurm_job.sbatch \
- scripts/awa2_resnet50_bench/train_awa2_lds.sh \
- --start "$start" --end "$end"
-done
diff --git a/scripts/awa2_resnet50_bench/train_awa2_per_bench.sh b/scripts/awa2_resnet50_bench/train_awa2_per_bench.sh
deleted file mode 100755
index 291ab955..00000000
--- a/scripts/awa2_resnet50_bench/train_awa2_per_bench.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-# Train a single benchmark. Hydra n_jobs is capped by MAX_PARALLEL
-# (default 6 — sized for one 40GB GPU at batch_size=64; lower if you
-# hit OOM, raise on a bigger GPU). The sweep cardinality (n_trials) is
-# whatever bench_defs.sh sets per benchmark; optuna runs them in
-# batches of MAX_PARALLEL.
-#
-# Usage: train_awa2_per_bench.sh BENCH_NAME [extra train.sh args]
-# Env: MAX_PARALLEL — override the parallelism cap.
-
-source "$(dirname "$0")/bench_defs.sh"
-
-CONFIG_NAME="awa2_resnet50"
-CONFIG_MAP_PREFIX="awa2"
-
-BENCH="$1"
-shift
-
-MAX_PARALLEL=${MAX_PARALLEL:-3}
-
-if [ -z "${BENCH_PARAMS[$BENCH]+x}" ]; then
- echo "Error: unknown benchmark '$BENCH'" >&2
- exit 1
-fi
-
-BENCH_SWEEP[$BENCH]="${BENCH_SWEEP[$BENCH]} hydra.launcher.n_jobs=${MAX_PARALLEL} hydra.sweeper.n_jobs=${MAX_PARALLEL}"
-
-benchmarks=("$BENCH")
-
-source "$(dirname "$0")/../train.sh" "$@"
diff --git a/scripts/awa2_resnet50_bench/train_awa2_per_bench_all.sh b/scripts/awa2_resnet50_bench/train_awa2_per_bench_all.sh
deleted file mode 100755
index caee94fc..00000000
--- a/scripts/awa2_resnet50_bench/train_awa2_per_bench_all.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/bash
-
-BENCHMARKS=(
- ClassDetection
- SubclassDetection
- MixedDatasets
- ShortcutDetection
- MislabelingDetection
-)
-
-for bench in "${BENCHMARKS[@]}"; do
- sbatch slurm/slurm_job.sbatch \
- scripts/awa2_resnet50_bench/train_awa2_per_bench.sh \
- "$bench"
-done
-
\ No newline at end of file
diff --git a/scripts/awa2_resnet50_bench/train_awa2_pipeline_all.sh b/scripts/awa2_resnet50_bench/train_awa2_pipeline_all.sh
deleted file mode 100755
index b54708ec..00000000
--- a/scripts/awa2_resnet50_bench/train_awa2_pipeline_all.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/bin/bash
-# Chain two fan-outs with SLURM dependencies:
-# 1. LDS subset training (M/STRIDE jobs)
-# 2. LDS subset logit computation — runs after stage 1 succeeds
-# Any failure in a stage cancels the dependent stages via --kill-on-invalid-dep.
-
-set -euo pipefail
-
-M=100
-STRIDE=1
-
-# ---- Stage 1: LDS subset training ------------------------------------------
-stage2_ids=()
-for start in $(seq 0 "$STRIDE" "$((M - STRIDE))"); do
- end=$((start + STRIDE - 1))
- jid=$(sbatch --parsable \
- slurm/slurm_job.sbatch \
- scripts/awa2_resnet50_bench/train_awa2_lds.sh \
- --start "$start" --end "$end")
- stage2_ids+=("$jid")
-done
-dep2=$(IFS=:; echo "${stage2_ids[*]}")
-
-# ---- Stage 2: compute LDS subset logits ------------------------------------
-BATCH_SIZE=64
-MAX_EVAL_N=1000
-EVAL_SEED=42
-INFERENCE_BATCH_SIZE=64
-DEVICE=cuda:0
-
-stage3_ids=()
-for start in $(seq 0 "$STRIDE" "$((M - STRIDE))"); do
- end=$((start + STRIDE))
- jid=$(sbatch --parsable \
- --dependency=afterok:"$dep2" --kill-on-invalid-dep=yes \
- slurm/slurm_job.sbatch \
- scripts/awa2_resnet50_bench/compute_lds_subset_logits_awa2.sh \
- --start "$start" --end "$end" \
- --batch-size "$BATCH_SIZE" \
- --max-eval-n "$MAX_EVAL_N" \
- --eval-seed "$EVAL_SEED" \
- --inference-batch-size "$INFERENCE_BATCH_SIZE" \
- --device "$DEVICE")
- stage3_ids+=("$jid")
-done
-
-echo "Stage 1 (LDS train): ${stage2_ids[*]}"
-echo "Stage 2 (LDS logits): ${stage3_ids[*]}"
diff --git a/scripts/bert_qnli_bench/compute_lds_subset_logits_qnli_all.sh b/scripts/bert_qnli_bench/compute_lds_subset_logits_qnli_all.sh
deleted file mode 100755
index 3662a7be..00000000
--- a/scripts/bert_qnli_bench/compute_lds_subset_logits_qnli_all.sh
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/bin/bash
-
-M=100
-STRIDE=10
-
-BATCH_SIZE=8
-MAX_EVAL_N=1000
-EVAL_SEED=42
-INFERENCE_BATCH_SIZE=32
-DEVICE=cuda:0
-
-for start in $(seq 0 "$STRIDE" "$((M - STRIDE))"); do
- end=$((start + STRIDE))
- sbatch slurm/slurm_job.sbatch \
- scripts/bert_qnli_bench/compute_lds_subset_logits_qnli.sh \
- --start "$start" --end "$end" \
- --batch-size "$BATCH_SIZE" \
- --max-eval-n "$MAX_EVAL_N" \
- --eval-seed "$EVAL_SEED" \
- --inference-batch-size "$INFERENCE_BATCH_SIZE" \
- --device "$DEVICE"
-done
diff --git a/scripts/bert_qnli_bench/train_qnli.sh b/scripts/bert_qnli_bench/train_qnli.sh
index 949af5b2..3300891a 100755
--- a/scripts/bert_qnli_bench/train_qnli.sh
+++ b/scripts/bert_qnli_bench/train_qnli.sh
@@ -6,7 +6,10 @@ CONFIG_NAME="bert_qnli"
CONFIG_MAP_PREFIX="qnli"
benchmarks=(
+ ClassDetection
MixedDatasets
+ MislabelingDetection
+ LDS
)
source "$(dirname "$0")/../train.sh" --parallel false --train-only false "$@"
diff --git a/scripts/bert_qnli_bench/train_qnli_lds.sh b/scripts/bert_qnli_bench/train_qnli_lds.sh
index bcb0466f..4ceded13 100755
--- a/scripts/bert_qnli_bench/train_qnli_lds.sh
+++ b/scripts/bert_qnli_bench/train_qnli_lds.sh
@@ -10,6 +10,4 @@ source "$(dirname "$0")/../train_lds.sh" \
--hf-push-sleep 30 \
--gpu-split false \
--start 0 \
- --end 100 \
- --train-only true \
- --push-only true
+ --end 100
diff --git a/scripts/cifar_resnet9_bench/eval_all_cifar.sh b/scripts/cifar_resnet9_bench/eval_all_cifar.sh
deleted file mode 100644
index a5ef4fe7..00000000
--- a/scripts/cifar_resnet9_bench/eval_all_cifar.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-set -euo pipefail
-
-jid1=$(sbatch --parsable slurm/slurm_job.sbatch scripts/cifar_resnet9_bench/eval_cifar_pt1.sh)
-[[ -n $jid1 ]] || { echo "pt1 submission failed"; exit 1; }
-
-sbatch --dependency=afterok:$jid1 slurm/slurm_job.sbatch scripts/cifar_resnet9_bench/eval_cifar_pt2.sh
-
diff --git a/scripts/cifar_resnet9_bench/train_cifar_lds.sh b/scripts/cifar_resnet9_bench/train_cifar_lds.sh
index 636d2c7a..9f9428b0 100755
--- a/scripts/cifar_resnet9_bench/train_cifar_lds.sh
+++ b/scripts/cifar_resnet9_bench/train_cifar_lds.sh
@@ -6,4 +6,6 @@ CONFIG_NAME="cifar_resnet9"
source "$(dirname "$0")/../train_lds.sh" \
--n-lds-parallel 20 \
- --hf-push-sleep 20
+ --hf-push-sleep 20 \
+ --start 0 \
+ --end 100
diff --git a/scripts/gpt2_trex_bench/eval_all_gpt2_trex.sh b/scripts/gpt2_trex_bench/eval_all_gpt2_trex.sh
deleted file mode 100755
index 43c78db6..00000000
--- a/scripts/gpt2_trex_bench/eval_all_gpt2_trex.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-
-methods=(
- random
- kronfluence_gpt2
- dattri_if_datainf
- dattri_trak
- similarity
-)
-
-mrr_jids=()
-for method in "${methods[@]}"; do
- jid=$(sbatch --parsable slurm/slurm_job.sbatch \
- scripts/gpt2_trex_bench/eval_mrr.sh \
- --method "$method")
- [[ -n $jid ]] || { echo "mrr submission failed for $method"; exit 1; }
- mrr_jids+=("$jid")
-done
-
-for i in "${!methods[@]}"; do
- method="${methods[$i]}"
- jid="${mrr_jids[$i]}"
-
- sbatch --dependency=afterok:$jid slurm/slurm_job.sbatch \
- scripts/gpt2_trex_bench/eval_recall_at_k.sh \
- --method "$method"
-
- sbatch --dependency=afterok:$jid slurm/slurm_job.sbatch \
- scripts/gpt2_trex_bench/eval_tail_patch.sh \
- --method "$method"
-done
diff --git a/scripts/gpt2_trex_bench/eval_mrr_all.sh b/scripts/gpt2_trex_bench/eval_mrr_all.sh
deleted file mode 100755
index 29983cfe..00000000
--- a/scripts/gpt2_trex_bench/eval_mrr_all.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-
-methods=(
- random
- kronfluence_gpt2
- dattri_if_datainf
- dattri_trak
- similarity
-)
-
-for method in "${methods[@]}"; do
- sbatch slurm/slurm_job.sbatch \
- scripts/gpt2_trex_bench/eval_mrr.sh \
- --method "$method"
-done
diff --git a/scripts/gpt2_trex_bench/eval_recall_at_k_all.sh b/scripts/gpt2_trex_bench/eval_recall_at_k_all.sh
deleted file mode 100755
index a5debc9e..00000000
--- a/scripts/gpt2_trex_bench/eval_recall_at_k_all.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-
-methods=(
- random
- kronfluence_gpt2
- dattri_if_datainf
- dattri_trak
- similarity
-)
-
-for method in "${methods[@]}"; do
- sbatch slurm/slurm_job.sbatch \
- scripts/gpt2_trex_bench/eval_recall_at_k.sh \
- --method "$method"
-done
diff --git a/scripts/gpt2_trex_bench/eval_tail_patch_all.sh b/scripts/gpt2_trex_bench/eval_tail_patch_all.sh
deleted file mode 100755
index 4325f229..00000000
--- a/scripts/gpt2_trex_bench/eval_tail_patch_all.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-
-methods=(
- random
- kronfluence_gpt2
- dattri_if_datainf
- dattri_trak
- similarity
-)
-
-for method in "${methods[@]}"; do
- sbatch slurm/slurm_job.sbatch \
- scripts/gpt2_trex_bench/eval_tail_patch.sh \
- --method "$method"
-done
diff --git a/scripts/mnsit_lenet_bench/eval_all_mnist.sh b/scripts/mnsit_lenet_bench/eval_all_mnist.sh
deleted file mode 100644
index dc3837c6..00000000
--- a/scripts/mnsit_lenet_bench/eval_all_mnist.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-set -euo pipefail
-
-jid1=$(sbatch --parsable slurm/slurm_job.sbatch scripts/mnsit_lenet_bench/eval_mnist_pt1.sh)
-[[ -n $jid1 ]] || { echo "pt1 submission failed"; exit 1; }
-
-sbatch --dependency=afterok:$jid1 slurm/slurm_job.sbatch scripts/mnsit_lenet_bench/eval_mnist_pt2.sh
diff --git a/scripts/mnsit_lenet_bench/train_mnist_lds.sh b/scripts/mnsit_lenet_bench/train_mnist_lds.sh
index 53acde4b..47bccda7 100755
--- a/scripts/mnsit_lenet_bench/train_mnist_lds.sh
+++ b/scripts/mnsit_lenet_bench/train_mnist_lds.sh
@@ -6,4 +6,6 @@ CONFIG_NAME="mnist_lenet"
source "$(dirname "$0")/../train_lds.sh" \
--n-lds-parallel 16 \
- --hf-push-sleep 60
+ --hf-push-sleep 60 \
+ --start 0 \
+ --end 100
From 6d2ea96413b650ce9d0a5e78ef23e7639845bdaa Mon Sep 17 00:00:00 2001
From: Dilyara Bareeva
Date: Mon, 4 May 2026 14:59:03 +0200
Subject: [PATCH 03/39] chore: start anonymization pt1
---
CODE_OF_CONDUCT.md | 2 +-
CONTRIBUTING.md | 2 +-
LICENSE | 2 +-
README.md | 4 ++--
config/eval/awa2_resnet50.yaml | 2 +-
config/eval/bert_qnli.yaml | 2 +-
config/eval/cifar_resnet9.yaml | 2 +-
config/eval/gpt2_trex.yaml | 2 +-
config/eval/mnist_lenet.yaml | 2 +-
pyproject.toml | 4 ++--
.../configs/5d5968d-awa2_resnet50_ClassDetection.yaml | 2 +-
.../resources/configs/5d5968d-awa2_resnet50_LDS.yaml | 2 +-
.../configs/5d5968d-awa2_resnet50_MislabelingDetection.yaml | 2 +-
.../configs/5d5968d-awa2_resnet50_SubclassDetection.yaml | 2 +-
14 files changed, 16 insertions(+), 16 deletions(-)
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
index 6b511400..9f0d809e 100644
--- a/CODE_OF_CONDUCT.md
+++ b/CODE_OF_CONDUCT.md
@@ -60,7 +60,7 @@ representative at an online or offline event.
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported to the community leaders responsible for enforcement at
-dilyabareeva@gmail.com.
+.
All complaints will be reviewed and investigated promptly and fairly.
All community leaders are obligated to respect the privacy and security of the
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 28767e0b..db2893f0 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -5,7 +5,7 @@
In this guide, you will get a summary of the main components of **quanda**, as well as best practices for your own contributions.
-If you have any questions regarding the codebase, please [open an issue](https://github.com/dilyabareeva/quanda/issues/new/choose) or write us at [dilyabareeva@gmail.com](mailto:dilyabareeva@gmail.com) or [galip.uemit.yolcu@hhi.fraunhofer.de](mailto:galip.uemit.yolcu@hhi.fraunhofer.de).
+If you have any questions regarding the codebase, please [open an issue](https://github.com/dilyabareeva/quanda/issues/new/choose) or write us at [](mailto:) or [](mailto:).
## Table of Contents
diff --git a/LICENSE b/LICENSE
index d7c82505..49d4d9a2 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,6 +1,6 @@
MIT License
-Copyright (c) 2024 Dilyara Bareeva, Galip Ümit Yolcu
+Copyright (c) 2026 Anynomous quanda authors
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
diff --git a/README.md b/README.md
index ba0c2781..f64d05d2 100644
--- a/README.md
+++ b/README.md
@@ -553,14 +553,14 @@ We welcome contributions to **quanda**! You could contribute by:
A detailed guide on how to contribute to **quanda** can be found [here](CONTRIBUTING.md).
## ✉️ Contact
-If you have any questions regarding the codebase, please open an issue or contact us via email at [dilyabareeva@gmail.com](mailto:dilyabareeva@gmail.com) or [galip.uemit.yolcu@hhi.fraunhofer.de](mailto:galip.uemit.yolcu@hhi.fraunhofer.de).
+If you have any questions regarding the codebase, please open an issue or contact us via email at [](mailto:) or [](mailto:).
## 🔗Citation
```bibtex
@misc{bareeva2024quandainterpretabilitytoolkittraining,
title={Quanda: An Interpretability Toolkit for Training Data Attribution Evaluation and Beyond},
- author={Dilyara Bareeva and Galip Ümit Yolcu and Anna Hedström and Niklas Schmolenski and Thomas Wiegand and Wojciech Samek and Sebastian Lapuschkin},
+ author={Author 1 and Author 2 and Anna Hedström and Niklas Schmolenski and Thomas Wiegand and Wojciech Samek and Sebastian Lapuschkin},
year={2024},
eprint={2410.07158},
archivePrefix={arXiv},
diff --git a/config/eval/awa2_resnet50.yaml b/config/eval/awa2_resnet50.yaml
index 6a9dac06..83c9bf5c 100644
--- a/config/eval/awa2_resnet50.yaml
+++ b/config/eval/awa2_resnet50.yaml
@@ -12,7 +12,7 @@ hydra:
n_jobs: 1
bench: awa2_class_detection
-root_dir: ${cluster_or_local:/data/cluster/users/bareeva/quanda_output_new2,/data2/bareeva/Projects/quanda/cluster_output_new2}
+root_dir: bench_out
cache_dir: ${root_dir}/eval_bench/awa2
results_dir: ${root_dir}/eval_results/awa2
diff --git a/config/eval/bert_qnli.yaml b/config/eval/bert_qnli.yaml
index d366700e..e088ddf6 100644
--- a/config/eval/bert_qnli.yaml
+++ b/config/eval/bert_qnli.yaml
@@ -12,7 +12,7 @@ hydra:
n_jobs: 1
bench: qnli_class_detection
-root_dir: ${cluster_or_local:/data/cluster/users/bareeva/quanda_output_new2,/data2/bareeva/Projects/quanda/cluster_output_new2}
+root_dir: bench_out
cache_dir: ${root_dir}/eval_bench/qnli
results_dir: ${root_dir}/eval_results/qnli
device: cuda:0
diff --git a/config/eval/cifar_resnet9.yaml b/config/eval/cifar_resnet9.yaml
index 3279642c..92f8f5a8 100644
--- a/config/eval/cifar_resnet9.yaml
+++ b/config/eval/cifar_resnet9.yaml
@@ -12,7 +12,7 @@ hydra:
n_jobs: 1
bench: cifar_class_detection
-root_dir: ${cluster_or_local:/data/cluster/users/bareeva/quanda_output_new2,/data2/bareeva/Projects/quanda/cluster_output_new2}
+root_dir: bench_out
cache_dir: ${root_dir}/eval_bench/cifar
results_dir: ${root_dir}/eval_results/cifar
diff --git a/config/eval/gpt2_trex.yaml b/config/eval/gpt2_trex.yaml
index 53f6e348..b0c27203 100644
--- a/config/eval/gpt2_trex.yaml
+++ b/config/eval/gpt2_trex.yaml
@@ -12,7 +12,7 @@ hydra:
n_jobs: 1
bench: gpt2_trex_openwebtext_ft_mrr
-root_dir: ${cluster_or_local:/data/cluster/users/bareeva/quanda_output_new2,/data2/bareeva/Projects/quanda/cluster_output_new2}
+root_dir: bench_out
cache_dir: ${root_dir}/eval_bench/gpt2_trex
results_dir: ${root_dir}/eval_results/gpt2_trex
device: cuda:0
diff --git a/config/eval/mnist_lenet.yaml b/config/eval/mnist_lenet.yaml
index c2a8890e..914dbddd 100644
--- a/config/eval/mnist_lenet.yaml
+++ b/config/eval/mnist_lenet.yaml
@@ -12,7 +12,7 @@ hydra:
n_jobs: 1
bench: mnist_class_detection
-root_dir: ${cluster_or_local:/data/cluster/users/bareeva/quanda_output_new2,/data2/bareeva/Projects/quanda/cluster_output_new2}
+root_dir: bench_out
cache_dir: ${root_dir}/eval_bench/mnist
results_dir: ${root_dir}/eval_results/mnist
diff --git a/pyproject.toml b/pyproject.toml
index d7c232c8..b25f2d2c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -2,8 +2,8 @@
name = "quanda"
dynamic = ["version"]
authors = [
- { name="Dilyara Bareeva", email="dilyabareeva@gmail.com" },
- {name = "Galip Ümit Yolcu", email = "galip.uemit.yolcu@hhi.fraunhofer.de" },
+ { name="Author 1", email="" },
+ {name = "Author 2", email = "" },
]
description = "Toolkit for quantitative evaluation of data attribution methods in PyTorch."
license = { file = "LICENSE" }
diff --git a/quanda/benchmarks/resources/configs/5d5968d-awa2_resnet50_ClassDetection.yaml b/quanda/benchmarks/resources/configs/5d5968d-awa2_resnet50_ClassDetection.yaml
index 6ad63adb..92d96528 100644
--- a/quanda/benchmarks/resources/configs/5d5968d-awa2_resnet50_ClassDetection.yaml
+++ b/quanda/benchmarks/resources/configs/5d5968d-awa2_resnet50_ClassDetection.yaml
@@ -44,7 +44,7 @@ val_dataset:
id: GIT_TAG-awa2_resnet50_ClassDetection
bench: ClassDetection
adversarial_label: 0
-bench_save_dir: /data/cluster/users/bareeva/quanda_output_new2/eval_bench/awa2
+bench_save_dir: bench_out/eval_bench/awa2
log_dir: hydra_logs
repo_id: quanda-bench-test
cache_dir: tmp
diff --git a/quanda/benchmarks/resources/configs/5d5968d-awa2_resnet50_LDS.yaml b/quanda/benchmarks/resources/configs/5d5968d-awa2_resnet50_LDS.yaml
index 18b825f3..7b2c2ba1 100644
--- a/quanda/benchmarks/resources/configs/5d5968d-awa2_resnet50_LDS.yaml
+++ b/quanda/benchmarks/resources/configs/5d5968d-awa2_resnet50_LDS.yaml
@@ -44,7 +44,7 @@ val_dataset:
id: GIT_TAG-awa2_resnet50_LDS
bench: LDS
adversarial_label: 0
-bench_save_dir: /data/cluster/users/bareeva/quanda_output_new2/eval_bench/awa2
+bench_save_dir: bench_out/eval_bench/awa2
log_dir: hydra_logs
repo_id: quanda-bench-test
cache_dir: tmp
diff --git a/quanda/benchmarks/resources/configs/5d5968d-awa2_resnet50_MislabelingDetection.yaml b/quanda/benchmarks/resources/configs/5d5968d-awa2_resnet50_MislabelingDetection.yaml
index 3a0ad4b9..91d02008 100644
--- a/quanda/benchmarks/resources/configs/5d5968d-awa2_resnet50_MislabelingDetection.yaml
+++ b/quanda/benchmarks/resources/configs/5d5968d-awa2_resnet50_MislabelingDetection.yaml
@@ -50,7 +50,7 @@ val_dataset:
id: GIT_TAG-awa2_resnet50_MislabelingDetection
bench: MislabelingDetection
adversarial_label: 0
-bench_save_dir: /data/cluster/users/bareeva/quanda_output_new2/eval_bench/awa2
+bench_save_dir: bench_out/eval_bench/awa2
log_dir: hydra_logs
repo_id: quanda-bench-test
cache_dir: tmp
diff --git a/quanda/benchmarks/resources/configs/5d5968d-awa2_resnet50_SubclassDetection.yaml b/quanda/benchmarks/resources/configs/5d5968d-awa2_resnet50_SubclassDetection.yaml
index 7b2cbed4..d239974a 100644
--- a/quanda/benchmarks/resources/configs/5d5968d-awa2_resnet50_SubclassDetection.yaml
+++ b/quanda/benchmarks/resources/configs/5d5968d-awa2_resnet50_SubclassDetection.yaml
@@ -80,7 +80,7 @@ val_dataset:
id: GIT_TAG-awa2_resnet50_SubclassDetection
bench: SubclassDetection
adversarial_label: 0
-bench_save_dir: /data/cluster/users/bareeva/quanda_output_new2/eval_bench/awa2
+bench_save_dir: bench_out/eval_bench/awa2
log_dir: hydra_logs
repo_id: quanda-bench-test
cache_dir: tmp
From 84dd6d3a3f7a15dbb7b9ca01b99ef7c1bb6859c6 Mon Sep 17 00:00:00 2001
From: Dilyara Bareeva
Date: Mon, 4 May 2026 15:00:40 +0200
Subject: [PATCH 04/39] chore: remove slurm folder
---
.gitignore | 3 +-
slurm/build.sh | 7 -----
slurm/copy_slurm_job.sh | 12 --------
slurm/debug.sh | 12 --------
slurm/env_quanda.def | 62 -----------------------------------------
slurm/run.sh | 16 -----------
slurm/slurm_job.sbatch | 22 ---------------
7 files changed, 2 insertions(+), 132 deletions(-)
delete mode 100755 slurm/build.sh
delete mode 100644 slurm/copy_slurm_job.sh
delete mode 100755 slurm/debug.sh
delete mode 100644 slurm/env_quanda.def
delete mode 100755 slurm/run.sh
delete mode 100644 slurm/slurm_job.sbatch
diff --git a/.gitignore b/.gitignore
index 300414ee..30e20e0e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -91,4 +91,5 @@ slurm/get_logs.sh
scripts/delete_hf.py
scripts/plot_results_LOCAL.py
scripts/**/plot_*_LOCAL.sh
-scripts/PLOT_ALL_LOCAL.sh
\ No newline at end of file
+scripts/PLOT_ALL_LOCAL.sh
+slurm_LOCAL/*
diff --git a/slurm/build.sh b/slurm/build.sh
deleted file mode 100755
index 374ab1ca..00000000
--- a/slurm/build.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-# Build the quanda apptainer image.
-
-set -euo pipefail
-
-cd "$(dirname "$0")/.."
-apptainer build --force --fakeroot slurm/env_quanda.sif slurm/env_quanda.def
diff --git a/slurm/copy_slurm_job.sh b/slurm/copy_slurm_job.sh
deleted file mode 100644
index b7fd9ca9..00000000
--- a/slurm/copy_slurm_job.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-
-SRC="bareeva@vca-gpu-0503-01:/data/cluster/users/bareeva/quanda_output_new2"
-DST="/data2/bareeva/Projects/quanda/cluster_output_new2"
-
-mkdir -p "$DST"
-rsync -au "$SRC/" "$DST/"
-rsync -au "$DST/" "$SRC/"
-
-
-
-#before=$(find /data/cluster/users/bareeva/quanda_output_new2/eval_results -type f | wc -l); find /data/cluster/users/bareeva/quanda_output_new2/eval_results -type f -not -newermt 2026-04-29 -delete; after=$(find /data/cluster/users/bareeva/quanda_output_new2/eval_results -type f | wc -l); echo "Files before: $before"; echo "Files after: $after"; echo "Deleted: $((before - after))"; echo; echo "Remaining oldest files:"; find /data/cluster/users/bareeva/quanda_output_new2/eval_results -type f -printf '%TY-%Tm-%Td %TH:%TM %p\n' | sort | head -3
\ No newline at end of file
diff --git a/slurm/debug.sh b/slurm/debug.sh
deleted file mode 100755
index cc812257..00000000
--- a/slurm/debug.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-# Drop into an interactive shell inside the quanda container.
-set -euo pipefail
-
-export CUDA_VISIBLE_DEVICES=1
-
-apptainer shell --nv \
- --env HF_HOME=/data/cluster/users/bareeva/.hf_cache \
- --bind "$(pwd):/workspace" \
- --bind /data/cluster/users/bareeva:/data/cluster/users/bareeva \
- --pwd /workspace \
- "$(dirname "$0")/env_quanda.sif"
diff --git a/slurm/env_quanda.def b/slurm/env_quanda.def
deleted file mode 100644
index 7797200f..00000000
--- a/slurm/env_quanda.def
+++ /dev/null
@@ -1,62 +0,0 @@
-Bootstrap: docker
-From: pytorch/pytorch:2.6.0-cuda12.4-cudnn9-devel
-
-%files
- pyproject.toml /opt/quanda/pyproject.toml
- README.md /opt/quanda/README.md
-
-%post
- # Install system dependencies
- apt-get update && apt-get install -y \
- git \
- wget \
- curl \
- build-essential \
- gcc \
- g++ \
- && rm -rf /var/lib/apt/lists/*
-
- # Install uv
- curl -LsSf https://astral.sh/uv/install.sh | sh
- . $HOME/.local/bin/env
-
- # Create virtual environment with uv using Python 3.11
- # (quanda requires >=3.10,<3.12; 3.11 matches tox type env)
- $HOME/.local/bin/uv venv /opt/venv --python 3.11
- . /opt/venv/bin/activate
-
- # Stub out the package tree so ``pip install .[dev]`` resolves without
- # needing the real source (which is bind-mounted at runtime).
- mkdir -p /opt/quanda/quanda
- touch /opt/quanda/quanda/__init__.py
-
- cd /opt/quanda
- # setuptools-scm needs a version when building outside a git tree.
- SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 \
- $HOME/.local/bin/uv pip install --no-cache-dir ".[dev]"
-
- # `.[dev]` can pull a newer torch (cu13) that mismatches the base
- # image's nvcc 12.4. Re-pin to the cu124 wheel before building fast-jl.
- $HOME/.local/bin/uv pip install --no-cache-dir --force-reinstall \
- --index-url https://download.pytorch.org/whl/cu124 \
- torch==2.6.0 torchvision==0.21.0
-
- # traker[fast] builds fast-jl against the base image's torch + nvcc;
- # --no-build-isolation is required so it sees the env's torch.
- $HOME/.local/bin/uv pip install --no-cache-dir --no-build-isolation "traker[fast]"
-
-%environment
- export PATH=/opt/venv/bin:$HOME/.local/bin:$PATH
-
- # Prevent Python from using ~/.local packages
- export PYTHONNOUSERSITE=1
-
- # Make the bind-mounted source tree importable as ``quanda``
- export PYTHONPATH=/workspace:$PYTHONPATH
-
- # Cache locations
- export HF_HOME=/data/cluster/users/bareeva/.hf_cache
- export PIP_CACHE_DIR=~/.cache/pip
-
-%runscript
- exec python "$@"
diff --git a/slurm/run.sh b/slurm/run.sh
deleted file mode 100755
index 40ec3906..00000000
--- a/slurm/run.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/bash
-# Interactive one-off run of a Python or Bash script inside the quanda container.
-# Usage: ./slurm/run.sh path/to/script.{py,sh} [args...]
-set -euo pipefail
-
-case "$1" in
- *.sh) interpreter=bash ;;
- *) interpreter=python ;;
-esac
-
-apptainer exec --nv \
- --env HF_HOME=/data/cluster/users/bareeva/.hf_cache \
- --bind "$(pwd):/workspace" \
- --bind /data/cluster/users/bareeva:/data/cluster/users/bareeva \
- --pwd /workspace \
- "$(dirname "$0")/env_quanda.sif" "$interpreter" "$@"
diff --git a/slurm/slurm_job.sbatch b/slurm/slurm_job.sbatch
deleted file mode 100644
index 122e36d8..00000000
--- a/slurm/slurm_job.sbatch
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/bin/bash
-
-#SBATCH --job-name=quanda
-#SBATCH --output=log/%j_%x.out
-#SBATCH --error=log/%j_%x.err
-#SBATCH --ntasks=1
-#SBATCH --cpus-per-task=8
-#SBATCH --gpus=1
-#SBATCH --mem=64G
-#SBATCH --partition=gpu3,gpu5
-
-# Hydra surfaces full tracebacks for scripts under scripts/*.
-export HYDRA_FULL_ERROR=1
-mkdir -p log
-
-apptainer exec --nv \
- --env HF_HOME=/data/cluster/users/bareeva/.hf_cache \
- --bind ${PWD}:/workspace \
- --bind /data/cluster/users/bareeva:/data/cluster/users/bareeva \
- --pwd /workspace \
- $PWD/slurm/env_quanda.sif \
- bash "$@"
From 0fb0d978b3a94ef0e8aa7e7e84082d2f5d459137 Mon Sep 17 00:00:00 2001
From: Dilyara Bareeva
Date: Mon, 4 May 2026 15:10:19 +0200
Subject: [PATCH 05/39] chore: anonymization pt2
---
.gitignore | 2 +-
CONTRIBUTING.md | 8 ++++----
pyproject.toml | 4 ----
.../configs/2fc831c-awa2_resnet50_MixedDatasets.yaml | 2 +-
.../2fc831c-awa2_resnet50_ShortcutDetection.yaml | 2 +-
.../configs/99a4f7b-bert_qnli_MixedDatasets.yaml | 2 +-
scripts/compute_lds_subset_logits.sh | 6 +-----
scripts/prefetch_bench.py | 10 +---------
scripts/run_bench_eval.py | 8 --------
scripts/train.sh | 6 +-----
scripts/train_lds.sh | 6 +-----
11 files changed, 12 insertions(+), 44 deletions(-)
diff --git a/.gitignore b/.gitignore
index 30e20e0e..0386db58 100644
--- a/.gitignore
+++ b/.gitignore
@@ -71,7 +71,7 @@ assets/demo/*
bench_out/*
fig_1_images/*
tutorials/*
-scripts/bench_out/*
+scriptsbench_out/*
CLAUDE.md
.vscode/
quanda_benchmark_tutorial_cache/*
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index db2893f0..d1f31e99 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -5,7 +5,7 @@
In this guide, you will get a summary of the main components of **quanda**, as well as best practices for your own contributions.
-If you have any questions regarding the codebase, please [open an issue](https://github.com/dilyabareeva/quanda/issues/new/choose) or write us at [](mailto:) or [](mailto:).
+If you have any questions regarding the codebase, please open an issue or write us at [](mailto:) or [](mailto:).
## Table of Contents
@@ -25,7 +25,7 @@ If you have any questions regarding the codebase, please [open an issue](https:/
## Reporting Bugs
-If you come across a bug in the software, please check the repository [Issues](https://github.com/dilyabareeva/quanda/issues) to see if this bug has already been reported. If the bug is not yet reported, please report the bug by [opening an issue](https://github.com/dilyabareeva/quanda/issues/new). Please pay attention to add a descriptive title for the bug. Briefly explain the bug in the issue body, and add details on how to reproduce the faulty behaviour whenever possible.
+If you come across a bug in the software, please check the repository Issues to see if this bug has already been reported. If the bug is not yet reported, please report the bug by opening an issue. Please pay attention to add a descriptive title for the bug. Briefly explain the bug in the issue body, and add details on how to reproduce the faulty behaviour whenever possible.
We will address the issue at our earliest convenience.
@@ -128,10 +128,10 @@ python3 -m tox run -e coverage
```
Once you are done with your contributions, and have went through the above checklist:
-- Create a [pull request](https://github.com/dilyabareeva/quanda/compare)
+- Create a pull request
- Provide a summary of the changes you are introducing, give details on points which might not be easily understandable.
- If the contribution is concerning an existing issue, refer to it in the body of the pull request.
-- Request a review from [dilyabareeva](https://github.com/dilyabareeva) or [gumityolcu](https://github.com/gumityolcu).
+- Request a review from the main contributors.
## Contributing Metrics and Benchmarks
diff --git a/pyproject.toml b/pyproject.toml
index b25f2d2c..fae9bd65 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -41,10 +41,6 @@ dependencies = [
]
-[project.urls]
-Homepage = "https://github.com/dilyabareeva/quanda"
-Issues = "https://github.com/dilyabareeva/quanda/issues"
-
[build-system]
requires = ["setuptools>=42", "setuptools-scm[toml]>=6.0"]
build-backend = "setuptools.build_meta"
diff --git a/quanda/benchmarks/resources/configs/2fc831c-awa2_resnet50_MixedDatasets.yaml b/quanda/benchmarks/resources/configs/2fc831c-awa2_resnet50_MixedDatasets.yaml
index 2adf3bce..08187283 100644
--- a/quanda/benchmarks/resources/configs/2fc831c-awa2_resnet50_MixedDatasets.yaml
+++ b/quanda/benchmarks/resources/configs/2fc831c-awa2_resnet50_MixedDatasets.yaml
@@ -47,7 +47,7 @@ val_dataset:
id: 2fc831c-awa2_resnet50_MixedDatasets
bench: MixedDatasets
adversarial_label: 0
-bench_save_dir: /data2/bareeva/Projects/quanda/cluster_output_new2/eval_bench/awa2
+bench_save_dir: bench_out/eval_bench/awa2
log_dir: hydra_logs
repo_id: quanda-bench-test
cache_dir: tmp
diff --git a/quanda/benchmarks/resources/configs/2fc831c-awa2_resnet50_ShortcutDetection.yaml b/quanda/benchmarks/resources/configs/2fc831c-awa2_resnet50_ShortcutDetection.yaml
index 33b3d138..cf1c2ff8 100644
--- a/quanda/benchmarks/resources/configs/2fc831c-awa2_resnet50_ShortcutDetection.yaml
+++ b/quanda/benchmarks/resources/configs/2fc831c-awa2_resnet50_ShortcutDetection.yaml
@@ -73,7 +73,7 @@ val_dataset:
id: 2fc831c-awa2_resnet50_ShortcutDetection
bench: ShortcutDetection
adversarial_label: 0
-bench_save_dir: /data2/bareeva/Projects/quanda/cluster_output_new2/eval_bench/awa2
+bench_save_dir: bench_out/eval_bench/awa2
log_dir: hydra_logs
repo_id: quanda-bench-test
cache_dir: tmp
diff --git a/quanda/benchmarks/resources/configs/99a4f7b-bert_qnli_MixedDatasets.yaml b/quanda/benchmarks/resources/configs/99a4f7b-bert_qnli_MixedDatasets.yaml
index 9975264b..c69e8e31 100644
--- a/quanda/benchmarks/resources/configs/99a4f7b-bert_qnli_MixedDatasets.yaml
+++ b/quanda/benchmarks/resources/configs/99a4f7b-bert_qnli_MixedDatasets.yaml
@@ -65,7 +65,7 @@ logger:
project: quanda-bench
id: 99a4f7b-bert_qnli_MixedDatasets
bench: MixedDatasets
-bench_save_dir: /data2/bareeva/Projects/quanda/cluster_output_new2/eval_bench/qnli
+bench_save_dir: bench_out/eval_bench/qnli
log_dir: hydra_logs
repo_id: quanda-bench-test
cache_dir: tmp
diff --git a/scripts/compute_lds_subset_logits.sh b/scripts/compute_lds_subset_logits.sh
index 00edadb2..a0347c19 100755
--- a/scripts/compute_lds_subset_logits.sh
+++ b/scripts/compute_lds_subset_logits.sh
@@ -34,11 +34,7 @@ if [ -z "$START" ] || [ -z "$END" ]; then
exit 1
fi
-if [ -d "/data/cluster/users/bareeva" ]; then
- BENCH_SAVE_DIR="/data/cluster/users/bareeva/quanda_output_new2/eval_bench/${CONFIG_MAP_PREFIX}"
-else
- BENCH_SAVE_DIR="/data2/bareeva/Projects/quanda/cluster_output_new2/eval_bench/${CONFIG_MAP_PREFIX}"
-fi
+BENCH_SAVE_DIR="bench_out/${CONFIG_MAP_KEY}"
CONFIG_PATH=$(python -c "
from quanda.benchmarks.resources.config_map import config_map
diff --git a/scripts/prefetch_bench.py b/scripts/prefetch_bench.py
index 34f48e58..37b4fcf2 100644
--- a/scripts/prefetch_bench.py
+++ b/scripts/prefetch_bench.py
@@ -5,18 +5,10 @@
import os
import hydra
-from omegaconf import DictConfig, OmegaConf
+from omegaconf import DictConfig
from quanda.benchmarks import bench_dict
-OmegaConf.register_new_resolver(
- "cluster_or_local",
- lambda cluster, local: (
- cluster if os.path.isdir("/data/cluster/users/bareeva") else local
- ),
- replace=True,
-)
-
_SUFFIX_TO_CLASS = {
"class_detection": "ClassDetection",
"subclass_detection": "SubclassDetection",
diff --git a/scripts/run_bench_eval.py b/scripts/run_bench_eval.py
index 6bac1e37..e7f98125 100644
--- a/scripts/run_bench_eval.py
+++ b/scripts/run_bench_eval.py
@@ -16,14 +16,6 @@
from quanda.benchmarks.base import default_explanations_id
from quanda.benchmarks.resources.config_map import config_map
-OmegaConf.register_new_resolver(
- "cluster_or_local",
- lambda cluster, local: (
- cluster if os.path.isdir("/data/cluster/users/bareeva") else local
- ),
- replace=True,
-)
-
_SUFFIX_TO_CLASS = {
"class_detection": "ClassDetection",
"subclass_detection": "SubclassDetection",
diff --git a/scripts/train.sh b/scripts/train.sh
index 991707e6..c7619de6 100755
--- a/scripts/train.sh
+++ b/scripts/train.sh
@@ -23,11 +23,7 @@ cfg_output_dir="quanda/benchmarks/resources/configs"
commit_tag=$(git rev-parse --short HEAD 2>/dev/null || echo "GIT_TAG")
mkdir -p logs
-if [ -d "/data/cluster/users/bareeva" ]; then
- bench_save_dir_override="bench_save_dir=/data/cluster/users/bareeva/quanda_output_new2/eval_bench/${CONFIG_MAP_PREFIX}"
-else
- bench_save_dir_override="bench_save_dir=/data2/bareeva/Projects/quanda/cluster_output_new2/eval_bench/${CONFIG_MAP_PREFIX}"
-fi
+bench_save_dir_override="bench_save_dir=bench_out/${CONFIG_MAP_PREFIX}"
# Map benchmark names to config_map.py keys
declare -A BENCH_CONFIG_MAP_KEY
diff --git a/scripts/train_lds.sh b/scripts/train_lds.sh
index 6ea2da28..967da083 100755
--- a/scripts/train_lds.sh
+++ b/scripts/train_lds.sh
@@ -50,11 +50,7 @@ resolve_indices() {
CFG_DIR="quanda/benchmarks/resources/configs"
mkdir -p logs
-if [ -d "/data/cluster/users/bareeva" ]; then
- BENCH_SAVE_DIR="/data/cluster/users/bareeva/quanda_output_new2/eval_bench/${CONFIG_MAP_PREFIX}"
-else
- BENCH_SAVE_DIR="/data2/bareeva/Projects/quanda/cluster_output_new2/eval_bench/${CONFIG_MAP_PREFIX}"
-fi
+BENCH_SAVE_DIR="bench_out/${CONFIG_MAP_PREFIX}"
SAVE_OVERRIDE="bench_save_dir=${BENCH_SAVE_DIR}"
# ---------- helpers ----------
From 07b6e57fdfe84208fd382e566dd1667f52815592 Mon Sep 17 00:00:00 2001
From: Dilyara Bareeva
Date: Mon, 4 May 2026 15:19:40 +0200
Subject: [PATCH 06/39] chore: anonymize docs
---
docs/source/conf.py | 4 ++--
docs/source/contributing.rst | 17 +++++++----------
docs/source/index.rst | 19 +------------------
docs/source/tutorial_pages/benchmarks.rst | 2 +-
docs/source/tutorials.rst | 4 ++--
5 files changed, 13 insertions(+), 33 deletions(-)
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 4ecb87af..607df294 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -13,8 +13,8 @@
sys.path.insert(0, os.path.abspath("../.."))
project = "quanda"
-copyright = f"{str(datetime.utcnow().year)}, Dilyara Bareeva, Galip Ümit Yolcu"
-author = "Dilyara Bareeva, Galip Ümit Yolcu"
+copyright = f"{str(datetime.utcnow().year)}, Anonymous quanda authors"
+author = "Anonymous quanda authors"
release = "05.10.2024"
# -- General configuration ---------------------------------------------------
diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst
index a3d7cd02..3b028206 100644
--- a/docs/source/contributing.rst
+++ b/docs/source/contributing.rst
@@ -8,9 +8,8 @@ to report any bugs you encounter while using |quanda|.
In this guide, you will get a summary of the main components of
|quanda|, as well as best practices for your own contributions.
-If you have any questions regarding the codebase, please `open an
-issue `__ or write us
-at dilyabareeva@gmail.com or galip.uemit.yolcu@hhi.fraunhofer.de.
+If you have any questions regarding the codebase, please open an
+issue or write us an e-mail.
Table of Contents
-----------------
@@ -36,10 +35,10 @@ Reporting Bugs
--------------
If you come across a bug in the software, please check the repository
-`Issues `__ to see if
+Issues to see if
this bug has already been reported. If the bug is not yet reported,
-please report the bug by `opening an
-issue `__. Please pay
+please report the bug by opening an
+issue. Please pay
attention to add a descriptive title for the bug. Briefly explain
the bug in the issue body, and add details on how to reproduce the faulty
behaviour whenever possible.
@@ -183,13 +182,11 @@ ensure a seamless review process:
python3 -m tox run -e coverage
Once you are done with your contributions, and have went through the
-above checklist: - Create a `pull
-request `__ - Provide a
+above checklist: - Create a pull request. - Provide a
summary of the changes you are introducing, give details on points which
might not be easily understandable. - If the contribution is concerning
an existing issue, refer to it in the body of the pull request. -
-Request a review from `dilyabareeva `__
-or `gumityolcu `__.
+Request a review from the main contributors.
Contributing Metrics and Benchmarks
-----------------------------------
diff --git a/docs/source/index.rst b/docs/source/index.rst
index e3a8b657..03562791 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -4,7 +4,7 @@ Welcome to |quanda|'s documentation!
|quanda| is a toolkit for **quan**\ titative evaluation of **d**\ ata **a**\ ttribution methods in **PyTorch**.
.. note::
- |quanda| is currently in development. We are actively working on expanding the library and improving the documentation. If you have any questions, please `open an issue `_ or write us at dilyabareeva@gmail.com or galip.uemit.yolcu@hhi.fraunhofer.de.
+ |quanda| is currently in development. We are actively working on expanding the library and improving the documentation. If you have any questions, please open an issue or write us an e-mail.
.. figure:: _static/fig_1_source.png
:alt: Figure 1
@@ -225,23 +225,6 @@ Benchmarks
- Vision / Text
- mnist_linear_datamodeling, cifar_linear_datamodeling, awa2_linear_datamodeling, qnli_linear_datamodeling
-Citation
---------
-If you find |quanda| useful and want to use it in your research, please cite it using the following BibTeX entry:
-
-.. code:: bibtex
-
- @misc{bareeva2024quandainterpretabilitytoolkittraining,
- title={Quanda: An Interpretability Toolkit for Training Data Attribution Evaluation and Beyond},
- author={Dilyara Bareeva and Galip Ümit Yolcu and Anna Hedström and Niklas Schmolenski and Thomas Wiegand and Wojciech Samek and Sebastian Lapuschkin},
- year={2024},
- eprint={2410.07158},
- archivePrefix={arXiv},
- primaryClass={cs.LG},
- url={https://arxiv.org/abs/2410.07158},
- }
-
-If you are using |quanda| for your scientific research, please also make sure to cite the original authors for the implemented metrics and TDA methods.
.. toctree::
:caption: Usage
diff --git a/docs/source/tutorial_pages/benchmarks.rst b/docs/source/tutorial_pages/benchmarks.rst
index b9ea56be..1a36316a 100644
--- a/docs/source/tutorial_pages/benchmarks.rst
+++ b/docs/source/tutorial_pages/benchmarks.rst
@@ -11,7 +11,7 @@ To install the library with tutorial dependencies, run:
.. note::
- This tutorial is also available as a `notebook `_.
+ This tutorial is also available as a `notebook `_.
Throughout this tutorial, we will be using a LeNet model trained on the MNIST dataset. Let's start the tutorial by importing the necessary libraries and components:
diff --git a/docs/source/tutorials.rst b/docs/source/tutorials.rst
index 0fcd3fb2..2ae2d9e0 100644
--- a/docs/source/tutorials.rst
+++ b/docs/source/tutorials.rst
@@ -8,8 +8,8 @@ We have included a few tutorials to demonstrate the usage of |quanda|. To instal
The tutorials currently included in |quanda| are:
-- `Explainers `_: shows how different explainers can be used with |quanda|. This tutorial goes through all the explainers that are included in |quanda| and walks through the steps of initializing the ``Explainer`` object, generating explanations and plotting them.
-- `Metrics `_: shows how to use the metrics in |quanda| to evaluate the performance of a method. This tutorial goes through all the metrics that are included in |quanda| and walks through the steps of initializing the metric and evaluating the performance of a TDA method.
+- `Explainers `_: shows how different explainers can be used with |quanda|. This tutorial goes through all the explainers that are included in |quanda| and walks through the steps of initializing the ``Explainer`` object, generating explanations and plotting them.
+- `Metrics `_: shows how to use the metrics in |quanda| to evaluate the performance of a method. This tutorial goes through all the metrics that are included in |quanda| and walks through the steps of initializing the metric and evaluating the performance of a TDA method.
- :doc:`Benchmarks `: shows how to use the benchmarking tools in |quanda| to evaluate a data attribution method. This tutorial includes 3 different examples of benchmarks.
.. toctree::
From 6ae2ab3bc110dab9fcd5381728b1f3911dbaa61f Mon Sep 17 00:00:00 2001
From: Dilyara Bareeva
Date: Mon, 4 May 2026 15:31:46 +0200
Subject: [PATCH 07/39] chore: update readme and docs
---
README.md | 41 +++-------
docs/source/background.rst | 2 +-
docs/source/how_to_evaluate.rst | 2 +-
docs/source/index.rst | 141 +++++++++++++++++++++-----------
4 files changed, 107 insertions(+), 79 deletions(-)
diff --git a/README.md b/README.md
index f64d05d2..07cb94e0 100644
--- a/README.md
+++ b/README.md
@@ -11,14 +11,11 @@
-
-
+


-[](https://codecov.io/gh/dilyabareeva/quanda)
+

-[](https://quanda.readthedocs.io/en/latest/?badge=latest)
-[](https://arxiv.org/abs/2410.07158)
**quanda** _is currently under active development. Note the release version to ensure reproducibility of your work. Expect changes to API._
@@ -56,24 +53,22 @@ Although there are various demonstrations of TDA’s potential for interpretabil
- **Metrics**: **quanda** provides a set of metrics to evaluate the effectiveness of TDA methods. These metrics are based on the latest research in the field.
- **Benchmarking**: **quanda** provides a benchmarking tool to evaluate the performance of TDA methods on a given model, dataset and problem. As many TDA evaluation methods require access to ground truth, our benchmarking tools allow to generate a controlled setting with ground truth, and then compare the performance of different TDA methods on this setting.
-### Supported TDA Methods
+### Supported TDA Libraries
-| Method Name | Repository | Reference |
-|----------------------------|------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------|
-| Similarity Influence | [Captum](https://github.com/pytorch/captum/tree/master) | [Caruana et al., 1999](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2232607/) |
-| Arnoldi Influence Function | [Captum](https://github.com/pytorch/captum/tree/master) | [Schioppa et al., 2022](https://arxiv.org/abs/2112.03052); [Koh and Liang, 2017](https://proceedings.mlr.press/v70/koh17a.html) |
-| TracIn | [Captum](https://github.com/pytorch/captum/tree/master) | [Pruthi et al., 2020](https://proceedings.neurips.cc/paper/2020/hash/e6385d39ec9394f2f3a354d9d2b88eec-Abstract.html) |
-| TRAK | [TRAK](https://github.com/MadryLab/trak) | [Park et al., 2023](https://proceedings.mlr.press/v202/park23c.html) |
-| Representer Point Selection | [Representer Point Selection](https://github.com/chihkuanyeh/Representer_Point_Selection) | [Yeh et al., 2018](https://proceedings.neurips.cc/paper/2018/hash/8a7129b8f3edd95b7d969dfc2c8e9d9d-Abstract.html) |
-| Kronfluence | [Kronfluence](https://github.com/pomonam/kronfluence) | [Grosse et al., 2023](https://arxiv.org/abs/2308.03296) |
-| Dattri (Influence Functions: Explicit / CG / LiSSA / DataInf, Arnoldi, EK-FAC, TracInCP, Grad-Dot, Grad-Cos, TRAK) | [Dattri](https://github.com/TRAIS-Lab/dattri) | [Deng et al., 2024](https://arxiv.org/abs/2410.04555) |
+| Library | Reference |
+|-----------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| [Captum](https://github.com/pytorch/captum/tree/master) (Similarity Influence, Arnoldi Influence Function, TracIn) | [Caruana et al., 1999](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2232607/); [Schioppa et al., 2022](https://arxiv.org/abs/2112.03052); [Koh and Liang, 2017](https://proceedings.mlr.press/v70/koh17a.html); [Pruthi et al., 2020](https://proceedings.neurips.cc/paper/2020/hash/e6385d39ec9394f2f3a354d9d2b88eec-Abstract.html) |
+| [TRAK](https://github.com/MadryLab/trak) (TRAK) | [Park et al., 2023](https://proceedings.mlr.press/v202/park23c.html) |
+| [Representer Point Selection](https://github.com/chihkuanyeh/Representer_Point_Selection) (Representer Point Selection) | [Yeh et al., 2018](https://proceedings.neurips.cc/paper/2018/hash/8a7129b8f3edd95b7d969dfc2c8e9d9d-Abstract.html) |
+| [Kronfluence](https://github.com/pomonam/kronfluence) (Kronfluence) | [Grosse et al., 2023](https://arxiv.org/abs/2308.03296) |
+| [Dattri](https://github.com/TRAIS-Lab/dattri) (Influence Functions: Explicit / CG / LiSSA / DataInf, Arnoldi, EK-FAC, TracInCP, Grad-Dot, Grad-Cos, TRAK) | [Deng et al., 2024](https://arxiv.org/abs/2410.04555) |
### Metrics
- **Linear Datamodeling Score** ([Park et al., 2023](https://proceedings.mlr.press/v202/park23c.html)): Measures the correlation between the (grouped) attribution scores and the actual output of models trained on different subsets of the training set. For each subset, the linear datamodeling score compares the actual model output to the sum of attribution scores from the subset using Spearman rank correlation.
-- **Identical Class / Identical Subclass** ([Hanawa et al., 2021](https://openreview.net/forum?id=9uvhpyQwzM_)): Measures the proportion of identical classes or subclasses in the top-1 training samples over the test dataset. If the attributions are based on similarity, they are expected to be predictive of the class of the test datapoint, as well as different subclasses under a single label.
+- **Class Detection / Subclass Detection** ([Hanawa et al., 2021](https://openreview.net/forum?id=9uvhpyQwzM_)): Measures the proportion of identical classes or subclasses in the top-1 training samples over the test dataset. If the attributions are based on similarity, they are expected to be predictive of the class of the test datapoint, as well as different subclasses under a single label.
- **Model Randomization** ([Hanawa et al., 2021](https://openreview.net/forum?id=9uvhpyQwzM_)): Measures the correlation between the original TDA and the TDA of a model with randomized weights. Since the attributions are expected to depend on model parameters, the correlation between original and randomized attributions should be low.
@@ -554,17 +549,3 @@ A detailed guide on how to contribute to **quanda** can be found [here](CONTRIBU
## ✉️ Contact
If you have any questions regarding the codebase, please open an issue or contact us via email at [](mailto:) or [](mailto:).
-
-## 🔗Citation
-
-```bibtex
-@misc{bareeva2024quandainterpretabilitytoolkittraining,
- title={Quanda: An Interpretability Toolkit for Training Data Attribution Evaluation and Beyond},
- author={Author 1 and Author 2 and Anna Hedström and Niklas Schmolenski and Thomas Wiegand and Wojciech Samek and Sebastian Lapuschkin},
- year={2024},
- eprint={2410.07158},
- archivePrefix={arXiv},
- primaryClass={cs.LG},
- url={https://arxiv.org/abs/2410.07158},
-}
-```
diff --git a/docs/source/background.rst b/docs/source/background.rst
index 9ad6e0e8..beb4c88f 100644
--- a/docs/source/background.rst
+++ b/docs/source/background.rst
@@ -1,5 +1,5 @@
What is Training Data Attribution?
-==========
+==================================
The interpretability of neural network decisions is an active area of research which has seen a variety of approaches over time. Most of the initial focus was on feature attribution methods, which highlight features in the input space that are responsible for a specific prediction (`Simonyan et al., 2014 `_; `Bach et al., 2015 `_; `Lundberg and Lee, 2017 `_). These methods were often criticized for being unreliable and difficult to understand (`Adebayo et al., 2018 `_; `Ghorbani et al., 2019 `_). In response, researchers explored new directions, such as concept-based (`Poeta et al., 2023 `_) and mechanistic interpretability (`Bereska and Gavves `_) methods. Recently, **Training Data Attribution** (TDA) has gained attention as a promising approach for enhancing the interpretability of neural networks.
diff --git a/docs/source/how_to_evaluate.rst b/docs/source/how_to_evaluate.rst
index b4ccf5ce..84ac2c66 100644
--- a/docs/source/how_to_evaluate.rst
+++ b/docs/source/how_to_evaluate.rst
@@ -1,5 +1,5 @@
How to Assess the Quality of Attributions?
-==========
+==========================================
Evaluation of interpretability approaches is a challenging task, as it is often difficult to define a ground truth for interpretability. Although there are various demonstrations of TDA’s potential for interpretability and practical applications, the critical question of how TDA methods should be effectively evaluated remains open. While methods based on estimating counterfactual retraining effects have a well-defined ground truth, this ground truth is computationally demanding and is not feasibly computable for large scale experiments. To address these shortcomings, several approaches have been proposed by the community, which can be categorized into three groups:
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 03562791..d1dab0ee 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -79,43 +79,23 @@ Note that many metrics require training models in controlled settings, e.g. with
-Supported TDA Methods
----------------------
+Supported TDA Libraries
+-----------------------
.. list-table::
:header-rows: 1
- * - Method
- - Repository
+ * - Library
- Reference
- - Description
- * - Similarity Influence
- - `Captum `_
- - `Caruana et al., 1999 `_
- - Ranks the training samples based on their similarity to the test sample
- * - Arnoldi Influence Functions
- - `Captum `_
- - `Schioppa et al., 2022 `_
- - Estimates LOO effects, following (`Koh and Liang, 2017 `_)
- * - TracIn
- - `Captum `_
- - `Pruthi et al., 2020 `_
- - Tracks the contribution of training points in the loss reduction throughout training, via a linear approximation
- * - Representer Point Selection
- - `Representer Point Selection `_
+ * - `Captum `_ (Similarity Influence, Arnoldi Influence Functions, TracIn)
+ - `Caruana et al., 1999 `_; `Schioppa et al., 2022 `_; `Koh and Liang, 2017 `_; `Pruthi et al., 2020 `_
+ * - `Representer Point Selection `_ (Representer Point Selection)
- `Yeh et al., 2018 `_
- - Trains the model with L2 regularization on the final layer, which produces an interpretable surrogate model
- * - TRAK
- - `TRAK `_
+ * - `TRAK `_ (TRAK)
- `Park et al., 2023 `_
- - Uses an empirical Neural Tangent Kernel surrogate model for which a theoretical TDA formula exists
- * - Kronfluence
- - `Kronfluence `_
+ * - `Kronfluence `_ (Kronfluence)
- `Grosse et al., 2023 `_
- - Estimates LOO effects with EK-FAC-based approximations to the inverse Hessian
- * - Dattri (Influence Functions: Explicit / CG / LiSSA / DataInf, Arnoldi, EK-FAC, TracInCP, Grad-Dot, Grad-Cos, TRAK)
- - `Dattri `_
+ * - `Dattri `_ (Influence Functions: Explicit / CG / LiSSA / DataInf, Arnoldi, EK-FAC, TracInCP, Grad-Dot, Grad-Cos, TRAK)
- `Deng et al., 2024 `_
- - Provides a unified family of TDA methods (influence functions, TracIn, gradient similarity, TRAK) via the ``Dattri`` library.
Evaluation Metrics
------------------
@@ -132,7 +112,7 @@ In this section, we list the evaluation criteria that are currently available in
- `Park et al., 2023 `_
- Measures the correlation between the (grouped) attribution scores and the actual output of models trained on different subsets of the training set. For each subset, the linear datamodeling score compares the actual model output with the sum of attribution scores from the subset using Spearman rank correlation.
- Ground Truth
- * - Identical Class / Identical Subclass
+ * - Class Detection / Subclass Detection
- `Hanawa et al., 2021 `_
- Measures the proportion of identical classes or subclasses in the top-1 training samples over the test dataset. If the attributions are based on similarity, they are expected to be predictive of the class of the test datapoint, as well as different subclasses under a single label.
- Downstream Task Evaluator
@@ -169,6 +149,49 @@ In this section, we list the evaluation criteria that are currently available in
- For fact-tracing settings, measures the incremental change in target-sequence probability after taking a single training step on retrieved proponents.
- Downstream Task Evaluator
+Metric Interpretation Guideline
++++++++++++++++++++++++++++++++
+
+.. list-table::
+ :header-rows: 1
+
+ * - Metric
+ - Output range
+ - Better
+ * - `ClassDetection `_
+ - ``[0, 1]``
+ - higher
+ * - `SubclassDetection `_
+ - ``[0, 1]``
+ - higher
+ * - `MislabelingDetection `_
+ - ``[0, 1]``
+ - higher
+ * - `ShortcutDetection `_
+ - ``[0, 1]``
+ - higher
+ * - `MixedDatasets `_
+ - ``[0, 1]``
+ - higher
+ * - `TopKCardinality `_
+ - ``[0, 1]``
+ - higher
+ * - `ModelRandomization `_
+ - ``[-1, 1]``
+ - closer to 0
+ * - `LinearDatamodelingScore `_
+ - ``[-1, 1]``
+ - higher
+ * - `MRR `_
+ - ``[0, 1]``
+ - higher
+ * - `RecallAtK `_
+ - ``[0, 1]``
+ - higher
+ * - `TailPatch `_
+ - ``[-1, 1]``
+ - higher
+
Benchmarks
----------
|quanda| comes with a number of pre-computed benchmarks that can be conveniently used for evaluation in a plug-and-play manner. We are planning to significantly expand the number of benchmarks in the future. Currently available benchmarks span vision (MNIST / LeNet, CIFAR-10 / ResNet-9, AWA2 / ResNet-50), text classification (QNLI / BERT), and causal language modeling (T-REx / GPT-2 fine-tuned on OpenWebText).
@@ -182,48 +205,72 @@ Benchmarks
- Benchmarks (Dataset / Model)
* - `TopKCardinalityMetric `_
- Heuristic
- - Vision / Text
- - mnist_top_k_cardinality, cifar_top_k_cardinality, awa2_top_k_cardinality, qnli_top_k_cardinality
+ - Vision
+ - mnist_top_k_cardinality (MNIST / LeNet), cifar_top_k_cardinality (CIFAR-10 / ResNet-9), awa2_top_k_cardinality (AWA2 / ResNet-50)
+ * -
+ -
+ - Text
+ - qnli_top_k_cardinality (QNLI / BERT)
* - `ModelRandomizationMetric `_
- Heuristic
- - Vision / Text
- - mnist_model_randomization, cifar_model_randomization, awa2_model_randomization, qnli_model_randomization
+ - Vision
+ - mnist_model_randomization (MNIST / LeNet), cifar_model_randomization (CIFAR-10 / ResNet-9), awa2_model_randomization (AWA2 / ResNet-50)
+ * -
+ -
+ - Text
+ - qnli_model_randomization (QNLI / BERT)
* - `MixedDatasetsMetric `_
- Heuristic
- - Vision / Text
- - mnist_mixed_datasets, cifar_mixed_datasets, awa2_mixed_datasets, qnli_mixed_datasets
+ - Vision
+ - mnist_mixed_datasets (MNIST / LeNet), cifar_mixed_datasets (CIFAR-10 / ResNet-9), awa2_mixed_datasets (AWA2 / ResNet-50)
+ * -
+ -
+ - Text
+ - qnli_mixed_datasets (QNLI / BERT)
* - `ClassDetectionMetric `_
- Downstream Task Evaluator
- - Vision / Text
- - mnist_class_detection, cifar_class_detection, awa2_class_detection, qnli_class_detection
+ - Vision
+ - mnist_class_detection (MNIST / LeNet), cifar_class_detection (CIFAR-10 / ResNet-9), awa2_class_detection (AWA2 / ResNet-50)
+ * -
+ -
+ - Text
+ - qnli_class_detection (QNLI / BERT)
* - `SubclassDetectionMetric `_
- Downstream Task Evaluator
- Vision
- - mnist_subclass_detection, cifar_subclass_detection, awa2_subclass_detection
+ - mnist_subclass_detection (MNIST / LeNet), cifar_subclass_detection (CIFAR-10 / ResNet-9), awa2_subclass_detection (AWA2 / ResNet-50)
* - `MislabelingDetectionMetric `_
- Downstream Task Evaluator
- - Vision / Text
- - mnist_mislabeling_detection, cifar_mislabeling_detection, awa2_mislabeling_detection, qnli_mislabeling_detection
+ - Vision
+ - mnist_mislabeling_detection (MNIST / LeNet), cifar_mislabeling_detection (CIFAR-10 / ResNet-9), awa2_mislabeling_detection (AWA2 / ResNet-50)
+ * -
+ -
+ - Text
+ - qnli_mislabeling_detection (QNLI / BERT)
* - `ShortcutDetectionMetric `_
- Downstream Task Evaluator
- Vision
- - mnist_shortcut_detection, cifar_shortcut_detection, awa2_shortcut_detection
+ - mnist_shortcut_detection (MNIST / LeNet), cifar_shortcut_detection (CIFAR-10 / ResNet-9), awa2_shortcut_detection (AWA2 / ResNet-50)
* - `MRRMetric `_
- Downstream Task Evaluator
- Causal LM
- - gpt2_trex_openwebtext_ft_mrr
+ - gpt2_trex_openwebtext_ft_mrr (T-REx / GPT-2 fine-tuned on OpenWebText)
* - `RecallAtKMetric `_
- Downstream Task Evaluator
- Causal LM
- - gpt2_trex_openwebtext_ft_recall_at_k
+ - gpt2_trex_openwebtext_ft_recall_at_k (T-REx / GPT-2 fine-tuned on OpenWebText)
* - `TailPatchMetric `_
- Downstream Task Evaluator
- Causal LM
- - gpt2_trex_openwebtext_ft_tail_patch
+ - gpt2_trex_openwebtext_ft_tail_patch (T-REx / GPT-2 fine-tuned on OpenWebText)
* - `LinearDatamodelingMetric `_
- Ground Truth
- - Vision / Text
- - mnist_linear_datamodeling, cifar_linear_datamodeling, awa2_linear_datamodeling, qnli_linear_datamodeling
+ - Vision
+ - mnist_linear_datamodeling (MNIST / LeNet), cifar_linear_datamodeling (CIFAR-10 / ResNet-9), awa2_linear_datamodeling (AWA2 / ResNet-50)
+ * -
+ -
+ - Text
+ - qnli_linear_datamodeling (QNLI / BERT)
.. toctree::
From 47f890c5e1787f59fc490e6e832d5053f533d604 Mon Sep 17 00:00:00 2001
From: Dilyara Bareeva
Date: Mon, 4 May 2026 15:32:09 +0200
Subject: [PATCH 08/39] fix: problematic docstrings
---
.../downstream_eval/subclass_detection.py | 3 ---
.../explainers/wrappers/captum_influence.py | 2 +-
.../explainers/wrappers/dattri_influence.py | 20 ++++++++++---------
quanda/explainers/wrappers/kronfluence.py | 10 +++++-----
quanda/utils/datasets/dataset_handlers.py | 2 +-
5 files changed, 18 insertions(+), 19 deletions(-)
diff --git a/quanda/benchmarks/downstream_eval/subclass_detection.py b/quanda/benchmarks/downstream_eval/subclass_detection.py
index 4027d1ed..c2dbaf9e 100644
--- a/quanda/benchmarks/downstream_eval/subclass_detection.py
+++ b/quanda/benchmarks/downstream_eval/subclass_detection.py
@@ -132,9 +132,6 @@ def evaluate(
Keyword arguments for the explainer, by default None.
batch_size: int, optional
Batch size for the evaluation, by default 8.
- max_eval_n: Optional[int], optional
- Maximum number of evaluation samples to use. If None, uses the
- entire evaluation dataset. By default 1000.
max_eval_n: Optional[int], optional
Maximum number of evaluation samples to use. If None, uses the
entire evaluation dataset. By default 1000.
diff --git a/quanda/explainers/wrappers/captum_influence.py b/quanda/explainers/wrappers/captum_influence.py
index b6b9b551..dbeba1f4 100644
--- a/quanda/explainers/wrappers/captum_influence.py
+++ b/quanda/explainers/wrappers/captum_influence.py
@@ -1003,7 +1003,7 @@ class CaptumTracInCPFast(CaptumInfluence):
----------
(1) Pruthi, Garima, et al. (2020). "Estimating training data influence by
tracing gradient descent."
- Advances in Neural Information Processing Systems 33. (19920-19930).
+ Advances in Neural Information Processing Systems 33. (19920-19930).
(2) https://github.com/pytorch/captum/blob/master/captum/influence/_core/
tracincp_fast_rand_proj.py
diff --git a/quanda/explainers/wrappers/dattri_influence.py b/quanda/explainers/wrappers/dattri_influence.py
index 803decfe..1cac3a37 100644
--- a/quanda/explainers/wrappers/dattri_influence.py
+++ b/quanda/explainers/wrappers/dattri_influence.py
@@ -91,16 +91,18 @@ def __init__(
train_dataset : torch.utils.data.Dataset
Training dataset to be used for the influence computation.
loss_func : Callable
- Builder for dattri's `AttributionTask` loss, with signature:
- ```
- def loss_func(
- model: torch.nn.Module,
- ) -> Callable[[Dict[str, torch.Tensor], Tuple], torch.Tensor]:
- ...
- ```
- The returned callable takes `(params, batch)` and returns a
+ Builder for dattri's ``AttributionTask`` loss, with signature::
+
+ def loss_func(
+ model: torch.nn.Module,
+ ) -> Callable[
+ [Dict[str, torch.Tensor], Tuple], torch.Tensor
+ ]:
+ ...
+
+ The returned callable takes ``(params, batch)`` and returns a
per-sample loss tensor (compatible with
- `torch.func.functional_call`).
+ ``torch.func.functional_call``).
attributor_cls : type
The dattri attributor class.
attributor_kwargs : Dict[str, Any]
diff --git a/quanda/explainers/wrappers/kronfluence.py b/quanda/explainers/wrappers/kronfluence.py
index 761c6eb6..0803d7cc 100644
--- a/quanda/explainers/wrappers/kronfluence.py
+++ b/quanda/explainers/wrappers/kronfluence.py
@@ -49,11 +49,11 @@ class Kronfluence(Explainer):
----------
(1) Roger Grosse, Juhan Bae, Cem Anil, Nelson Elhage, Alex Tamkin,
Amirhossein Tajdini, Benoit Steiner,
- Dustin Li, Esin Durmus, Ethan Perez, Evan Hubinger, Kamilė Lukošiūtė,
- Karina Nguyen, Nicholas Joseph,
- Sam McCandlish, Jared Kaplan, Samuel R. Bowman. (2023).
- "Studying large language model generalization with influence
- functions". arXiv preprint arXiv:2308.03296.
+ Dustin Li, Esin Durmus, Ethan Perez, Evan Hubinger, Kamilė Lukošiūtė,
+ Karina Nguyen, Nicholas Joseph,
+ Sam McCandlish, Jared Kaplan, Samuel R. Bowman. (2023).
+ "Studying large language model generalization with influence
+ functions". arXiv preprint arXiv:2308.03296.
(2) https://github.com/pomonam/kronfluence
diff --git a/quanda/utils/datasets/dataset_handlers.py b/quanda/utils/datasets/dataset_handlers.py
index e753e510..a26b31cf 100644
--- a/quanda/utils/datasets/dataset_handlers.py
+++ b/quanda/utils/datasets/dataset_handlers.py
@@ -421,7 +421,7 @@ def __init__(
self.label_key = label_key
def collate(self, samples: List[Dict[str, Any]]) -> List[torch.Tensor]:
- """Stack HF dict samples into a list [*input_keys, label_key].
+ """Stack HF dict samples into a list ``[*input_keys, label_key]``.
Projects each sample onto the required keys *before* collation so
that non-numeric columns (e.g. raw ``"sentence"``/``"hypothesis"``
From 662344b90d8c0d47baf38e0295429be4e21f6478 Mon Sep 17 00:00:00 2001
From: Dilyara Bareeva
Date: Mon, 4 May 2026 15:34:59 +0200
Subject: [PATCH 09/39] fix: docs compile errors
---
docs/source/conf.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 607df294..a06d3a38 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -30,6 +30,9 @@
]
source_suffix = [".rst", ".md"]
autosummary_generate = True
+numpydoc_class_members_toctree = False
+numpydoc_show_class_members = False
+suppress_warnings = ["ref.python"]
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
rst_epilog = """
.. |quanda| raw:: html
From c379bb431dab51a78498be5bcdf77c40cf8bc390 Mon Sep 17 00:00:00 2001
From: Dilyara Bareeva
Date: Mon, 4 May 2026 16:23:16 +0200
Subject: [PATCH 10/39] fix: docs compile errors pt2
---
docs/Makefile | 1 +
1 file changed, 1 insertion(+)
diff --git a/docs/Makefile b/docs/Makefile
index 9ef23a98..901d37c4 100644
--- a/docs/Makefile
+++ b/docs/Makefile
@@ -22,6 +22,7 @@ help:
rst:
@sphinx-apidoc -o source/docs_api ../quanda --module-first --separate --force
+ @sed -i '/^\.\. automodule:: quanda$$/,/^$$/ { /:members:/d; /:undoc-members:/d; }' source/docs_api/quanda.rst
clean:
From 480da4ccb663ad51c6f8301b24c438eb2f088da2 Mon Sep 17 00:00:00 2001
From: Dilyara Bareeva
Date: Mon, 4 May 2026 16:27:11 +0200
Subject: [PATCH 11/39] chore: anonymous links to readme assets
---
README.md | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/README.md b/README.md
index 07cb94e0..35fb0ed4 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,8 @@
-
-
-
+
+
+
From e0bb072fcd15dfc3f1be8e31ffee78604b41b074 Mon Sep 17 00:00:00 2001
From: Dilyara Bareeva
Date: Mon, 4 May 2026 16:37:11 +0200
Subject: [PATCH 12/39] chore: remove pip references
---
README.md | 6 +++---
docs/source/quickstart.rst | 4 ++--
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/README.md b/README.md
index 35fb0ed4..071eeeaa 100644
--- a/README.md
+++ b/README.md
@@ -217,10 +217,10 @@ Although there are various demonstrations of TDA’s potential for interpretabil
### Installation
-To install the latest release of **quanda** use:
+To install **quanda** from a local clone of this repository, run:
```setup
-pip install quanda
+pip install -e .
```
**quanda** requires Python 3.10 or 3.11. It is recommended to use a virtual environment to install the package.
@@ -537,7 +537,7 @@ We have included a few [tutorials](tutorials) to demonstrate the usage of **quan
To install the library with tutorial dependencies, run:
```bash
-pip install quanda[tutorials]
+pip install -e '.[tutorials]'
```
## 👩💻Contributing
diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst
index 44b44326..4b53899e 100644
--- a/docs/source/quickstart.rst
+++ b/docs/source/quickstart.rst
@@ -4,11 +4,11 @@ Quickstart
Installation
------------
-To install the latest release of |quanda|, use the following command in your terminal:
+To install |quanda| from a local clone of the repository, use the following command in your terminal:
.. code-block:: console
- (.venv) $ pip install quanda
+ (.venv) $ pip install -e .
|quanda| requires Python 3.10 or 3.11. It is recommended to use a virtual environment to install the package.
From 8747f9cfd0f8f2dc025aaf39b2877abb65082ca0 Mon Sep 17 00:00:00 2001
From: Dilyara Bareeva
Date: Mon, 4 May 2026 21:04:34 +0200
Subject: [PATCH 13/39] chore: remove paper link
---
README.md | 2 --
1 file changed, 2 deletions(-)
diff --git a/README.md b/README.md
index 071eeeaa..8f7f54de 100644
--- a/README.md
+++ b/README.md
@@ -20,8 +20,6 @@
**quanda** _is currently under active development. Note the release version to ensure reproducibility of your work. Expect changes to API._
-[📑 Shortcut to paper!](https://arxiv.org/pdf/2410.07158)
-
## 🐼 Library overview
**Training data attribution** (TDA) methods attribute model output on a specific test sample to the training dataset that it was trained on. They reveal the training datapoints responsible for the model's decisions. Existing methods achieve this by estimating the counterfactual effect of removing datapoints from the training set ([Koh and Liang, 2017](https://proceedings.mlr.press/v70/koh17a.html); [Park et al., 2023](https://proceedings.mlr.press/v202/park23c.html); [Bae et al., 2024](https://arxiv.org/abs/2405.12186)) tracking the contributions of training points to the loss reduction throughout training ([Pruthi et al., 2020](https://proceedings.neurips.cc/paper/2020/hash/e6385d39ec9394f2f3a354d9d2b88eec-Abstract.html)), using interpretable surrogate models ([Yeh et al., 2018](https://proceedings.neurips.cc/paper/2018/hash/8a7129b8f3edd95b7d969dfc2c8e9d9d-Abstract.html)) or finding training samples that are deemed similar to the test sample by the model ([Caruana et. al, 1999](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2232607/); [Hanawa et. al, 2021](https://openreview.net/forum?id=9uvhpyQwzM_)). In addition to model understanding, TDA has been used in a variety of applications such as debugging model behavior ([Koh and Liang, 2017](https://proceedings.mlr.press/v70/koh17a.html); [Yeh et al., 2018](https://proceedings.neurips.cc/paper/2018/hash/8a7129b8f3edd95b7d969dfc2c8e9d9d-Abstract.html); [K and Søgaard, 2021](https://arxiv.org/abs/2111.04683); [Guo et al., 2021](https://aclanthology.org/2021.emnlp-main.808)), data summarization ([Khanna et al., 2019](https://proceedings.mlr.press/v89/khanna19a.html); [Marion et al., 2023](https://openreview.net/forum?id=XUIYn3jo5T); [Yang et al., 2023](https://openreview.net/forum?id=4wZiAXD29TQ)), dataset selection ([Engstrom et al., 2024](https://openreview.net/forum?id=GC8HkKeH8s); [Chhabra et al., 2024](https://openreview.net/forum?id=HE9eUQlAvo)), fact tracing ([Akyurek et al., 2022](https://aclanthology.org/2022.findings-emnlp.180)) and machine unlearning ([Warnecke
et al., 2023](https://arxiv.org/abs/2108.11577)).
From f7dd1d3ff6bbd7a7e0ed6d9324f27b5cb90915c8 Mon Sep 17 00:00:00 2001
From: Dilyara Bareeva
Date: Mon, 4 May 2026 21:08:10 +0200
Subject: [PATCH 14/39] chore: fixing anynomyzation gone wrong
---
CODE_OF_CONDUCT.md | 2 +-
CONTRIBUTING.md | 2 +-
README.md | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
index 9f0d809e..51705c3a 100644
--- a/CODE_OF_CONDUCT.md
+++ b/CODE_OF_CONDUCT.md
@@ -60,7 +60,7 @@ representative at an online or offline event.
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported to the community leaders responsible for enforcement at
-.
+AUTHOR_1_E_MAIL_ANONYMIZED.
All complaints will be reviewed and investigated promptly and fairly.
All community leaders are obligated to respect the privacy and security of the
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index d1f31e99..38ac349c 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -5,7 +5,7 @@
In this guide, you will get a summary of the main components of **quanda**, as well as best practices for your own contributions.
-If you have any questions regarding the codebase, please open an issue or write us at [](mailto:) or [](mailto:).
+If you have any questions regarding the codebase, please open an issue or write us.
## Table of Contents
diff --git a/README.md b/README.md
index 8f7f54de..e6d383e1 100644
--- a/README.md
+++ b/README.md
@@ -546,4 +546,4 @@ We welcome contributions to **quanda**! You could contribute by:
A detailed guide on how to contribute to **quanda** can be found [here](CONTRIBUTING.md).
## ✉️ Contact
-If you have any questions regarding the codebase, please open an issue or contact us via email at [](mailto:) or [](mailto:).
+If you have any questions regarding the codebase, please open an issue or contact us via email.
From 2cb2724c0b11f4c4613be983ff8a5301313c67f5 Mon Sep 17 00:00:00 2001
From: Dilyara Bareeva
Date: Tue, 5 May 2026 14:59:15 +0200
Subject: [PATCH 15/39] chore: add randomit initialization to representer
points
---
.../explainers/wrappers/representer_points.py | 39 +++++++++++++++++--
scripts/awa2_resnet50_bench/eval_defs.sh | 4 +-
scripts/bert_qnli_bench/eval_defs.sh | 4 +-
3 files changed, 40 insertions(+), 7 deletions(-)
diff --git a/quanda/explainers/wrappers/representer_points.py b/quanda/explainers/wrappers/representer_points.py
index d856668b..6270c7d1 100644
--- a/quanda/explainers/wrappers/representer_points.py
+++ b/quanda/explainers/wrappers/representer_points.py
@@ -178,6 +178,8 @@ def __init__(
checkpoints: Optional[Union[str, List[str]]] = None,
checkpoints_load_func: Optional[CheckpointLoadFunc] = None,
cache_dir: str = "./cache",
+ activations_cache_dir: Optional[str] = None,
+ activations_id: Optional[str] = None,
features_postprocess: Optional[Callable] = None,
lmbd: float = 0.003,
epoch: int = 3000,
@@ -188,6 +190,7 @@ def __init__(
batch_size: int = 32,
load_from_disk: bool = True,
show_progress: bool = True,
+ random_init: bool = False,
):
"""Initialize the RepresenterPoints class.
@@ -214,7 +217,21 @@ def __init__(
Ignored, for the same reason as ``checkpoints``.
Defaults to None.
cache_dir : str, optional
- The directory to save the cache, defaults to "./cache".
+ Directory for the trained representer coefficients
+ (``*_repr_weights.pt``). Depends on training hyperparameters,
+ so it is safe — and expected — to vary this per run.
+ Defaults to "./cache".
+ activations_cache_dir : Optional[str], optional
+ Directory for cached penultimate-layer activations of the
+ training set. Activations only depend on
+ ``(model checkpoint, train_dataset, features_layer)``, so this
+ can be pointed at a shared location to reuse them across runs
+ with different hyperparameters. Defaults to ``cache_dir``.
+ activations_id : Optional[str], optional
+ Identifier under which activations are stored inside
+ ``activations_cache_dir``. Should encode the model + dataset +
+ ``features_layer`` but NOT the training hyperparameters.
+ Defaults to ``model_id``.
features_postprocess : Optional[Callable], optional
A postprocessing function for the features, defaults to None.
lmbd : float, optional
@@ -236,6 +253,10 @@ def __init__(
Whether to load the activations from disk, defaults to True.
show_progress : bool, optional
Whether to show the training progress, defaults to True.
+ random_init : bool, optional
+ If True, the initial representer W is initialized randomly
+ instead of from the trained classifier weights. Defaults to
+ False.
"""
logger.info("Initializing Representer Point Selection explainer...")
@@ -249,6 +270,8 @@ def __init__(
self.model_id = model_id
self.cache_dir = cache_dir
+ self.activations_cache_dir = activations_cache_dir or cache_dir
+ self.activations_id = activations_id or model_id
self.normalize = normalize
self.features_layer = features_layer
self.classifier_layer = classifier_layer
@@ -259,16 +282,19 @@ def __init__(
self.epsilon = epsilon
self.features_postprocess = features_postprocess
self.show_progress = show_progress
+ self.random_init = random_init
self.dataloader = torch.utils.data.DataLoader(
self.train_dataset, batch_size=batch_size, shuffle=False
)
+ os.makedirs(self.cache_dir, exist_ok=True)
+ os.makedirs(self.activations_cache_dir, exist_ok=True)
with default_tensor_type(self.device):
act_dataset = AV.generate_dataset_activations(
- path=cache_dir,
+ path=self.activations_cache_dir,
model=model,
- model_id=model_id,
+ model_id=self.activations_id,
layers=[features_layer],
dataloader=self.dataloader,
load_from_disk=load_from_disk,
@@ -457,6 +483,8 @@ def train(self):
w_and_b = torch.concatenate(
[weight_linear.T, bias_linear.unsqueeze(0)]
)
+ if self.random_init:
+ w_and_b = torch.randn_like(w_and_b)
model = RepresenterSoftmax(w_and_b, self.device)
x = nn.Parameter(samples_with_bias.to(self.device))
@@ -512,6 +540,11 @@ def train(self):
pbar.update(1)
+ if grad_loss == init_grad:
+ raise ValueError(
+ "Gradient did not decrease during training. Consider increasing "
+ "the number of epochs or the learning rate."
+ )
# calculate w based on the representer theorem's decomposition
temp = torch.matmul(
x, nn.Parameter(best_W.to(self.device), requires_grad=True)
diff --git a/scripts/awa2_resnet50_bench/eval_defs.sh b/scripts/awa2_resnet50_bench/eval_defs.sh
index 2518bd42..a4ac2b21 100755
--- a/scripts/awa2_resnet50_bench/eval_defs.sh
+++ b/scripts/awa2_resnet50_bench/eval_defs.sh
@@ -4,9 +4,9 @@
declare -A EXPL_SWEEP
EXPL_SWEEP[similarity]="explainer.kwargs.layers=flatten explainer.kwargs.batch_size=128 device=cuda:0 hydra.launcher.n_jobs=1 batch_size=128"
-EXPL_SWEEP[representer_points]="explainer.kwargs.features_layer=flatten explainer.kwargs.classifier_layer=fc explainer.kwargs.batch_size=128 device=cuda:0 explainer.kwargs.normalize=true,false hydra.launcher.n_jobs=1 batch_size=128"
+EXPL_SWEEP[representer_points]="explainer.kwargs.features_layer=flatten explainer.kwargs.classifier_layer=fc explainer.kwargs.batch_size=128 device=cuda:1 explainer.kwargs.normalize=true,false hydra.launcher.n_jobs=1 batch_size=128 +explainer.kwargs.random_init=true"
EXPL_SWEEP[tracincpfast]="explainer.kwargs.batch_size=256 batch_size=256 device=cuda:0"
EXPL_SWEEP[arnoldi]="explainer.kwargs.layers=[fc] explainer.kwargs.projection_dim=50 explainer.kwargs.arnoldi_dim=100 explainer.kwargs.batch_size=256 +explainer.kwargs.precompute_data_ratio=0.1 device=cuda:1 hydra.launcher.n_jobs=1"
-EXPL_SWEEP[trak]="explainer.kwargs.proj_dim=1024,2048 explainer.kwargs.batch_size=32 device=cuda:0 hydra.launcher.n_jobs=1"
+EXPL_SWEEP[trak]="explainer.kwargs.proj_dim=1024,2048,4096 explainer.kwargs.batch_size=32 device=cuda:0 hydra.launcher.n_jobs=1"
EXPL_SWEEP[random]="device=cuda:0 explainer.kwargs.seed=0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40 hydra.launcher.n_jobs=5 batch_size=128 device=cuda:1"
EXPL_SWEEP[kronfluence]="explainer.kwargs.task_module._target_=quanda.explainers.wrappers.kronfluence_tasks.ImageClassificationTask explainer.kwargs.task_module.tracked_modules=[layer4.2.conv3,fc] +explainer.kwargs.score_args._target_=kronfluence.arguments.ScoreArguments +explainer.kwargs.score_args.use_measurement_for_self_influence=true explainer.kwargs.batch_size=64 device=cuda:0 batch_size=1000 inference_batch_size=64 hydra.launcher.n_jobs=1"
diff --git a/scripts/bert_qnli_bench/eval_defs.sh b/scripts/bert_qnli_bench/eval_defs.sh
index c4936294..09699d4b 100755
--- a/scripts/bert_qnli_bench/eval_defs.sh
+++ b/scripts/bert_qnli_bench/eval_defs.sh
@@ -3,8 +3,8 @@
declare -A EXPL_SWEEP
EXPL_SWEEP[similarity]="explainer.kwargs.layers=dropout explainer.kwargs.similarity_metric.path=quanda.utils.functions.cosine_similarity,quanda.utils.functions.dot_product_similarity +explainer.kwargs.task=text_classification device=cuda:0 batch_size=32"
-EXPL_SWEEP[trak]="explainer.kwargs.proj_dim=2048 explainer.kwargs.lambda_reg=1e-5 +explainer.kwargs.task=text_classification device=cuda:0 explainer.kwargs.batch_size=8"
+EXPL_SWEEP[trak]="explainer.kwargs.proj_dim=2048 +explainer.kwargs.task=text_classification device=cuda:0 explainer.kwargs.batch_size=8 +explainer.kwargs.random_init=true,false"
EXPL_SWEEP[dattri_tracin]="+explainer.kwargs.task=text_classification +explainer.kwargs.layer_name=[classifier.weight,classifier.bias] device=cuda:0 explainer.kwargs.batch_size=8 batch_size=1000"
EXPL_SWEEP[random]="device=cuda:1 explainer.kwargs.seed=0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40 hydra.launcher.n_jobs=5"
EXPL_SWEEP[kronfluence]="explainer.kwargs.task_module.tracked_modules=[bert.pooler.dense,classifier] +explainer.kwargs.score_args._target_=kronfluence.arguments.ScoreArguments +explainer.kwargs.score_args.use_measurement_for_self_influence=true +explainer.kwargs.task=text_classification device=cuda:0 batch_size=1000 inference_batch_size=32"
-EXPL_SWEEP[representer_points]="explainer.kwargs.features_layer=dropout explainer.kwargs.classifier_layer=classifier explainer.kwargs.normalize=true +explainer.kwargs.task=text_classification explainer.kwargs.batch_size=32 device=cuda:0"
\ No newline at end of file
+EXPL_SWEEP[representer_points]="explainer.kwargs.features_layer=dropout explainer.kwargs.classifier_layer=classifier explainer.kwargs.normalize=true,false +explainer.kwargs.task=text_classification explainer.kwargs.batch_size=32 device=cuda:1 +explainer.kwargs.random_init=true,false"
\ No newline at end of file
From 9ca9f63cf0f33df67b5867b163c641d2f43bc7c0 Mon Sep 17 00:00:00 2001
From: Dilyara Bareeva
Date: Tue, 5 May 2026 15:20:25 +0200
Subject: [PATCH 16/39] chore: recover awa eval scripts
---
scripts/awa2_resnet50_bench/eval_awa2_pt1.sh | 8 ++++++++
scripts/awa2_resnet50_bench/eval_awa2_pt2.sh | 19 +++++++------------
2 files changed, 15 insertions(+), 12 deletions(-)
diff --git a/scripts/awa2_resnet50_bench/eval_awa2_pt1.sh b/scripts/awa2_resnet50_bench/eval_awa2_pt1.sh
index dd35cae1..8c928016 100755
--- a/scripts/awa2_resnet50_bench/eval_awa2_pt1.sh
+++ b/scripts/awa2_resnet50_bench/eval_awa2_pt1.sh
@@ -5,11 +5,19 @@ source "$(dirname "$0")/eval_defs.sh"
EVAL_CONFIG_NAME="awa2_resnet50"
benchmarks=(
+ awa2_class_detection
+ awa2_subclass_detection
+ awa2_shortcut_detection
awa2_mixed_datasets
)
methods=(
+ similarity
+ representer_points
+ tracincpfast
arnoldi
+ trak
+ random
)
PARALLEL=false
diff --git a/scripts/awa2_resnet50_bench/eval_awa2_pt2.sh b/scripts/awa2_resnet50_bench/eval_awa2_pt2.sh
index 10228d8c..416e27fa 100755
--- a/scripts/awa2_resnet50_bench/eval_awa2_pt2.sh
+++ b/scripts/awa2_resnet50_bench/eval_awa2_pt2.sh
@@ -7,11 +7,18 @@ EVAL_CONFIG_NAME="awa2_resnet50"
benchmarks=(
awa2_mixed_datasets
+ awa2_top_k_cardinality
+ awa2_model_randomization
+ awa2_mislabeling_detection
+ awa2_linear_datamodeling
)
methods=(
similarity
representer_points
+ tracincpfast
+ arnoldi
+ trak
random
)
PARALLEL=false
@@ -19,15 +26,3 @@ PARALLEL=false
source "$(dirname "$0")/../eval.sh" "$@"
-methods=(
- tracincpfast
-)
-
-source "$(dirname "$0")/../eval.sh" "$@"
-
-
-methods=(
- trak
-)
-
-source "$(dirname "$0")/../eval.sh" "$@"
\ No newline at end of file
From 1520e229d3513091a09a933e8812a039d2eb1221 Mon Sep 17 00:00:00 2001
From: Dilyara Bareeva
Date: Tue, 5 May 2026 17:07:40 +0200
Subject: [PATCH 17/39] refactor: remove load_meta_from_disk argument, accept
different types of config in explain and trrain for compatibility
---
quanda/benchmarks/base.py | 210 ++++++++++++++----
quanda/benchmarks/config_parser.py | 129 ++++++-----
.../downstream_eval/_fact_tracing.py | 49 ++--
.../downstream_eval/class_detection.py | 2 +-
.../downstream_eval/mislabeling_detection.py | 15 +-
.../downstream_eval/shortcut_detection.py | 2 +-
.../downstream_eval/subclass_detection.py | 2 +-
.../ground_truth/linear_datamodeling.py | 90 +++++---
.../benchmarks/heuristics/mixed_datasets.py | 31 +--
.../heuristics/model_randomization.py | 2 +-
.../heuristics/top_k_cardinality.py | 2 +-
quanda/explainers/base.py | 9 +-
.../explainers/wrappers/representer_points.py | 32 +--
quanda/metrics/base.py | 9 +-
quanda/utils/common.py | 33 +++
quanda/utils/datasets/transformed/metadata.py | 12 +-
scripts/train.py | 6 +-
scripts/train_and_push_to_hub.py | 1 +
tests/assets/mnist_local_bench/0_rand_0.pth | Bin 0 -> 181382 bytes
.../test_mislabeling_detection.py | 4 +-
.../test_shortcut_detection.py | 6 +-
.../ground_truth/test_linear_datamodeling.py | 23 --
tests/benchmarks/test_benchmarks.py | 84 ++-----
tests/benchmarks/test_config_parser.py | 30 ---
tests/explainers/test_cache_explainer.py | 4 +-
.../datasets/transformed/test_metadata.py | 4 +-
tests/utils/test_common.py | 50 +++++
27 files changed, 500 insertions(+), 341 deletions(-)
create mode 100644 tests/assets/mnist_local_bench/0_rand_0.pth
diff --git a/quanda/benchmarks/base.py b/quanda/benchmarks/base.py
index 2f1790cd..169ef4df 100644
--- a/quanda/benchmarks/base.py
+++ b/quanda/benchmarks/base.py
@@ -33,6 +33,7 @@
from quanda.utils.common import (
CheckpointLoadFunc,
DatasetSplit,
+ _resolve_config,
_stable_repr,
_subsample_dataset,
chunked_logits,
@@ -190,9 +191,13 @@ def load_pretrained(
(metadata, model) must already be present under
``cache_dir``. By default False.
load_fresh : bool, optional
- If True, re-download metadata and model from the Hub,
+ If True, re-download the metadata snapshot from the Hub
+ and regenerate any cached split/wrapper metadata,
overwriting the local cache. Incompatible with
- ``offline=True``. By default False.
+ ``offline=True``. By default False. Note: model
+ checkpoints are reused from the local cache when present
+ regardless of this flag — delete the ckpt directory to
+ force a re-download.
Returns
-------
@@ -227,7 +232,6 @@ def load_pretrained(
)
obj = cls.from_config(
cfg,
- load_meta_from_disk=True,
offline=offline,
load_fresh=load_fresh,
device=device,
@@ -238,18 +242,37 @@ def load_pretrained(
@classmethod
def from_config(
cls,
- config: dict,
- load_meta_from_disk: bool = True,
+ config: Union[dict, str],
offline: bool = False,
device: str = "cpu",
metadata_suffix: str = "",
load_fresh: bool = False,
) -> "Benchmark":
- """Initialize the benchmark from a dictionary."""
- if offline and load_fresh:
- raise ValueError(
- "offline=True and load_fresh=True are incompatible."
- )
+ """Initialize the benchmark from a config.
+
+ Parameters
+ ----------
+ config : Union[dict, str]
+ The benchmark configuration dictionary, a path to a YAML file
+ or registered ``bench_id``
+ (see :data:`quanda.benchmarks.resources.config_map.config_map`).
+ offline : bool, optional
+ If True, no HTTP request is issued to the Hub; all assets
+ (metadata, model) must already be present under
+ ``config['bench_save_dir']``. By default False.
+ device : str, optional
+ Device to load the model on, by default "cpu".
+ metadata_suffix : str, optional
+ Suffix to disambiguate metadata directories. By default "".
+ load_fresh: bool, False
+ If True, regenerate any cached split/wrapper metadata
+ (overwriting local files) and, when ``offline=False``,
+ re-download metadata/model from the Hub. When False
+ (default), cached metadata is reused if present and only
+ missing pieces are generated.
+
+ """
+ config = _resolve_config(config)
cache_dir = config.get("bench_save_dir", "./tmp")
metadata_dir = MetadataConfigParser.get_metadata_dir(
cfg=config,
@@ -260,20 +283,20 @@ def from_config(
train_dataset = DatasetConfigParser.parse_dataset_cfg(
ds_config=config.get("train_dataset"),
metadata_dir=metadata_dir,
- load_meta_from_disk=load_meta_from_disk,
+ load_fresh=load_fresh,
splits_cfg=splits_cfg,
)
val_dataset = DatasetConfigParser.parse_dataset_cfg(
ds_config=config.get("val_dataset"),
metadata_dir=metadata_dir,
- load_meta_from_disk=load_meta_from_disk,
+ load_fresh=load_fresh,
splits_cfg=splits_cfg,
)
eval_dataset = DatasetConfigParser.parse_dataset_cfg(
ds_config=config.get("eval_dataset"),
metadata_dir=metadata_dir,
- load_meta_from_disk=load_meta_from_disk,
+ load_fresh=load_fresh,
splits_cfg=splits_cfg,
)
@@ -283,7 +306,6 @@ def from_config(
bench_save_dir=config["bench_save_dir"],
ckpts=_resolve_ckpts(config),
offline=offline,
- load_fresh=load_fresh,
device=device,
)
)
@@ -293,7 +315,7 @@ def from_config(
train_dataset=train_dataset,
eval_dataset=eval_dataset,
metadata_dir=metadata_dir,
- load_meta_from_disk=load_meta_from_disk,
+ load_fresh=load_fresh,
)
return cls(
@@ -317,7 +339,7 @@ def _extra_kwargs_from_config(
train_dataset: Union[torch.utils.data.Dataset, datasets.Dataset],
eval_dataset: torch.utils.data.Dataset,
metadata_dir: str,
- load_meta_from_disk: bool,
+ load_fresh: bool,
) -> dict:
"""Extract subclass-specific kwargs from config.
@@ -333,8 +355,9 @@ def _extra_kwargs_from_config(
The parsed evaluation dataset.
metadata_dir : str
Path to the metadata directory.
- load_meta_from_disk : bool
- Whether metadata was loaded from disk.
+ load_fresh : bool
+ If True, regenerate any cached subclass-specific metadata
+ instead of reusing it.
Returns
-------
@@ -347,39 +370,48 @@ def _extra_kwargs_from_config(
@classmethod
def train(
cls,
- config: dict,
+ config: Union[dict, str],
logger: Optional[L.pytorch.loggers.logger.Logger] = None,
device: str = "cpu",
batch_size: int = 64,
- load_meta_from_disk: bool = False,
+ load_fresh: bool = True,
+ use_pid: bool = False,
) -> "Benchmark":
"""Train a model using the provided configuration.
Parameters
----------
- config : dict
- Dictionary containing the configuration.
+ config : dict | str
+ Either a configuration dict, a registered ``bench_id`` (see
+ :data:`quanda.benchmarks.resources.config_map.config_map`),
+ or a path to a benchmark YAML. The config must specify
+ ``bench_save_dir``, the directory under which the trained
+ benchmark (checkpoints and metadata) is saved.
logger : Optional[lightning.pytorch.loggers.logger.Logger], optional
Logger to be used for logging, by default None.
device : str, optional
Device to use for training, by default "cpu"
batch_size : int, optional
Batch size for training, by default 8
- load_meta_from_disk : bool, optional
- If True, reuse existing metadata (splits, class mappings,
- etc.) from the cache instead of regenerating. By default
- False — training regenerates metadata so that a fresh
- training run is reproducible from the config alone.
+ load_fresh : bool, optional
+ If True (default), regenerate splits/class mappings/etc.
+ so a fresh training run is reproducible from the config
+ alone. Set to False to reuse cached metadata.
+ use_pid : bool, optional
+ If True, suffix checkpoint and metadata directories with
+ the current process id to disambiguate concurrent runs. By
+ default False.
Returns
-------
None
"""
- pid_suffix = f"_pid{os.getpid()}"
+ config = _resolve_config(config)
+ pid_suffix = f"_pid{os.getpid()}" if use_pid else ""
obj = cls.from_config(
config,
- load_meta_from_disk=load_meta_from_disk,
+ load_fresh=load_fresh,
device=device,
metadata_suffix=pid_suffix,
)
@@ -489,18 +521,48 @@ def train(
@classmethod
def train_and_push_to_hub(
cls,
- config: dict,
+ config: Union[dict, str],
logger: Optional[L.pytorch.loggers.logger.Logger] = None,
device: str = "cpu",
batch_size: int = 64,
- load_meta_from_disk: bool = False,
+ load_fresh: bool = True,
+ use_pid: bool = False,
): # pragma: no cover
- """Train a model using the provided config and push to HF hub."""
+ """Train a model using the provided config and push to HF hub.
+
+ Parameters
+ ----------
+ config : Union[dict, str]
+ Either a configuration dict, a registered ``bench_id`` (see
+ :data:`quanda.benchmarks.resources.config_map.config_map`),
+ or a path to a benchmark YAML.
+ logger : Optional[lightning.pytorch.loggers.logger.Logger], optional
+ Logger to be used for logging, by default None.
+ device : str, optional
+ Device to use for training, by default "cpu".
+ batch_size : int, optional
+ Batch size for training, by default 64.
+ load_fresh : bool, optional
+ If True (default), regenerate splits/class mappings/etc. so a
+ fresh training run is reproducible from the config alone. Set
+ to False to reuse cached metadata.
+ use_pid : bool, optional
+ If True, suffix checkpoint and metadata directories with the
+ current process id to disambiguate concurrent runs. By default
+ False.
+
+ Returns
+ -------
+ Benchmark
+ The trained benchmark instance.
+
+ """
+ config = _resolve_config(config)
skip_main_train = bool(config.get("skip_main_train", False))
if skip_main_train:
obj = cls.from_config(
config,
- load_meta_from_disk=load_meta_from_disk,
+ load_fresh=load_fresh,
device=device,
)
obj._compute_and_save_indices(config, batch_size)
@@ -510,7 +572,8 @@ def train_and_push_to_hub(
logger=logger,
device=device,
batch_size=batch_size,
- load_meta_from_disk=load_meta_from_disk,
+ load_fresh=load_fresh,
+ use_pid=use_pid,
)
if not isinstance(obj.model, PyTorchModelHubMixin):
raise TypeError(
@@ -954,7 +1017,7 @@ def _download_explanations(
@classmethod
def explain(
cls,
- config: dict,
+ config: Union[dict, str],
explainer_cls: type,
expl_kwargs: Optional[dict] = None,
batch_size: int = 8,
@@ -967,11 +1030,43 @@ def explain(
) -> "Benchmark":
"""Compute and persist explanations for ``eval_dataset`` to disk.
- Mirrors :meth:`train` but produces per-batch explanation tensors
- plus an ``explanations_config.yaml`` describing how the cache
- was generated. Returns the benchmark instance with
- ``self._explanations_dir`` and ``self._explanations_id`` set.
+ Parameters
+ ----------
+ config : Union[dict, str]
+ Benchmark config dict, registered ``bench_id``, or path to a
+ benchmark YAML.
+ explainer_cls : type
+ Explainer subclass to instantiate.
+ expl_kwargs : Optional[dict], optional
+ Extra kwargs forwarded to ``explainer_cls``, by default None.
+ batch_size : int, optional
+ Batch size used when iterating the eval dataset, by default 8.
+ explanations_id : Optional[str], optional
+ HF-style id for the cached explanations. If None, derived from
+ ``config`` via :func:`default_explanations_id`. By default None.
+ cache_dir : Optional[str], optional
+ Directory to write explanations into. If None, derived from
+ ``config['bench_save_dir']`` and ``explanations_id``.
+ device : str, optional
+ Device to load the model on, by default "cpu".
+ max_eval_n : Optional[int], optional
+ Cap on the number of eval samples; ``None`` means all. By
+ default 1000.
+ eval_seed : int, optional
+ Seed used when sampling the eval subset, by default 42.
+ inference_batch_size : Optional[int], optional
+ If set, every model forward run during prediction is split
+ into sub-batches of this size. ``None`` keeps the full
+ ``batch_size`` forward.
+
+ Returns
+ -------
+ Benchmark
+ The benchmark instance with ``_explanations_dir`` and
+ ``_explanations_id`` populated.
+
"""
+ config = _resolve_config(config)
obj = cls.from_config(config, device=device)
if explanations_id is None:
explanations_id = default_explanations_id(
@@ -1045,7 +1140,7 @@ def explain(
@classmethod
def explain_and_push_to_hub(
cls,
- config: dict,
+ config: Union[dict, str],
explainer_cls: type,
expl_kwargs: Optional[dict] = None,
batch_size: int = 8,
@@ -1055,7 +1150,40 @@ def explain_and_push_to_hub(
max_eval_n: Optional[int] = 1000,
eval_seed: int = 42,
): # pragma: no cover
- """Compute explanations then upload them as a HF dataset repo."""
+ """Compute explanations then upload them as a HF dataset repo.
+
+ Parameters
+ ----------
+ config : Union[dict, str]
+ Benchmark config dict, registered ``bench_id``, or path to a
+ benchmark YAML.
+ explainer_cls : type
+ Explainer subclass to instantiate.
+ expl_kwargs : Optional[dict], optional
+ Extra kwargs forwarded to ``explainer_cls``, by default None.
+ batch_size : int, optional
+ Batch size used when iterating the eval dataset, by default 8.
+ explanations_id : Optional[str], optional
+ HF repo id under which to upload the explanations. If None,
+ derived from ``config`` via :func:`default_explanations_id`.
+ cache_dir : Optional[str], optional
+ Directory to write explanations into before upload, by
+ default None.
+ device : str, optional
+ Device to load the model on, by default "cpu".
+ max_eval_n : Optional[int], optional
+ Cap on the number of eval samples; ``None`` means all. By
+ default 1000.
+ eval_seed : int, optional
+ Seed used when sampling the eval subset, by default 42.
+
+ Returns
+ -------
+ Benchmark
+ The benchmark instance after upload.
+
+ """
+ config = _resolve_config(config)
obj = cls.explain(
config=config,
explainer_cls=explainer_cls,
diff --git a/quanda/benchmarks/config_parser.py b/quanda/benchmarks/config_parser.py
index 9ea32dc0..78bb4c4b 100644
--- a/quanda/benchmarks/config_parser.py
+++ b/quanda/benchmarks/config_parser.py
@@ -91,7 +91,7 @@ def parse_dataset_cfg(
cls,
ds_config: Optional[dict],
metadata_dir: str = ".tmp/meta",
- load_meta_from_disk: bool = True,
+ load_fresh: bool = False,
splits_cfg: Optional[dict] = None,
):
"""Return the dataset using the given parameters.
@@ -102,9 +102,10 @@ def parse_dataset_cfg(
Dataset configuration dictionary.
metadata_dir : str
Directory used for on-disk split and wrapper metadata.
- load_meta_from_disk : bool
- If True, load pre-existing split/wrapper metadata from disk
- instead of regenerating.
+ load_fresh : bool
+ If False (default), reuse cached split/wrapper metadata from
+ disk when present and only generate what's missing. If True,
+ regenerate everything and overwrite any cached files.
splits_cfg : Optional[dict]
Top-level ``splits:`` registry mapping split names to their
recipes (``{filename, ratios, seed}``). Datasets reference an
@@ -116,13 +117,13 @@ def parse_dataset_cfg(
splits_cfg = splits_cfg or {}
dataset = cls._load_dataset_from_cfg(
- ds_config, metadata_dir, load_meta_from_disk, splits_cfg
+ ds_config, metadata_dir, load_fresh, splits_cfg
)
wrapper = copy.deepcopy(ds_config.get("wrapper", None))
if wrapper is not None:
return cls._apply_wrapper(
- dataset, ds_config, wrapper, metadata_dir, load_meta_from_disk
+ dataset, ds_config, wrapper, metadata_dir, load_fresh
)
return dataset
@@ -132,7 +133,7 @@ def split_dataset(
dataset: torch.utils.data.Dataset,
ds_config: dict,
metadata_dir: str,
- load_meta_from_disk: bool = True,
+ load_fresh: bool = False,
splits_cfg: Optional[dict] = None,
):
"""Split the dataset using the given parameters.
@@ -145,8 +146,8 @@ def split_dataset(
The dataset configuration dictionary.
metadata_dir: str
Directory to store the metadata.
- load_meta_from_disk: bool
- Whether to load metadata from disk.
+ load_fresh: bool
+ If True, regenerate the split even if a cached file exists.
splits_cfg: Optional[dict]
Top-level splits registry (name -> recipe).
@@ -163,7 +164,7 @@ def split_dataset(
recipe = cls._resolve_split_recipe(split_ref, splits_cfg or {})
splits = cls._load_split_if_exists_or_generate(
dataset,
- load_meta_from_disk,
+ load_fresh,
metadata_dir,
recipe["filename"],
split_ratios=recipe["ratios"],
@@ -186,13 +187,13 @@ def _load_dataset_from_cfg(
cls,
ds_config: dict,
metadata_dir: str,
- load_meta_from_disk: bool = True,
+ load_fresh: bool = False,
splits_cfg: Optional[dict] = None,
) -> torch.utils.data.Dataset:
"""Load dataset based on configuration."""
if "single_class_dataset" not in ds_config:
return cls._load_hf_dataset_from_config(
- ds_config, metadata_dir, load_meta_from_disk, splits_cfg
+ ds_config, metadata_dir, load_fresh, splits_cfg
)
elif ds_config["single_class_dataset"]:
return cls._load_single_class_dataset(
@@ -206,7 +207,7 @@ def _load_hf_dataset_from_config(
cls,
ds_config: dict,
metadata_dir: str,
- load_meta_from_disk: bool = True,
+ load_fresh: bool = False,
splits_cfg: Optional[dict] = None,
) -> Union[torch.utils.data.Dataset, hf_datasets.Dataset]:
"""Load a HuggingFace dataset based on configuration."""
@@ -223,7 +224,7 @@ def _load_hf_dataset_from_config(
base_dataset,
ds_config,
metadata_dir,
- load_meta_from_disk,
+ load_fresh,
splits_cfg or {},
)
@@ -273,7 +274,7 @@ def _apply_indices(
base_dataset: Union[torch.utils.data.Dataset, hf_datasets.Dataset],
ds_config: dict,
metadata_dir: str,
- load_meta_from_disk: bool = True,
+ load_fresh: bool = False,
splits_cfg: Optional[dict] = None,
) -> Union[torch.utils.data.Dataset, hf_datasets.Dataset]:
"""Apply indices to the dataset based on configuration."""
@@ -284,7 +285,7 @@ def _apply_indices(
split_name = ds_config.get("split_name", "train")
split = cls._load_split_if_exists_or_generate(
base_dataset,
- load_meta_from_disk,
+ load_fresh,
metadata_dir,
split_recipe["filename"],
split_ratios=split_recipe["ratios"],
@@ -321,7 +322,7 @@ def _apply_filter(
dataset: torch.utils.data.Dataset,
ds_config: dict,
metadata_dir: str,
- load_meta_from_disk: bool = True,
+ load_fresh: bool = False,
):
"""Apply the filter to the dataset.
@@ -329,12 +330,14 @@ def _apply_filter(
produced by ``_compute_and_save_indices`` only when a
``filter_by_*`` flag is set. Its absence is treated as "no
filter applied" rather than a strict error — configs commonly
- declare a filename without ever producing the file.
+ declare a filename without ever producing the file. When
+ ``load_fresh`` is True, any existing filter is skipped (the
+ post-training step will regenerate it).
"""
filter_indices_cfg = ds_config.get("filter_indices", None)
if filter_indices_cfg is None:
return dataset
- if not load_meta_from_disk:
+ if load_fresh:
return dataset
filter_filename = filter_indices_cfg.get(
"split_filename", "DOESNT_EXIST"
@@ -357,9 +360,14 @@ def _apply_wrapper(
ds_config: dict,
wrapper_cfg: dict,
metadata_dir: str,
- load_meta_from_disk: bool,
+ load_fresh: bool,
) -> torch.utils.data.Dataset:
- """Apply a wrapper to the dataset based on configuration."""
+ """Apply a wrapper to the dataset based on configuration.
+
+ Wrapper metadata is auto-resolved: cached files are reused when
+ present and ``load_fresh`` is False; otherwise metadata is
+ generated and saved.
+ """
wrapper_cfg = dict(wrapper_cfg)
wrapper_cls = transform_wrappers[wrapper_cfg.pop("type")]
# check if wrapper_cls is a subclass of TransformedDataset
@@ -369,24 +377,20 @@ def _apply_wrapper(
)
kwargs = wrapper_cfg
+ meta_filename = "DOESNT_EXIST"
+ loaded_from_disk = False
if "metadata" in kwargs:
metadata_args = dict(kwargs.pop("metadata", {}))
meta_filename = metadata_args.pop(
"metadata_filename", "DOESNT_EXIST"
)
- if load_meta_from_disk:
- if not wrapper_cls.metadata_cls.exists(
- metadata_dir, meta_filename
- ):
- raise FileNotFoundError(
- f"Wrapper metadata '{meta_filename}' not found in "
- f"{metadata_dir}. Re-run with "
- f"load_meta_from_disk=False to regenerate it."
- )
- loaded_meta = wrapper_cls.metadata_cls.load(
+ if not load_fresh and wrapper_cls.metadata_cls.exists(
+ metadata_dir, meta_filename
+ ):
+ kwargs["metadata"] = wrapper_cls.metadata_cls.load(
metadata_dir, meta_filename
)
- kwargs["metadata"] = loaded_meta
+ loaded_from_disk = True
else:
kwargs["metadata"] = wrapper_cls.metadata_cls(**metadata_args)
@@ -394,7 +398,7 @@ def _apply_wrapper(
mapping = ClassMapping.resolve(
kwargs.pop("class_to_group"),
metadata_dir,
- load_meta_from_disk,
+ load_fresh=load_fresh,
)
kwargs["class_to_group"] = mapping.class_to_group
kwargs["n_classes"] = mapping.n_classes
@@ -411,9 +415,9 @@ def _apply_wrapper(
wrapped_dataset,
ds_config,
metadata_dir,
- load_meta_from_disk,
+ load_fresh,
)
- if not load_meta_from_disk:
+ if not loaded_from_disk and meta_filename != "DOESNT_EXIST":
filtered_dataset.metadata.save(metadata_dir, meta_filename)
return filtered_dataset
@@ -421,28 +425,22 @@ def _apply_wrapper(
def _load_split_if_exists_or_generate(
cls,
dataset,
- load_meta_from_disk,
+ load_fresh,
metadata_dir,
split_filename,
split_ratios: Optional[dict] = None,
):
"""Load the split from disk or generate it.
- When ``load_meta_from_disk=True``, the split file must already
- exist; a ``FileNotFoundError`` is raised if it does not. When
- ``load_meta_from_disk=False``, a new split is generated and
- saved to disk.
+ When ``load_fresh=False`` (default), reuses the cached split if
+ present; otherwise generates a new split and saves it. When
+ ``load_fresh=True``, always regenerates and overwrites.
"""
if split_ratios is None:
split_ratios = {"train": 0.9, "test": 0.1}
- if load_meta_from_disk:
- if not DatasetSplit.exists(metadata_dir, split_filename):
- raise FileNotFoundError(
- f"Split file '{split_filename}' not found in "
- f"{metadata_dir}. Re-run with "
- f"load_meta_from_disk=False to regenerate it, or "
- f"populate the cache first."
- )
+ if not load_fresh and DatasetSplit.exists(
+ metadata_dir, split_filename
+ ):
return DatasetSplit.load(metadata_dir, split_filename)
split = DatasetSplit.split(len(dataset), 42, split_ratios)
split.save(metadata_dir, split_filename)
@@ -732,12 +730,41 @@ class FactTracingConfigParser:
@classmethod
def parse_fact_tracing_cfg(
- cls, cfg: dict
+ cls,
+ cfg: dict,
+ offline: bool = False,
+ load_fresh: bool = False,
) -> Tuple[hf_datasets.Dataset, hf_datasets.Dataset, torch.Tensor, int]:
- """Build ``(prompt_ds, evidence_ds, entailment_labels, pad_id)``."""
+ """Build ``(prompt_ds, evidence_ds, entailment_labels, pad_id)``.
+
+ Parameters
+ ----------
+ cfg : dict
+ Fact-tracing configuration dictionary.
+ offline : bool, optional
+ If True, no HTTP request is issued; the HF source dataset
+ must already be present in the local cache. By default False.
+ load_fresh : bool, optional
+ If True, force re-download of the HF source dataset,
+ overwriting the local cache. Incompatible with
+ ``offline=True``. By default False.
+ """
+ if offline and load_fresh:
+ raise ValueError(
+ "offline=True and load_fresh=True are incompatible: "
+ "cannot refresh the cache without network access."
+ )
tokenize, pad_id = resolve_tokenizer(cfg["tokenizer"])
+ if load_fresh:
+ download_mode = "force_redownload"
+ elif offline:
+ download_mode = "reuse_cache_if_exists"
+ else:
+ download_mode = "reuse_dataset_if_exists"
ds = load_dataset(
- cfg["dataset_str"], split=cfg.get("dataset_split", "train")
+ cfg["dataset_str"],
+ split=cfg.get("dataset_split", "train"),
+ download_mode=download_mode,
)
num_prompts = cfg.get("num_prompts", 20)
diff --git a/quanda/benchmarks/downstream_eval/_fact_tracing.py b/quanda/benchmarks/downstream_eval/_fact_tracing.py
index dac52305..a3be66ef 100644
--- a/quanda/benchmarks/downstream_eval/_fact_tracing.py
+++ b/quanda/benchmarks/downstream_eval/_fact_tracing.py
@@ -19,7 +19,12 @@
from quanda.explainers import Explainer
from quanda.metrics import Metric
from quanda.utils.cache import BatchedCachedExplanations
-from quanda.utils.common import CheckpointLoadFunc, _subsample_indices, ds_len
+from quanda.utils.common import (
+ CheckpointLoadFunc,
+ _resolve_config,
+ _subsample_indices,
+ ds_len,
+)
class FactTracingBenchmark(Benchmark):
@@ -70,29 +75,42 @@ def __init__(
@classmethod
def from_config(
cls,
- config: dict,
- load_meta_from_disk: bool = True,
+ config: Union[dict, str],
offline: bool = False,
device: str = "cpu",
metadata_suffix: str = "",
load_fresh: bool = False,
) -> "FactTracingBenchmark":
- """Build the benchmark from a YAML-derived config dict.
+ """Initialize the benchmark from a config.
+
+ Parameters
+ ----------
+ config : Union[dict, str]
+ The benchmark configuration dictionary, a path to a YAML file
+ or registered ``bench_id``
+ (see :data:`quanda.benchmarks.resources.config_map.config_map`).
+ offline : bool, optional
+ If True, no HTTP request is issued to the Hub; all assets
+ (metadata, model) must already be present under
+ ``config['bench_save_dir']``. By default False.
+ device : str, optional
+ Device to load the model on, by default "cpu".
+ metadata_suffix : str, optional
+ Suffix to disambiguate metadata directories. By default "".
+ load_fresh: bool, False
+ If True, regenerate any cached split/wrapper metadata
+ (overwriting local files) and, when ``offline=False``,
+ re-download metadata/model from the Hub. When False
+ (default), cached metadata is reused if present and only
+ missing pieces are generated.
- Loads prompts/evidence/entailment via
- :func:`load_fact_tracing_datasets_from_cfg` (which bypasses the
- generic dataset parser because one HF dataset fans out into
- both splits) and the model via the standard
- :class:`ModelConfigParser` path.
"""
- if offline and load_fresh:
- raise ValueError(
- "offline=True and load_fresh=True are incompatible."
- )
-
+ config = _resolve_config(config)
prompt_ds, evidence_ds, entailment_labels, _ = (
FactTracingConfigParser.parse_fact_tracing_cfg(
- config["fact_tracing"]
+ config["fact_tracing"],
+ offline=offline,
+ load_fresh=load_fresh,
)
)
@@ -102,7 +120,6 @@ def from_config(
bench_save_dir=config.get("bench_save_dir", "./tmp"),
ckpts=_resolve_ckpts(config),
offline=offline,
- load_fresh=load_fresh,
device=device,
)
)
diff --git a/quanda/benchmarks/downstream_eval/class_detection.py b/quanda/benchmarks/downstream_eval/class_detection.py
index 8a21fd8a..51430dd0 100644
--- a/quanda/benchmarks/downstream_eval/class_detection.py
+++ b/quanda/benchmarks/downstream_eval/class_detection.py
@@ -74,7 +74,7 @@ def _extra_kwargs_from_config(
train_dataset: torch.utils.data.Dataset,
eval_dataset: torch.utils.data.Dataset,
metadata_dir: str,
- load_meta_from_disk: bool,
+ load_fresh: bool,
) -> dict:
"""Extract class detection kwargs from config."""
return {
diff --git a/quanda/benchmarks/downstream_eval/mislabeling_detection.py b/quanda/benchmarks/downstream_eval/mislabeling_detection.py
index 088680ca..f2e749dc 100644
--- a/quanda/benchmarks/downstream_eval/mislabeling_detection.py
+++ b/quanda/benchmarks/downstream_eval/mislabeling_detection.py
@@ -2,7 +2,7 @@
import logging
import os
-from typing import Optional
+from typing import Optional, Union
import torch
import yaml
@@ -14,7 +14,12 @@
)
from quanda.metrics.downstream_eval import MislabelingDetectionMetric
from quanda.utils.cache import ExplanationsCache
-from quanda.utils.common import _subsample_dataset, class_accuracy, ds_len
+from quanda.utils.common import (
+ _resolve_config,
+ _subsample_dataset,
+ class_accuracy,
+ ds_len,
+)
from quanda.utils.datasets.transformed.label_flipping import (
LabelFlippingDataset,
)
@@ -231,7 +236,7 @@ def evaluate(
@classmethod
def explain(
cls,
- config: dict,
+ config: Union[dict, str],
explainer_cls: type,
expl_kwargs: Optional[dict] = None,
batch_size: int = 8,
@@ -251,7 +256,11 @@ def explain(
here parameterize the train-dataset subsample over which
self-influence is computed. ``inference_batch_size`` is ignored
since there is no eval-time inference pass.
+
+ ``config`` accepts a config dict, a registered ``bench_id``, or
+ a path to a benchmark YAML.
"""
+ config = _resolve_config(config)
obj = cls.from_config(config, device=device)
if explanations_id is None:
explanations_id = default_explanations_id(
diff --git a/quanda/benchmarks/downstream_eval/shortcut_detection.py b/quanda/benchmarks/downstream_eval/shortcut_detection.py
index 99390d79..bd266921 100644
--- a/quanda/benchmarks/downstream_eval/shortcut_detection.py
+++ b/quanda/benchmarks/downstream_eval/shortcut_detection.py
@@ -89,7 +89,7 @@ def _extra_kwargs_from_config(
train_dataset: torch.utils.data.Dataset,
eval_dataset: torch.utils.data.Dataset,
metadata_dir: str,
- load_meta_from_disk: bool,
+ load_fresh: bool,
) -> dict:
"""Extract shortcut detection kwargs from config."""
if not isinstance(eval_dataset, SampleTransformationDataset):
diff --git a/quanda/benchmarks/downstream_eval/subclass_detection.py b/quanda/benchmarks/downstream_eval/subclass_detection.py
index c2dbaf9e..c0dbce28 100644
--- a/quanda/benchmarks/downstream_eval/subclass_detection.py
+++ b/quanda/benchmarks/downstream_eval/subclass_detection.py
@@ -90,7 +90,7 @@ def _extra_kwargs_from_config(
train_dataset: torch.utils.data.Dataset,
eval_dataset: torch.utils.data.Dataset,
metadata_dir: str,
- load_meta_from_disk: bool,
+ load_fresh: bool,
) -> dict:
"""Extract subclass detection kwargs from config."""
if not isinstance(train_dataset, LabelGroupingDataset):
diff --git a/quanda/benchmarks/ground_truth/linear_datamodeling.py b/quanda/benchmarks/ground_truth/linear_datamodeling.py
index 51132373..812989f0 100644
--- a/quanda/benchmarks/ground_truth/linear_datamodeling.py
+++ b/quanda/benchmarks/ground_truth/linear_datamodeling.py
@@ -6,7 +6,7 @@
import random
import warnings
from copy import deepcopy
-from typing import Callable, List, Optional, Tuple
+from typing import Callable, List, Optional, Tuple, Union
import lightning as L
import torch
@@ -22,6 +22,7 @@
LinearDatamodelingMetric,
)
from quanda.utils.common import (
+ _resolve_config,
_subsample_dataset,
chunked_logits,
class_accuracy,
@@ -213,12 +214,13 @@ def _train_subset_models(
@classmethod
def train( # type: ignore[override]
cls,
- config: dict,
+ config: Union[dict, str],
logger: Optional[L.pytorch.loggers.logger.Logger] = None,
device: str = "cpu",
batch_size: int = 64,
skip_subsets: bool = False,
- load_meta_from_disk: bool = False,
+ load_fresh: bool = True,
+ use_pid: bool = False,
) -> "LinearDatamodeling":
"""Train main model and subset models.
@@ -239,9 +241,13 @@ def train( # type: ignore[override]
If True, skip the subset training loop. Used when subsets
are trained out-of-band (e.g. one-by-one in parallel
workers via :meth:`train_subset`).
- load_meta_from_disk : bool, optional
- If True, reuse existing metadata (splits, subset_ids, etc.)
- from the cache instead of regenerating. By default False.
+ load_fresh : bool, optional
+ If True (default), regenerate splits/subset_ids/etc.
+ instead of reusing the cache.
+ use_pid : bool, optional
+ If True, suffix checkpoint and metadata directories with
+ the current process id to disambiguate concurrent runs. By
+ default False.
Returns
-------
@@ -249,12 +255,14 @@ def train( # type: ignore[override]
The trained benchmark instance.
"""
+ config = _resolve_config(config)
obj = super().train(
config=config,
logger=logger,
device=device,
batch_size=batch_size,
- load_meta_from_disk=load_meta_from_disk,
+ load_fresh=load_fresh,
+ use_pid=use_pid,
)
if not isinstance(obj, LinearDatamodeling):
raise TypeError("Expected a LinearDatamodeling instance.")
@@ -278,12 +286,12 @@ def train( # type: ignore[override]
@classmethod
def train_subset(
cls,
- config: dict,
+ config: Union[dict, str],
idx: int,
device: str = "cpu",
batch_size: int = 64,
push_to_hub: bool = False,
- load_meta_from_disk: bool = True,
+ load_fresh: bool = False,
) -> "LinearDatamodeling":
"""Train and save a single subset model by index.
@@ -304,16 +312,16 @@ def train_subset(
Batch size.
push_to_hub : bool, optional
If True, push the resulting subset checkpoint to HF Hub.
- load_meta_from_disk : bool, optional
- Whether to load existing metadata (subset_ids, etc.) from disk.
- If False, will regenerate metadata from the main model and
- which may lead to different subset splits if the generation is
- not deterministic (e.g. if the seed is not fixed). By default True.
+ load_fresh : bool, optional
+ If True, regenerate cached metadata (subset_ids, etc.).
+ Doing so can change the subset splits if generation is not
+ deterministic. By default False — reuse cached metadata.
"""
+ config = _resolve_config(config)
obj = cls.from_config(
config,
- load_meta_from_disk=load_meta_from_disk,
+ load_fresh=load_fresh,
offline=True,
device=device,
)
@@ -343,23 +351,24 @@ def train_subset(
@classmethod
def generate_and_push_metadata(
- cls, config: dict
+ cls, config: Union[dict, str]
) -> None: # pragma: no cover
"""Regenerate LDS metadata locally and push it to HF Hub.
- Calls ``from_config`` with ``load_meta_from_disk=False, offline=True``
- to materialize splits and subset_ids under the metadata dir, then
+ Calls ``from_config`` with ``load_fresh=True, offline=True`` to
+ materialize splits and subset_ids under the metadata dir, then
uploads that dir to ``meta_id``.
"""
from huggingface_hub import HfApi # local import; optional dep path
+ config = _resolve_config(config)
metadata_dir = MetadataConfigParser.get_metadata_dir(
cfg=config, bench_save_dir=config["bench_save_dir"]
)
meta_id = config.get(
"meta_id", f"{config['repo_id']}/{config['id']}_metadata"
)
- cls.from_config(config, load_meta_from_disk=False, offline=True)
+ cls.from_config(config, load_fresh=True, offline=True)
api = HfApi()
api.create_repo(repo_id=meta_id, repo_type="dataset", exist_ok=True)
@@ -372,7 +381,7 @@ def generate_and_push_metadata(
@classmethod
def push_subset(
cls,
- config: dict,
+ config: Union[dict, str],
idx: int,
) -> None:
"""Push an already-trained subset checkpoint to HF Hub.
@@ -381,6 +390,7 @@ def push_subset(
"""
from huggingface_hub import HfApi # local import; optional dep path
+ config = _resolve_config(config)
local_ckpt_dir, repo_id = _subset_ckpt_paths(config, idx)
if not os.path.isdir(local_ckpt_dir):
@@ -400,7 +410,7 @@ def _extra_kwargs_from_config(
train_dataset: torch.utils.data.Dataset,
eval_dataset: torch.utils.data.Dataset,
metadata_dir: str,
- load_meta_from_disk: bool,
+ load_fresh: bool,
) -> dict:
"""Extract linear datamodeling kwargs from config."""
m = config.get("m", 100)
@@ -428,13 +438,7 @@ def _extra_kwargs_from_config(
generator.manual_seed(seed)
subset_meta = f"{metadata_dir}/{config['subset_ids']}"
- if load_meta_from_disk:
- if not os.path.exists(subset_meta):
- raise FileNotFoundError(
- f"Subset ids file not found at {subset_meta}. "
- f"Re-run with load_meta_from_disk=False to "
- f"regenerate it."
- )
+ if not load_fresh and os.path.exists(subset_meta):
with open(subset_meta, "r") as f:
subset_ids = yaml.safe_load(f)
else:
@@ -464,13 +468,15 @@ def _extra_kwargs_from_config(
@classmethod
def train_and_push_to_hub(
cls,
- config: dict,
+ config: Union[dict, str],
logger: Optional[L.pytorch.loggers.logger.Logger] = None,
device: str = "cpu",
batch_size: int = 64,
- load_meta_from_disk: bool = False,
+ load_fresh: bool = True,
+ use_pid: bool = False,
): # pragma: no cover
"""Train a model using the provided config and push to HF hub."""
+ config = _resolve_config(config)
skip_subsets = bool(config.get("skip_subsets", False))
cls._push_subsets_during_train = not skip_subsets
cls._lds_skip_subsets = skip_subsets
@@ -480,7 +486,8 @@ def train_and_push_to_hub(
logger=logger,
device=device,
batch_size=batch_size,
- load_meta_from_disk=load_meta_from_disk,
+ load_fresh=load_fresh,
+ use_pid=use_pid,
)
finally:
cls._push_subsets_during_train = False
@@ -537,12 +544,13 @@ def _load_subset_model(
@classmethod
def subset_logits_cache_dir(
cls,
- config: dict,
+ config: Union[dict, str],
batch_size: int = 8,
max_eval_n: Optional[int] = 1000,
eval_seed: int = 42,
) -> str:
"""Return default local cache dir for counterfactual subset logits."""
+ config = _resolve_config(config)
repo = config.get("repo_id", "quanda-bench-test")
group = config.get("explanations_group", config["id"])
logits_id = (
@@ -583,7 +591,7 @@ def _collect_eval_batches(
@classmethod
def cache_subset_logits_per_idx(
cls,
- config: dict,
+ config: Union[dict, str],
idx: int,
batch_size: int = 8,
cache_dir: Optional[str] = None,
@@ -592,7 +600,12 @@ def cache_subset_logits_per_idx(
eval_seed: int = 42,
inference_batch_size: Optional[int] = None,
) -> str:
- """Cache counterfactual logits for a **single** subset index."""
+ """Cache counterfactual logits for a **single** subset index.
+
+ ``config`` accepts a config dict, a registered ``bench_id``, or
+ a path to a benchmark YAML.
+ """
+ config = _resolve_config(config)
obj = cls.from_config(config, device=device)
if not isinstance(obj, LinearDatamodeling):
raise TypeError("Expected a LinearDatamodeling instance.")
@@ -633,7 +646,7 @@ def cache_subset_logits_per_idx(
@classmethod
def cache_subset_logits(
cls,
- config: dict,
+ config: Union[dict, str],
batch_size: int = 8,
cache_dir: Optional[str] = None,
device: str = "cpu",
@@ -641,7 +654,12 @@ def cache_subset_logits(
eval_seed: int = 42,
inference_batch_size: Optional[int] = None,
) -> str:
- """Cache counterfactual logits for every (subset, eval batch)."""
+ """Cache counterfactual logits for every (subset, eval batch).
+
+ ``config`` accepts a config dict, a registered ``bench_id``, or
+ a path to a benchmark YAML.
+ """
+ config = _resolve_config(config)
obj = cls.from_config(config, device=device)
if not isinstance(obj, LinearDatamodeling):
raise TypeError("Expected a LinearDatamodeling instance.")
diff --git a/quanda/benchmarks/heuristics/mixed_datasets.py b/quanda/benchmarks/heuristics/mixed_datasets.py
index 30ed81b4..9d2a369d 100644
--- a/quanda/benchmarks/heuristics/mixed_datasets.py
+++ b/quanda/benchmarks/heuristics/mixed_datasets.py
@@ -1,7 +1,7 @@
"""Mixed Datasets benchmark module."""
import logging
-from typing import List, Optional
+from typing import List, Optional, Union
import torch
from torch.utils.data import Subset
@@ -13,7 +13,7 @@
ModelConfigParser,
)
from quanda.metrics.heuristics.mixed_datasets import MixedDatasetsMetric
-from quanda.utils.common import class_accuracy, ds_len
+from quanda.utils.common import _resolve_config, class_accuracy, ds_len
logger = logging.getLogger(__name__)
@@ -88,8 +88,7 @@ def __init__(
@classmethod
def from_config(
cls,
- config: dict,
- load_meta_from_disk: bool = True,
+ config: Union[dict, str],
offline: bool = False,
device: str = "cpu",
metadata_suffix: str = "",
@@ -101,9 +100,6 @@ def from_config(
----------
config : dict
Dictionary containing the configuration.
- load_meta_from_disk : str
- Loads dataset metadata from disk if True, otherwise generates
- it, default True.
offline : bool, optional
If True, no HTTP request is issued to the Hub, by default
False.
@@ -113,15 +109,13 @@ def from_config(
Suffix to add to the metadata directory name, by default "".
User to prevent assets clashing when multiprocessing.
load_fresh : bool, optional
- If True, force re-download of the model checkpoints from the
- Hub, overwriting the local cache. Incompatible with
- ``offline=True``. By default False.
+ If True, force re-download of the model checkpoints from
+ the Hub and regenerate cached metadata, overwriting the
+ local cache. Incompatible with ``offline=True``.
+ By default False.
"""
- if offline and load_fresh:
- raise ValueError(
- "offline=True and load_fresh=True are incompatible."
- )
+ config = _resolve_config(config)
metadata_dir = MetadataConfigParser.get_metadata_dir(
cfg=config,
bench_save_dir=config.get("bench_save_dir", "./tmp"),
@@ -131,26 +125,26 @@ def from_config(
train_base_dataset = DatasetConfigParser.parse_dataset_cfg(
ds_config=config["train_dataset"],
metadata_dir=metadata_dir,
- load_meta_from_disk=load_meta_from_disk,
+ load_fresh=load_fresh,
splits_cfg=splits_cfg,
)
val_base_dataset = DatasetConfigParser.parse_dataset_cfg(
ds_config=config.get("val_dataset", None),
metadata_dir=metadata_dir,
- load_meta_from_disk=load_meta_from_disk,
+ load_fresh=load_fresh,
splits_cfg=splits_cfg,
)
adv_dataset = DatasetConfigParser.parse_dataset_cfg(
ds_config=config["adv_dataset"],
metadata_dir=metadata_dir,
- load_meta_from_disk=load_meta_from_disk,
+ load_fresh=load_fresh,
splits_cfg=splits_cfg,
)
split_datasets = DatasetConfigParser.split_dataset(
dataset=adv_dataset,
ds_config=config["adv_dataset"],
metadata_dir=metadata_dir,
- load_meta_from_disk=load_meta_from_disk,
+ load_fresh=load_fresh,
splits_cfg=splits_cfg,
)
adv_base_dataset = split_datasets["train"]
@@ -181,7 +175,6 @@ def from_config(
bench_save_dir=config["bench_save_dir"],
ckpts=_resolve_ckpts(config),
offline=offline,
- load_fresh=load_fresh,
device=device,
)
)
diff --git a/quanda/benchmarks/heuristics/model_randomization.py b/quanda/benchmarks/heuristics/model_randomization.py
index da988a4e..c4024a9f 100644
--- a/quanda/benchmarks/heuristics/model_randomization.py
+++ b/quanda/benchmarks/heuristics/model_randomization.py
@@ -76,7 +76,7 @@ def _extra_kwargs_from_config(
train_dataset: torch.utils.data.Dataset,
eval_dataset: torch.utils.data.Dataset,
metadata_dir: str,
- load_meta_from_disk: bool,
+ load_fresh: bool,
) -> dict:
"""Extract model randomization kwargs from config."""
return {
diff --git a/quanda/benchmarks/heuristics/top_k_cardinality.py b/quanda/benchmarks/heuristics/top_k_cardinality.py
index 41fa45e3..c9d35b8b 100644
--- a/quanda/benchmarks/heuristics/top_k_cardinality.py
+++ b/quanda/benchmarks/heuristics/top_k_cardinality.py
@@ -62,7 +62,7 @@ def _extra_kwargs_from_config(
train_dataset: Union[torch.utils.data.Dataset, datasets.Dataset],
eval_dataset: torch.utils.data.Dataset,
metadata_dir: str,
- load_meta_from_disk: bool,
+ load_fresh: bool,
) -> dict:
"""Extract top_k from config."""
return {"top_k": config["top_k"]}
diff --git a/quanda/explainers/base.py b/quanda/explainers/base.py
index a06a9ab3..0619e584 100644
--- a/quanda/explainers/base.py
+++ b/quanda/explainers/base.py
@@ -167,14 +167,7 @@ def self_influence(self, batch_size: int = 32) -> torch.Tensor:
return influences
def load_last_checkpoint(self):
- """Load the model from the checkpoint file.
-
- Parameters
- ----------
- checkpoint : str
- Path to the checkpoint file.
-
- """
+ """Load the last checkpoint in ``self.checkpoints`` into the model."""
load_last_checkpoint(
model=self.model,
checkpoints=self.checkpoints,
diff --git a/quanda/explainers/wrappers/representer_points.py b/quanda/explainers/wrappers/representer_points.py
index 6270c7d1..bca47e70 100644
--- a/quanda/explainers/wrappers/representer_points.py
+++ b/quanda/explainers/wrappers/representer_points.py
@@ -451,6 +451,18 @@ def explain(
explanations = torch.gather(explanations, dim=-1, index=indices)
return torch.squeeze(explanations)
+ def _train_step(self, model, optimizer, x, y, N):
+ """Run a single optimizer step and return loss, phi_loss, grad_loss."""
+ optimizer.zero_grad()
+ (Phi, L2) = model(x, y)
+ loss = L2 * self.lmbd + Phi / N
+ phi_loss = (Phi / N).detach().cpu().numpy()
+ loss.backward()
+ if model.W.grad is None:
+ raise ValueError("Gradient is None")
+ grad_loss = torch.mean(torch.abs(model.W.grad)).item()
+ return loss, phi_loss, grad_loss
+
def train(self):
"""Train the model to obtain the representer point coefficients.
@@ -502,21 +514,14 @@ def train(self):
best_W = model.W.data.clone()
init_grad = float("inf")
+ grad_loss = float("inf")
for epoch in range(self.epoch):
- phi_loss = 0
- optimizer.zero_grad()
- (Phi, L2) = model(x, y)
- loss = L2 * self.lmbd + Phi / N
- phi_loss += (Phi / N).detach().cpu().numpy()
- loss.backward()
+ loss, phi_loss, grad_loss = self._train_step(
+ model, optimizer, x, y, N
+ )
temp_W = model.W.data
- if model.W.grad is None:
- raise ValueError("Gradient is None")
-
- grad_loss = torch.mean(torch.abs(model.W.grad)).item()
-
if epoch == 0:
init_grad = grad_loss
best_W = temp_W
@@ -530,6 +535,7 @@ def train(self):
"Stopping criteria reached in epoch :{}".format(epoch)
)
break
+ assert model.W.grad is not None
self.backtracking_line_search(model, model.W.grad, x, y, loss, N)
if self.show_progress:
pbar.set_description(
@@ -542,8 +548,8 @@ def train(self):
if grad_loss == init_grad:
raise ValueError(
- "Gradient did not decrease during training. Consider increasing "
- "the number of epochs or the learning rate."
+ "Gradient did not decrease during training. Consider "
+ "increasing the number of epochs or the learning rate."
)
# calculate w based on the representer theorem's decomposition
temp = torch.matmul(
diff --git a/quanda/metrics/base.py b/quanda/metrics/base.py
index d777713d..0d134d0b 100644
--- a/quanda/metrics/base.py
+++ b/quanda/metrics/base.py
@@ -200,14 +200,7 @@ def state_dict(self) -> dict:
raise NotImplementedError
def load_last_checkpoint(self):
- """Load the model from the checkpoint file.
-
- Parameters
- ----------
- checkpoint : str
- Path to the checkpoint file.
-
- """
+ """Load the last checkpoint in ``self.checkpoints`` into the model."""
load_last_checkpoint(
model=self.model,
checkpoints=self.checkpoints,
diff --git a/quanda/utils/common.py b/quanda/utils/common.py
index fe52c393..25a3a773 100644
--- a/quanda/utils/common.py
+++ b/quanda/utils/common.py
@@ -29,6 +29,39 @@
CheckpointLoadFunc = Callable[[torch.nn.Module, str], Any]
+def _resolve_config(config: Union[dict, str]) -> dict:
+ """Resolve a benchmark ``config`` into a dict.
+
+ Accepts:
+ - a config dict (passes through unchanged),
+ - a registered ``bench_id`` (resolved via
+ :data:`quanda.benchmarks.resources.config_map.config_map`), or
+ - a path to a benchmark YAML file.
+
+ Raises ``TypeError`` for any other input, or if the loaded YAML
+ does not parse to a mapping.
+ """
+ if isinstance(config, dict):
+ return config
+ if isinstance(config, str):
+ # Lazy import to avoid a hard dep from utils → benchmarks.
+ from quanda.benchmarks.resources.config_map import config_map
+
+ path = str(config_map[config]) if config in config_map else config
+ with open(path, "r") as f:
+ cfg = yaml.safe_load(f)
+ if not isinstance(cfg, dict):
+ raise TypeError(
+ f"YAML at {path} did not parse to a dict (got "
+ f"{type(cfg).__name__})."
+ )
+ return cfg
+ raise TypeError(
+ f"config must be a dict, a registered bench_id, or a YAML path; "
+ f"got {type(config).__name__}."
+ )
+
+
def chunked_logits(
model: torch.nn.Module,
inputs: Any,
diff --git a/quanda/utils/datasets/transformed/metadata.py b/quanda/utils/datasets/transformed/metadata.py
index 930fdcc3..6a18136d 100644
--- a/quanda/utils/datasets/transformed/metadata.py
+++ b/quanda/utils/datasets/transformed/metadata.py
@@ -269,7 +269,7 @@ def resolve(
cls,
spec: dict,
metadata_dir: str,
- load_meta_from_disk: bool,
+ load_fresh: bool = False,
) -> "ClassMapping":
"""Resolve a ``class_to_group`` config spec to a ``ClassMapping``.
@@ -277,6 +277,8 @@ def resolve(
- ``{0: g0, 1: g1, ...}`` — direct mapping (integer keys).
- ``{ctg_filename, n_classes, n_groups, seed?}`` — file-backed;
load if exists, otherwise generate from ``seed`` and save.
+ ``load_fresh=True`` forces regeneration even if a cached file
+ is present.
"""
if spec and all(isinstance(k, int) for k in spec.keys()):
mapping = {int(k): int(v) for k, v in spec.items()}
@@ -291,13 +293,7 @@ def resolve(
n_groups = int(spec["n_groups"])
seed = int(spec.get("seed", 42))
- if load_meta_from_disk:
- if not cls.exists(metadata_dir, ctg_filename):
- raise FileNotFoundError(
- f"Class mapping '{ctg_filename}' not found in "
- f"{metadata_dir}. Re-run with "
- f"load_meta_from_disk=False to regenerate it."
- )
+ if not load_fresh and cls.exists(metadata_dir, ctg_filename):
return cls.load(metadata_dir, ctg_filename)
mapping = cls._generate(n_classes, n_groups, seed)
diff --git a/scripts/train.py b/scripts/train.py
index d0fd9e6c..343c18fd 100644
--- a/scripts/train.py
+++ b/scripts/train.py
@@ -18,7 +18,11 @@ def main(cfg: DictConfig) -> Tuple[float]:
bench_cls = bench_dict[cfg.bench]
logger = LoggerConfigParser.parse_logger(cfg)
bench = bench_cls.train(
- cfg, logger=logger, device=device, batch_size=cfg.batch_size
+ cfg,
+ logger=logger,
+ device=device,
+ batch_size=cfg.batch_size,
+ use_pid=True,
)
scores = bench.sanity_check()
print(f"Sanity check scores: {scores}")
diff --git a/scripts/train_and_push_to_hub.py b/scripts/train_and_push_to_hub.py
index 1e3f1f8b..5702ba8e 100644
--- a/scripts/train_and_push_to_hub.py
+++ b/scripts/train_and_push_to_hub.py
@@ -23,6 +23,7 @@ def main(cfg: DictConfig) -> Tuple[float]:
logger=logger,
device=cfg.device,
batch_size=cfg.batch_size,
+ use_pid=True,
)
return 0.0
diff --git a/tests/assets/mnist_local_bench/0_rand_0.pth b/tests/assets/mnist_local_bench/0_rand_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..a5afc5fab8b6ea99f046a2adebff2cd26354dee8
GIT binary patch
literal 181382
zcmb@t2{@JC_cweDAw!7@zdr6b#
zQZ!GRHP59|J*V&Y{XM_u_kZ8#xt{-Zy~lMQ>}%hj{WCvOpNva
zdQDgxsh$NTHXL`2$HNwK{D*B4hXFfckYGB_|kEIKMq+$lCFI5s%Q
zIxH|=oWs&`