Toy dataset generation for pseudo-experiments

ML/Calibration/calibration_runner.py

@@ -93,15 +93,19 @@ def _list_jobs_and_exit(cfg, args):
 # UID splitting
 # --------------------------------------------------------------------------------
 
-def _uid_c2st_intervals(job):
-    """Build the UID splitter and the 'c2st'/'c2st_val' bucket intervals for a job.
-
-    Calibration always evaluates the split held out from BIT training (the
-    'c2st_train'/'c2st_val' buckets), matching the convention used for C2ST checks.
+def _uid_split_interval(split_cfg: dict, *split_names: str):
+    """Build the UID splitter and the merged bucket interval for one or more named splits.
+
+    `split_cfg` is a `splitting:` block (`{enabled, uid_fields, seed, n_buckets, scheme}`),
+    either a job's or `defaults.splitting` directly -- both have the same shape.
+    Multiple split names are merged into one interval by concatenating their [lo,hi)
+    bucket ranges in scheme order; this is only a contiguous range when the named
+    splits are adjacent in `scheme`, which holds for ('c2st_train','c2st_val') and
+    for any standalone split such as ('final_eval',).
     """
-    split_cfg = job.get("splitting") or {}
+    split_cfg = split_cfg or {}
     if not bool(split_cfg.get("enabled", False)):
-        raise RuntimeError("Calibration requires job.splitting.enabled=True (UID splitting) to avoid data leakage.")
+        raise RuntimeError("Requires splitting.enabled=True (UID splitting) to avoid data leakage.")
 
     uid_fields = split_cfg.get("uid_fields", ["run", "luminosityBlock", "event"])
     uid_seed = int(split_cfg.get("seed", 0))
@@ -121,17 +125,27 @@ def _uid_c2st_intervals(job):
         uid_intervals[k] = (lo, lo + int(sz))
         lo += int(sz)
 
-    if "c2st_train" not in uid_intervals or "c2st_val" not in uid_intervals:
-        raise RuntimeError("splitting.scheme must define 'c2st_train' and 'c2st_val'.")
-
-    # assumes they're always in the order c2st_train, c2st_valid
-    c2st_interval = (uid_intervals["c2st_train"][0], uid_intervals["c2st_val"][1])
+    missing = [nm for nm in split_names if nm not in uid_intervals]
+    if missing:
+        raise RuntimeError(f"splitting.scheme must define {missing}.")
+
+    merged = (min(uid_intervals[nm][0] for nm in split_names),
+              max(uid_intervals[nm][1] for nm in split_names))
 
     logger.info("[UID] fields=%s seed=%d n_buckets=%d", uid_fields, uid_seed, uid_n_buckets)
     logger.info("[UID] scheme intervals: %s", uid_intervals)
-    logger.info("[UID] BIT train split 'c2st_train'+'c2st_val' -> %s", c2st_interval)
+    logger.info("[UID] split %s -> %s", split_names, merged)
+
+    return uid_splitter, list(uid_fields), merged
 
-    return uid_splitter, list(uid_fields), c2st_interval
+
+def _uid_c2st_intervals(job):
+    """Build the UID splitter and the 'c2st_train'+'c2st_val' bucket interval for a job.
+
+    Calibration always evaluates the split held out from BIT training (the
+    'c2st_train'/'c2st_val' buckets), matching the convention used for C2ST checks.
+    """
+    return _uid_split_interval(job.get("splitting") or {}, "c2st_train", "c2st_val")
 
 
 # --------------------------------------------------------------------------------

ML/Calibration/eft_calibration.py

@@ -20,7 +20,7 @@
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
-from common.derivative_providers import EFTDerivativeProvider
+from common.derivative_providers import build_derivative_provider
 
 import calibration_runner as cr
 
@@ -32,7 +32,7 @@ def main():
     if not job.get("eft"):
         raise RuntimeError(f"Job '{job['id']}' has no 'eft' block; use pdf_calibration.py for PDF jobs.")
 
-    provider = EFTDerivativeProvider(job["eft"].get("parameters", []))
+    provider = build_derivative_provider(job)
     cr.run_calibration(cfg, job, samples_mod, args, provider)
 
 

ML/Calibration/pdf_calibration.py

@@ -21,7 +21,7 @@
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
-from common.derivative_providers import PDFDerivativeProvider
+from common.derivative_providers import build_derivative_provider
 
 import calibration_runner as cr
 
@@ -33,7 +33,7 @@ def main():
     if not job.get("pdf"):
         raise RuntimeError(f"Job '{job['id']}' has no 'pdf' block; use eft_calibration.py for EFT jobs.")
 
-    provider = PDFDerivativeProvider(job["pdf"])
+    provider = build_derivative_provider(job)
     cr.run_calibration(cfg, job, samples_mod, args, provider)
 
 

common/derivative_providers.py

@@ -80,3 +80,12 @@ def __init__(self, parameters):
 
     def truth_weight_matrix(self, G, w, observer_names):
         return self._eft.make_weight_matrix(G, observer_names, w)
+
+
+def build_derivative_provider(job):
+    """Build the PDF or EFT derivative provider for a BIT job, dispatching on its config block."""
+    if job.get("pdf"):
+        return PDFDerivativeProvider(job["pdf"])
+    if job.get("eft"):
+        return EFTDerivativeProvider(job["eft"].get("parameters", []))
+    raise RuntimeError(f"Job '{job['id']}' has neither a 'pdf' nor an 'eft' block.")

configs/Eta_unbinned/toys_example.yaml

@@ -0,0 +1,28 @@
+# Example toy spec for fit/ToyGenerator.py, for configs/Eta_unbinned/Eta_unbinned_2016.yaml
+# (the multi-process reference region: SR_2016 classes are TTLep_pow_2016, EtaS_2016,
+# EtaP_2016 -- see the "Multi-process handling" section of
+# user/ricardo/claude/plans/toy-generation-for-pseudo-experiments.md, verification 6-7).
+#
+#   python fit/ToyGenerator.py configs/Eta_unbinned/Eta_unbinned_2016.yaml \
+#       --toySpec configs/Eta_unbinned/toys_example.yaml --toyPoint etaS_up \
+#       --seeds 0-499 --outputDir <some output dir>
+
+source: truth
+split: [c2st_train, c2st_val] # null makes cache-mode-vs-truth-mode pull *widths* comparable (verification 6)
+throw_nuisances: true
+allow_negative_weights: false
+
+points:
+  # nominal truth-mode toy: no modifiers -- the surrogate-correct limit. Every
+  # class gets an implicit ProcessSource at scale_factor=1.0 from its own sample.
+  - name: nominal
+
+  # EtaS_2016 has a rate_shift POI (a model parameter, so this specific case is
+  # also expressible in cache mode via hypothesis: {rate_shift_EtaS: ...}); this
+  # truth-mode version instead probes whether the ICH/ICPH-free unbinned
+  # surrogates absorb a 20% scale mismodelling correctly.
+  - name: etaS_up
+    injection:
+      SR_2016:
+        EtaS_2016:
+          scale_factor: 1.2

configs/unbinned_v6/toys_example.yaml

@@ -0,0 +1,53 @@
+# Example toy spec for fit/ToyGenerator.py, for configs/unbinned_v6/unbinned_2016.yaml.
+#
+#   python fit/ToyGenerator.py configs/unbinned_v6/unbinned_2016.yaml \
+#       --toySpec configs/unbinned_v6/toys_example.yaml --toyPoint c0_0p5_bit \
+#       --seeds 0-499 --outputDir <some output dir>
+#
+# The spec defines the ensemble (source, split, which points); the CLI only
+# selects which point and which seed(s) to generate, and where to write the
+# result. See user/ricardo/claude/plans/toy-generation-for-pseudo-experiments.md
+# for the full design.
+
+source: truth              # or cache
+split: [c2st_train, c2st_val]           # truth mode only; null for the whole dataset
+throw_nuisances: false
+allow_negative_weights: true
+
+points:
+  # nominal truth-mode toy: no modifiers, no hypothesis -- the surrogate-correct
+  # limit, useful as the mean-pull baseline for verification 6.
+  - name: nominal
+
+  # exact PDF weight injection at c0=0.5, generated from generator-level info.
+  - name: c0_0p5
+    injection:
+      SR_2016:
+        TTLep_pow_2016:
+          coefficients: {c0: 0.5}
+
+  # exact JES NP uncertainty injection via alternative sample
+  - name: CMS_scale_j_FlavorPureQuark_up
+    injection:
+      SR_2016:
+        TTLep_pow_2016:
+          sample_name: TTLep_pow_2016_CMS_scale_j_FlavorPureQuark_up
+
+  # exact renormalization scale uncertainty injection via additional weight
+  - name: mu_ren_up
+    injection:
+      SR_2016:
+        TTLep_pow_2016:
+          weight_branches: [scale_ren2p0_fac1p0]  
+
+  # the same point injected through the trained BIT instead (surrogate route),
+  # on the *same* truth-mode split -- the control for c0_0p5 (see the plan's
+  # "two POI injection routes" section). Do not also set 'coefficients' for the
+  # same POI on the same source -- the two routes are mutually exclusive.
+  - name: c0_0p5_bit
+    hypothesis: {c0: 0.5}
+
+# For a per-process rate mismodelling with no rate nuisance to absorb it (a
+# background scaled by e.g. 20%, everything else nominal), see the multi-process
+# example in configs/Eta_unbinned/toys_example.yaml, which has more than one class
+# per region -- this config's SR_2016 region has only TTLep_pow_2016.

configs/unbinned_v7/toys_cache_example.yaml

@@ -0,0 +1,19 @@
+# Cache-mode toy spec for configs/unbinned_v7/unbinned_2018.yaml, used for the
+# first real (non-synthetic) validation of fit/ToyGenerator.py cache-mode
+# generation. See user/ricardo/claude/plans/toy-generation-status.md.
+#
+#   python fit/ToyGenerator.py configs/unbinned_v7/unbinned_2018.yaml \
+#       --toySpec configs/unbinned_v7/toys_cache_example.yaml --toyPoint nominal \
+#       --seeds 0-9 --outputDir <some output dir>
+
+# TTLep_pow_2018/SingleTop_2018/DrellYan_LO_HTbinned_2018/TTSemi_pow_2018 are all
+# NLO-generated samples with ~0.33% negative-weight MC events (routine, not a
+# modelling issue), so allow_negative_weights is needed even at the nominal point.
+source: cache
+throw_nuisances: true
+allow_negative_weights: true
+
+points:
+  - name: nominal
+  - name: c0_0p5
+    hypothesis: {c0: 0.5}