task #657: behavior-direction alignment predictor (bake-off vs base prior)

eval_results/issue_657/lobo_pooled.json

@@ -0,0 +1,93 @@
+{
+  "experiment": "issue657_lobo_pooled",
+  "read": "leave-one-behavior-out (LOBO) higher-N predictor-skill pooled secondary read",
+  "registered_in_plan": "\u00a7324 / \u00a76.5 / \u00a7365 / \u00a7389 (higher-N secondary read)",
+  "lobo": {
+    "in_scope": true,
+    "behaviors": [
+      "em",
+      "refusal",
+      "sycophancy"
+    ],
+    "n_pooled_cells": 399,
+    "n_groups": 69,
+    "per_behavior_n": {
+      "em": 133,
+      "refusal": 133,
+      "sycophancy": 133
+    },
+    "pooling": "predictor-skill (within-behavior source-FE + percentile-rank)",
+    "resampling_unit": "(behavior, bystander) group",
+    "bootstrap_b": 10000,
+    "bootstrap_seed": 657,
+    "grouped_cv": {
+      "held_out_spearman": {
+        "point": 0.020187893947570094,
+        "ci_lo": -0.10646684712377563,
+        "ci_hi": 0.13377646078778863,
+        "n_valid": 10000,
+        "n_bystanders": 69
+      },
+      "delta_r2_align_beyond_prior": {
+        "point": 0.001819847838020694,
+        "ci_lo": -0.017169622318820955,
+        "ci_hi": 0.025398981350982137,
+        "n_valid": 10000,
+        "n_bystanders": 69
+      },
+      "doubly_partialled_rho": {
+        "point": 0.0882006998515973,
+        "ci_lo": -0.016527568512014097,
+        "ci_hi": 0.1711785327676149,
+        "n_valid": 10000,
+        "n_bystanders": 69
+      }
+    },
+    "strict_lobo": {
+      "held_out_spearman": 0.1161574608939571,
+      "delta_r2_align_beyond_prior": 0.016958062844530497,
+      "n_behaviors": 3
+    },
+    "h3_verdict": {
+      "verdict": "TIGHTER-INDETERMINATE",
+      "detail": "pooled \u0394R\u00b2 CI straddles 0 (does not clear from either side)"
+    }
+  },
+  "input_provenance": {
+    "em": {
+      "in_scope": true,
+      "n_joined_cells": 133,
+      "n_resolvable_bystanders": 23,
+      "source_path": "eval_results/issue_657/per_behavior/em.json",
+      "included": true
+    },
+    "marker": {
+      "in_scope": false,
+      "n_joined_cells": 48,
+      "n_resolvable_bystanders": 3,
+      "source_path": "eval_results/issue_657/per_behavior/marker.json",
+      "included": false
+    },
+    "refusal": {
+      "in_scope": true,
+      "n_joined_cells": 133,
+      "n_resolvable_bystanders": 23,
+      "source_path": "eval_results/issue_657/per_behavior/refusal.json",
+      "included": true
+    },
+    "sycophancy": {
+      "in_scope": true,
+      "n_joined_cells": 133,
+      "n_resolvable_bystanders": 23,
+      "source_path": "eval_results/issue_657/per_behavior/sycophancy.json",
+      "included": true
+    }
+  },
+  "reproducibility": {
+    "git_commit": "73648b00b0bcec3428b0a1a05f65bfbd181872a3",
+    "generated_utc": "2026-06-19T02:38:27.728495+00:00",
+    "bootstrap_b": 10000,
+    "bootstrap_seed": 657,
+    "python": "3.11.15"
+  }
+}

figures/issue_657/fig_h1_base_rate.meta.json

@@ -1,6 +1,6 @@
 {
-  "commit": "a03779a3f4",
-  "created": "2026-06-18T23:57:27Z",
+  "commit": "405a0b483f",
+  "created": "2026-06-19T03:58:17Z",
   "figsize": [
     6.5,
     3.8

figures/issue_657/fig_h3_forest.meta.json

@@ -1,6 +1,6 @@
 {
-  "commit": "a03779a3f4",
-  "created": "2026-06-18T23:57:28Z",
+  "commit": "405a0b483f",
+  "created": "2026-06-19T03:58:18Z",
   "figsize": [
     6.8,
     3.8

figures/issue_657/fig_k2_steering.meta.json

@@ -1,8 +1,8 @@
 {
-  "commit": "a03779a3f4",
-  "created": "2026-06-18T23:57:26Z",
+  "commit": "405a0b483f",
+  "created": "2026-06-19T03:58:16Z",
   "figsize": [
-    6.5,
-    3.6
+    6.6,
+    3.8
   ]
 }

figures/issue_657/fig_layer_sweep.meta.json

@@ -0,0 +1,8 @@
+{
+  "commit": "405a0b483f",
+  "created": "2026-06-19T03:58:21Z",
+  "figsize": [
+    6.5,
+    3.8
+  ]
+}

figures/issue_657/fig_lobo_pooled.meta.json

@@ -0,0 +1,8 @@
+{
+  "commit": "405a0b483f",
+  "created": "2026-06-19T03:58:22Z",
+  "figsize": [
+    6.6,
+    3.8
+  ]
+}

figures/issue_657/fig_reliability_band.meta.json

@@ -1,6 +1,6 @@
 {
-  "commit": "a03779a3f4",
-  "created": "2026-06-18T23:57:29Z",
+  "commit": "405a0b483f",
+  "created": "2026-06-19T03:58:20Z",
   "figsize": [
     6.5,
     3.8

scripts/issue623_extract_sycophancy_vector.py

@@ -170,25 +170,65 @@ def _strip_json_fence(text: str) -> str:
 def generate_artifacts(
     generator_model: str,
     n_questions: int,
+    trait_name: str = SYCOPHANCY_TRAIT_NAME,
+    trait_description: str = SYCOPHANCY_TRAIT_DESCRIPTION,
 ) -> dict:
     """Call the Claude generator to produce 5 pos/neg pairs + N questions + eval prompt.
 
     Mirrors the paper's single-prompt generation. ``n_questions`` lets the smoke
     request a tiny set; the production default is 40 (split 20 extraction/20 eval).
+
+    ``trait_name`` / ``trait_description`` default to the #623 sycophancy constants
+    (verbatim #623 behavior when omitted); #657 overrides them via CLI to extract
+    refusal / marker / EM directions with the SAME paper recipe.
     """
+    import time
+
     import anthropic
 
-    client = anthropic.Anthropic()
+    # R1: raise the SDK retry budget so the built-in exponential backoff rides
+    # through a transient org-wide 429 burst (the launch-attempt-2 crash class:
+    # the default max_retries=2 exhausted on a 1M-output-tokens/min cap burst).
+    # timeout=600s gives each individual request room.
+    client = anthropic.Anthropic(max_retries=10, timeout=600.0)
     prompt = _ARTIFACT_GEN_PROMPT.format(
-        trait=SYCOPHANCY_TRAIT_NAME,
-        trait_description=SYCOPHANCY_TRAIT_DESCRIPTION,
-    )
-    resp = client.messages.create(
-        model=generator_model,
-        max_tokens=8192,
-        temperature=1.0,
-        messages=[{"role": "user", "content": prompt}],
+        trait=trait_name,
+        trait_description=trait_description,
     )
+    # R3: belt-and-suspenders outer retry with MINUTE-scale sleeps for a
+    # sustained 429 burst wider than the SDK's ~10s-capped backoff window. The
+    # SDK (R1) handles the common short bursts; this catches the rare 60s+ wall.
+    last_err: anthropic.RateLimitError | None = None
+    resp = None
+    for attempt in range(3):
+        try:
+            resp = client.messages.create(
+                model=generator_model,
+                # R2: 4096 is ample for the artifact response (5 instruction
+                # pairs + N questions + eval prompt run ~2000-3000 tokens) while
+                # halving the per-call OUTPUT-TOKEN allocation against the
+                # org-wide 1M/min cap, so each request is likelier to fit the
+                # remaining per-minute budget.
+                max_tokens=4096,
+                temperature=1.0,
+                messages=[{"role": "user", "content": prompt}],
+            )
+            break
+        except anthropic.RateLimitError as e:
+            last_err = e
+            wait = 60 * (attempt + 1)  # 60s, 120s, 180s
+            print(
+                f"[generate_artifacts] 429 rate-limit on attempt {attempt + 1}/3; "
+                f"sleeping {wait}s before retry ...",
+                flush=True,
+            )
+            time.sleep(wait)
+    if resp is None:
+        raise (
+            last_err
+            if last_err is not None
+            else RuntimeError("generate_artifacts: messages.create produced no response")
+        )
     raw = resp.content[0].text if resp.content else ""
     data = json.loads(_strip_json_fence(raw))
 
@@ -668,6 +708,24 @@ def main() -> None:
     )
     parser.add_argument("--model", default=BASE_MODEL)
     parser.add_argument("--gpu-id", type=int, default=0)
+    parser.add_argument(
+        "--trait-name",
+        default=SYCOPHANCY_TRAIT_NAME,
+        help=(
+            "Trait name fed to the paper artifact-generation recipe (default: "
+            "sycophancy, reproducing #623 verbatim). #657 overrides with "
+            "refusal / marker / em to extract those behavior directions."
+        ),
+    )
+    parser.add_argument(
+        "--trait-description",
+        default=SYCOPHANCY_TRAIT_DESCRIPTION,
+        help=(
+            "Verbatim trait description fed to the artifact generator (default: "
+            "the #623 sycophancy description). Required override when --trait-name "
+            "is not sycophancy."
+        ),
+    )
     parser.add_argument("--layers", type=int, nargs="+", default=list(DEFAULT_LAYERS))
     parser.add_argument(
         "--n-questions",
@@ -706,6 +764,19 @@ def main() -> None:
 
     load_dotenv()
 
+    # Guard against an override that names a non-default trait but leaves the
+    # default sycophancy description in place (a silent recipe mismatch — the
+    # generated artifacts would describe sycophancy under a refusal label).
+    if args.trait_name != SYCOPHANCY_TRAIT_NAME and (
+        args.trait_description == SYCOPHANCY_TRAIT_DESCRIPTION
+    ):
+        raise ValueError(
+            f"--trait-name={args.trait_name!r} requires a matching "
+            f"--trait-description (the default sycophancy description is still "
+            f"in place — that would generate sycophancy artifacts under a "
+            f"{args.trait_name!r} label)."
+        )
+
     out_dir = (
         PROJECT_ROOT / args.output_dir
         if not Path(args.output_dir).is_absolute()
@@ -720,7 +791,12 @@ def main() -> None:
         artifacts = json.loads(artifacts_path.read_text())
         print(f"[phase=sycophancy_trait] loaded cached artifacts from {artifacts_path}", flush=True)
     else:
-        artifacts = generate_artifacts(args.generator_model, args.n_questions)
+        artifacts = generate_artifacts(
+            args.generator_model,
+            args.n_questions,
+            trait_name=args.trait_name,
+            trait_description=args.trait_description,
+        )
         artifacts_path.write_text(json.dumps(artifacts, indent=2))
         print(f"[phase=sycophancy_trait] generated artifacts -> {artifacts_path}", flush=True)
 
@@ -805,7 +881,8 @@ def main() -> None:
         json.dumps(
             {
                 "model": args.model,
-                "trait": SYCOPHANCY_TRAIT_NAME,
+                "trait": args.trait_name,
+                "trait_description": args.trait_description,
                 "layers": layers,
                 "n_instruction_pairs": len(instructions),
                 "n_extraction_questions": len(extraction_questions),
@@ -894,11 +971,11 @@ def main() -> None:
         flush=True,
     )
     if not probe["k2_pass"]:
-        # K2 HALT — the vector does not causally increase sycophancy at any layer.
+        # K2 HALT — the vector does not causally increase the trait at any layer.
         raise SystemExit(
-            "K2 HALT: sycophancy vector produced NO positive steering effect at any layer; "
-            "the trait vector is not measuring sycophancy. Diagnose extraction "
-            "(trait-description / judge-threshold) before reading rho."
+            f"K2 HALT: {args.trait_name} vector produced NO positive steering effect at "
+            f"any layer; the trait vector is not measuring {args.trait_name}. Diagnose "
+            "extraction (trait-description / judge-threshold) before reading rho."
         )