thinking-probes: read the reasoning field under both names

.gitignore

@@ -3,3 +3,7 @@
 models/qwopus-coder.md
 cards/qwopus-coder.html
 cards/img/qwopus-coder.png
+
+# python build artifacts
+__pycache__/
+*.pyc

scripts/thinking-probes/README.md

@@ -65,15 +65,46 @@ cannot make:
 
 | signal | verdict |
 |---|---|
-| `reasoning_content` non-empty | `REASONED_PARSED` |
+| `reasoning_content` or `reasoning` non-empty | `REASONED_PARSED` |
 | `<think>` markers in content | `REASONED_UNPARSED` |
 | neither | `NO_REASONING` |
 
 `REASONED_UNPARSED` is the important one. Any occurrence means a stack reading
-only `reasoning_content` would report that arm as not firing when it did, so a
+only the reasoning field would report that arm as not firing when it did, so a
 published firing rate from such a stack is measuring the parser as much as the
 model.
 
+**Servers disagree on the field name.** llama.cpp populates `reasoning_content`;
+vLLM exposes `reasoning`. These tools read both, because reading only the first
+makes every arm on a vLLM lane look like `NO_REASONING`, which is this tool
+inventing the very answer it exists to measure. Reported by @Blackwellboy in
+offlabel#8 after a run where all 42 rows logged zero reasoning.
+
+If a run reports `NO_REASONING` across **every** arm including `true`, do not
+conclude the model does not reason. Check the field name first: a stack that
+exposes thinking under a third name would still read as silent here.
+
+**Both sampling tools now say this themselves.** A guard rail only works on
+someone who reads it at the moment they need it, and this condition is
+machine-checkable, so the tools print a warning at summary time rather than
+leaving it to the reader:
+
+```
+  WARNING: every arm returned NO_REASONING, including true where thinking is
+  explicitly enabled. That is either a genuinely non-reasoning model or a
+  reasoning field this tool does not read.
+```
+
+That makes it a positive control rather than advice, which matters more here
+than usual: the thing being measured is absence, and a wrong key produces
+absence too.
+
+It deliberately stays quiet when it would be guessing. If the enabled arm
+errored rather than returning nothing, or every request failed, nothing was
+measured and no field-name conclusion is available. `persona_ab.py` has no
+`true` arm at all, so there it keys on the `absent` cells, since zero in the
+`false` control cells is the expected result rather than a symptom.
+
 ## `persona_ab.py`: does a system prompt change it
 
 Crosses a persona system prompt against the kwarg, four cells:

scripts/thinking-probes/persona_ab.py

@@ -59,6 +59,32 @@
 THINK_MARKER = re.compile(r"</?think>", re.I)
 
 
+def field_name_warning(rows):
+    """Positive control for the one failure this tool cannot otherwise see.
+
+    What is being measured is absence, and a reasoning field this tool does
+    not read produces absence too. Note this grid has no explicit `true`
+    arm: the `false` cells are the structural control, where zero is the
+    expected result, so the signal is zero in the `absent` cells as well.
+
+    Returns a warning string, or None when the check does not apply.
+    """
+    scored = [r for r in rows if r["reasoned"] is not None]
+    if not scored:
+        return None                 # nothing was measured; not this failure
+    if any(r["reasoned"] for r in scored):
+        return None                 # something reasoned; the reader works
+    available = [k for (_, k) in CELLS if k != "false"]
+    if not any(r["kwarg"] in available for r in scored):
+        return None                 # only control cells scored; says nothing
+    return ("  WARNING: no cell reasoned, including "
+            f"{'/'.join(sorted(set(available)))} where thinking is available\n"
+            "  rather than structurally suppressed. That is either a genuinely\n"
+            "  non-reasoning model or a reasoning field this tool does not read.\n"
+            "  Check the field name against the server's response before\n"
+            "  believing these numbers.")
+
+
 def post(url, body, timeout):
     req = urllib.request.Request(
         url, data=json.dumps(body).encode(),
@@ -104,7 +130,13 @@ def main():
             d = post(args.base + "/v1/chat/completions", body, args.timeout)
             ch = d["choices"][0]
             content = ch["message"].get("content") or ""
-            reasoning = ch["message"].get("reasoning_content") or ""
+            # Servers disagree on the field name: llama.cpp populates
+            # reasoning_content, vLLM exposes reasoning. Reading only the first
+            # makes every arm look like NO_REASONING on a vLLM lane, which is
+            # this tool inventing the answer it exists to measure
+            # (offlabel#8, @Blackwellboy).
+            reasoning = (ch["message"].get("reasoning_content")
+                         or ch["message"].get("reasoning") or "")
             finish = ch.get("finish_reason")
             reasoned = bool(reasoning.strip()) or bool(THINK_MARKER.search(content))
         except Exception as exc:                                # noqa: BLE001
@@ -129,6 +161,11 @@ def main():
         mr = sum(r["reasoning_chars"] for r in rs) / max(len(rs), 1)
         print(f"  {persona:<11} {kwarg:<7} {n:>4}/{len(rs):<5} {mr:>18.0f}")
 
+    warning = field_name_warning(rows)
+    if warning:
+        print()
+        print(warning)
+
     print("\n=== persona effect WITHIN the absent arm, per prompt shape ===")
     for pname in PROMPTS:
         line = f"  {pname:<15}"

scripts/thinking-probes/thinking_ab.py

@@ -75,6 +75,33 @@ def rendered_tail(base, arm, timeout):
     return p[-28:], p.rstrip().endswith("<think>")
 
 
+def field_name_warning(rows):
+    """Positive control for the one failure this tool cannot otherwise see.
+
+    What is being measured is absence, and a reasoning field this tool does
+    not read produces absence too. The two are indistinguishable in the
+    numbers, so when nothing reasoned anywhere, including the arm that
+    explicitly enables thinking, say so rather than let the zeros stand.
+
+    Returns a warning string, or None when the check does not apply.
+    """
+    scored = [r for r in rows if r["verdict"] != "ERROR"]
+    if not scored:
+        return None                 # nothing was measured; not this failure
+    if any(r["verdict"].startswith("REASONED_") for r in scored):
+        return None                 # something reasoned; the reader works
+    # Derived, not hardcoded, so it stays correct if ARMS changes.
+    enabled = [a for a, kw in ARMS.items()
+               if kw and kw.get("enable_thinking") is True]
+    if not any(r["arm"] in enabled for r in scored):
+        return None                 # the enabled arm produced no scored sample
+    return ("  WARNING: every arm returned NO_REASONING, including "
+            f"{'/'.join(enabled)} where thinking is explicitly enabled.\n"
+            "  That is either a genuinely non-reasoning model or a reasoning\n"
+            "  field this tool does not read. Check the field name against the\n"
+            "  server's response before believing these numbers.")
+
+
 def verdict(content, reasoning):
     if reasoning.strip():
         return "REASONED_PARSED"
@@ -128,7 +155,13 @@ def main():
             d = post(args.base + "/v1/chat/completions", body, args.timeout)
             ch = d["choices"][0]
             content = ch["message"].get("content") or ""
-            reasoning = ch["message"].get("reasoning_content") or ""
+            # Servers disagree on the field name: llama.cpp populates
+            # reasoning_content, vLLM exposes reasoning. Reading only the first
+            # makes every arm look like NO_REASONING on a vLLM lane, which is
+            # this tool inventing the answer it exists to measure
+            # (offlabel#8, @Blackwellboy).
+            reasoning = (ch["message"].get("reasoning_content")
+                         or ch["message"].get("reasoning") or "")
             finish = ch.get("finish_reason")
             v = verdict(content, reasoning)
         except Exception as exc:                                # noqa: BLE001
@@ -153,6 +186,11 @@ def main():
         print(f"  {arm:<8} {c('REASONED_PARSED'):>7} {c('REASONED_UNPARSED'):>9} "
               f"{c('NO_REASONING'):>6} {c('ERROR'):>4}  {mr:>10.0f}")
 
+    warning = field_name_warning(rows)
+    if warning:
+        print()
+        print(warning)
+
     print("\n=== per prompt shape (is the effect prompt-specific?) ===")
     for pname in PROMPTS:
         line = f"  {pname:<15}"