fix(analyze): count prompt characters, not dict length, in count_tokens

garak/analyze/count_tokens.py

@@ -19,6 +19,18 @@
 import garak
 
 
+def _to_text(value) -> str:
+    """Extract text from report values while tolerating legacy shapes."""
+    if isinstance(value, str):
+        return value
+    if isinstance(value, dict):
+        for key in ("text", "content", "response"):
+            nested = value.get(key)
+            if isinstance(nested, (str, dict)):
+                return _to_text(nested)
+    return str(value)
+
+
 def count_tokens(report_path: str) -> None:
     calls = 0
     input_length = 0
@@ -35,22 +47,10 @@ def count_tokens(report_path: str) -> None:
                 generations = r["run.generations"]
                 continue
             if "status" in r and r["status"] == 2:
-                input_length += len(r["prompt"]) * generations
+                input_length += len(_to_text(r["prompt"])) * generations
                 calls += generations
                 outputs = r.get("outputs", [])
                 if isinstance(outputs, list):
-
-                    def _to_text(o):
-                        if isinstance(o, str):
-                            return o
-                        if isinstance(o, dict):
-                            # common keys seen in generator outputs
-                            for k in ("text", "content", "response"):
-                                v = o.get(k)
-                                if isinstance(v, str):
-                                    return v
-                        return str(o)
-
                     output_text = "".join(_to_text(o) for o in outputs)
                 else:
                     output_text = str(outputs)

tests/analyze/test_count_tokens.py

@@ -0,0 +1,52 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+
+from garak.analyze.count_tokens import count_tokens
+
+
+def _write_report(tmp_path, prompt, generations):
+    report_path = tmp_path / "count_tokens.report.jsonl"
+    entries = [
+        {"run.generations": generations},
+        {
+            "status": 2,
+            "prompt": prompt,
+            "outputs": [],
+        },
+    ]
+    report_path.write_text(
+        "".join(json.dumps(entry) + "\n" for entry in entries),
+        encoding="utf-8",
+    )
+    return report_path
+
+
+def test_count_tokens_counts_nested_prompt_text(tmp_path, capsys):
+    prompt_text = "known prompt text"
+    generations = 3
+    report_path = _write_report(
+        tmp_path,
+        {
+            "role": "user",
+            "content": {"text": prompt_text},
+        },
+        generations,
+    )
+
+    count_tokens(str(report_path))
+
+    output = capsys.readouterr().out
+    assert f"Calls: {generations}" in output
+    assert f"Input chars: {len(prompt_text) * generations}" in output
+
+
+def test_count_tokens_accepts_legacy_string_prompt(tmp_path, capsys):
+    prompt_text = "legacy prompt"
+    report_path = _write_report(tmp_path, prompt_text, generations=2)
+
+    count_tokens(str(report_path))
+
+    output = capsys.readouterr().out
+    assert f"Input chars: {len(prompt_text) * 2}" in output