From 3a1046c8cd6db4500236d684c57e11437a78da60 Mon Sep 17 00:00:00 2001 From: Luca Belli <129434630+sator-labs@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:20:14 -0700 Subject: [PATCH] feat: one first_speaker setting replaces persona_speaks_first The flag said who speaks first from the chatbot's side (--provider-speaks-first) and the config field from the persona's (persona_speaks_first), so one had to be flipped to match the other. Both become first_speaker, "user" or "chatbot", in the CLI's u/c vocabulary: --first-speaker on the CLI (default user), first_speaker in a config file. A config with the old field stops with an error naming the new one. configs/ is updated. The generation domain still takes a bool; run_for_user_models converts at that one boundary. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + configs/hfo_claude_opus_5.json | 2 +- configs/hfo_gemini_3_1_pro.json | 2 +- configs/hfo_gpt_5_6_sol.json | 2 +- configs/recommended-SI.json | 2 +- docs/architecture.md | 2 +- docs/cli.md | 6 ++-- docs/vera-cli-use-cases.md | 4 +-- src/vera_mh/generate/run.py | 3 +- src/vera_mh/utils/config_schema.py | 15 +++++++--- src/vera_mh/vera_cli/generate.py | 27 +++++++++++------ tests/integration/test_vera_pipeline_e2e.py | 2 +- tests/safe_changes/replay.py | 2 +- tests/unit/test_vera_cli.py | 33 ++++++++++++++++++--- tests/unit/test_vera_pipeline.py | 2 +- tests/unit/utils/test_run_record.py | 2 +- 16 files changed, 75 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67a76703..80cc96ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### Breaking / migration +- **`first_speaker` replaces `persona_speaks_first` and `--provider-speaks-first`**—Who opens a conversation is now one setting with one name: `"first_speaker": "user"` or `"chatbot"` in a config file, and `--first-speaker {user,chatbot}` on the CLI (default `user`). Update configs: `"persona_speaks_first": true` becomes `"first_speaker": "user"`, and `false` becomes `"chatbot"`. A config with the old field stops with an error that says so. The files in `configs/` are updated. - **Code moved under `src/vera_mh/`**—`generate`, `judge`, `score`, `utils`, `llm_clients`, and `vera_cli` are now subpackages of one package, so imports gain a prefix: `from judge.runner import X` becomes `from vera_mh.judge.runner import X`, and the same goes for `patch()` targets. `python -m score.compare` is now `python -m vera_mh.score.compare`. `uv sync` installs the package and a `vera` command (`vera = "vera_mh.cli:main"`), which replaces `python vera.py`: the root `vera.py` is gone, so run `uv run vera `. Design: [docs/design/src-layout.md](docs/design/src-layout.md) (#285). - **Legacy scripts removed**—`legacy/` is gone (`legacy.generate`, `legacy.judge`, `legacy.score`, `legacy.run_pipeline`, `legacy.pool_vera_scores`, `run_recommended_vera_pipeline.sh`), as announced in v2.0.4, along with `docs/legacy-scripts.md`, the tests that drove the scripts, and the `data/SI/rubric_manifest.json` and `data/HFO/rubric_manifest.json` symlinks only they read. Use the `vera` commands; the flag mapping is in [v2.0.4's `docs/legacy-scripts.md`](https://github.com/SpringCare/VERA-MH/blob/v2.0.4/docs/legacy-scripts.md). `distribute_files.py`, `scripts/run_combinations.sh`, and `scripts/score_publication_evaluations.sh` are removed too. To run several chatbots, loop over `vera generate --chatbot`; `--user` already takes several models. diff --git a/configs/hfo_claude_opus_5.json b/configs/hfo_claude_opus_5.json index 7be2c468..3bfea093 100644 --- a/configs/hfo_claude_opus_5.json +++ b/configs/hfo_claude_opus_5.json @@ -9,7 +9,7 @@ "output": "output", "max_concurrent": 10, "max_total_words": null, - "persona_speaks_first": true, + "first_speaker": "user", "user_language": "English", "sessions": null }, diff --git a/configs/hfo_gemini_3_1_pro.json b/configs/hfo_gemini_3_1_pro.json index eb81a841..f488fe56 100644 --- a/configs/hfo_gemini_3_1_pro.json +++ b/configs/hfo_gemini_3_1_pro.json @@ -9,7 +9,7 @@ "output": "output", "max_concurrent": 10, "max_total_words": null, - "persona_speaks_first": true, + "first_speaker": "user", "user_language": "English", "sessions": null }, diff --git a/configs/hfo_gpt_5_6_sol.json b/configs/hfo_gpt_5_6_sol.json index c1c54010..9e4ebec8 100644 --- a/configs/hfo_gpt_5_6_sol.json +++ b/configs/hfo_gpt_5_6_sol.json @@ -9,7 +9,7 @@ "output": "output", "max_concurrent": 10, "max_total_words": null, - "persona_speaks_first": true, + "first_speaker": "user", "user_language": "English", "sessions": null }, diff --git a/configs/recommended-SI.json b/configs/recommended-SI.json index 8fa571af..a7964359 100644 --- a/configs/recommended-SI.json +++ b/configs/recommended-SI.json @@ -10,7 +10,7 @@ "output": "output", "max_concurrent": 10, "max_total_words": null, - "persona_speaks_first": true, + "first_speaker": "user", "user_language": "English", "sessions": null }, diff --git a/docs/architecture.md b/docs/architecture.md index ef7afcad..5d1bfa4e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -172,7 +172,7 @@ The two input forms use those defaults differently, split by whether a field changes what a run produces: - **Fields that change the results** (for `generate`: `turns`, `user_language`, - `persona_speaks_first`, `sessions`, `max_total_words`; for `score` and `pool`: + `first_speaker`, `sessions`, `max_total_words`; for `score` and `pool`: `personas`, `skip_risk_analysis`) are required in a config file, using `null` where the schema allows no limit or no session list. The CLI defaults them. - **Fields that change only where output goes or how fast a run goes** diff --git a/docs/cli.md b/docs/cli.md index a651e9d5..b6252f4d 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -55,7 +55,7 @@ A few **invocation controls** describe how to run, not what the run is, so they | `--print` | Print the fully resolved invocation (as a `VERA_RUN_CONFIG=...` command) and exit without running anything | | `--into ` | Continue an existing run (`generate`, `judge`); see [below](#continuing-an-interrupted-run---into) | -A config file must state every field that changes what a run produces: the models, the personas, conversations, or rubric (or a `target` that supplies them), and for `generate` also `turns`, `user_language`, `persona_speaks_first`, `sessions`, and `max_total_words` (`null` where no limit is intended). For `score` and `pool`, `personas` and `skip_risk_analysis` are required, because they decide whether the risk breakdown exists. Fields that only change where output goes or how fast a run goes (`output`, `max_concurrent`, `per_judge`) may be left out and get the same default as the flag. Unknown fields are rejected. +A config file must state every field that changes what a run produces: the models, the personas, conversations, or rubric (or a `target` that supplies them), and for `generate` also `turns`, `user_language`, `first_speaker`, `sessions`, and `max_total_words` (`null` where no limit is intended). For `score` and `pool`, `personas` and `skip_risk_analysis` are required, because they decide whether the risk breakdown exists. Fields that only change where output goes or how fast a run goes (`output`, `max_concurrent`, `per_judge`) may be left out and get the same default as the flag. Unknown fields are rejected. Every command prints its resolved config when it starts, so the log of a run shows exactly what it did. @@ -78,7 +78,7 @@ Paths inside a config resolve against the repository root, not the working direc "output": "output", "max_concurrent": 10, "max_total_words": null, - "persona_speaks_first": true, + "first_speaker": "user", "user_language": "English", "sessions": null }, @@ -117,7 +117,7 @@ uv run vera generate \ | `-o`, `--output` | Parent directory for new run folders | `output` | | `--max-concurrent` | Maximum concurrent conversations; set this if the chatbot times out or rate-limits | unlimited | | `--max-total-words` | Cap on total response words per conversation | unlimited | -| `--provider-speaks-first` | Chatbot speaks first | persona speaks first | +| `--first-speaker {user,chatbot}` | Who opens the conversation: the user-side LLM or the chatbot | `user` | | `--user-language` | Language the user-side LLM writes in, e.g. `"Mexican Spanish"`; see [user-language.md](user-language.md) | English | | `--sessions` | Comma-separated session types run in order per persona (e.g. `intake,coaching`) | one session | | `--user-params` | `k=v,...` provider parameters for every `-u` model | none | diff --git a/docs/vera-cli-use-cases.md b/docs/vera-cli-use-cases.md index 3e8b629c..22c525fd 100644 --- a/docs/vera-cli-use-cases.md +++ b/docs/vera-cli-use-cases.md @@ -312,7 +312,7 @@ Top-level `generation` and `judging` blocks are **completely orthogonal** — mo "output": "output", "max_concurrent": null, "max_total_words": null, - "persona_speaks_first": true, + "first_speaker": "user", "user_language": "English", "sessions": null, "persona_context_template": "data/SI/persona_context_template.txt" @@ -340,7 +340,7 @@ judging prompts. In the persisted canonical config, those entries contain the resolved concrete paths rather than a manifest reference. In config mode, the generation fields that change the conversations (`turns`, -`user_language`, `persona_speaks_first`, `sessions`, `max_total_words`) are +`user_language`, `first_speaker`, `sessions`, `max_total_words`) are required, including explicit `null` where no limit or session list is intended. `output` and `max_concurrent` only change where and how fast a run goes, so a config may leave them to the CLI defaults. diff --git a/src/vera_mh/generate/run.py b/src/vera_mh/generate/run.py index f9f76017..91ccd843 100644 --- a/src/vera_mh/generate/run.py +++ b/src/vera_mh/generate/run.py @@ -217,7 +217,8 @@ async def run_for_user_models( max_concurrent=generation.max_concurrent, max_total_words=generation.max_total_words, max_personas=max_personas, - persona_speaks_first=generation.persona_speaks_first, + # The domain still takes a bool; `first_speaker` is the public name. + persona_speaks_first=generation.first_speaker == "user", session_types=generation.sessions, resume=into is not None, persona_context_template_path=generation.persona_context_template, diff --git a/src/vera_mh/utils/config_schema.py b/src/vera_mh/utils/config_schema.py index 8c9c2cf4..120e1c20 100644 --- a/src/vera_mh/utils/config_schema.py +++ b/src/vera_mh/utils/config_schema.py @@ -160,6 +160,11 @@ def to_dict(self) -> dict[str, Any]: return dataclasses.asdict(self) +# Who opens the conversation, in the CLI's u/c vocabulary: the user-side LLM +# playing the persona, or the chatbot under test. +FIRST_SPEAKERS = ("user", "chatbot") + + @dataclasses.dataclass(frozen=True) class GenerationConfig: """What run to perform — the run-defining half of a `RunConfig`. @@ -181,7 +186,7 @@ class GenerationConfig: output: str max_concurrent: int | None max_total_words: int | None - persona_speaks_first: bool + first_speaker: str sessions: list[str] | None persona_context_template: str user_language: str @@ -217,8 +222,10 @@ def __post_init__(self) -> None: ) if self.max_total_words < 1: raise ValueError("generation.max_total_words must be at least 1") - if not isinstance(self.persona_speaks_first, bool): - raise ValueError("generation.persona_speaks_first must be a boolean") + if self.first_speaker not in FIRST_SPEAKERS: + raise ValueError( + "generation.first_speaker must be one of: " + ", ".join(FIRST_SPEAKERS) + ) if self.sessions is not None: if not self.sessions: raise ValueError("generation.sessions cannot be empty") @@ -242,7 +249,7 @@ def to_dict(self) -> dict[str, Any]: "output": self.output, "max_concurrent": self.max_concurrent, "max_total_words": self.max_total_words, - "persona_speaks_first": self.persona_speaks_first, + "first_speaker": self.first_speaker, "sessions": list(self.sessions) if self.sessions is not None else None, "persona_context_template": self.persona_context_template, "user_language": self.user_language, diff --git a/src/vera_mh/vera_cli/generate.py b/src/vera_mh/vera_cli/generate.py index 10936a42..6d1fd73d 100644 --- a/src/vera_mh/vera_cli/generate.py +++ b/src/vera_mh/vera_cli/generate.py @@ -29,7 +29,12 @@ from typing import Any from vera_mh.generate import run_for_user_models -from vera_mh.utils.config_schema import GenerationConfig, InvocationConfig, RunConfig +from vera_mh.utils.config_schema import ( + FIRST_SPEAKERS, + GenerationConfig, + InvocationConfig, + RunConfig, +) from vera_mh.utils.run_record import build_run_record from vera_mh.utils.utils import parse_key_value_list @@ -77,7 +82,7 @@ "output": "output", "max_concurrent": None, "max_total_words": None, - "provider_speaks_first": False, + "first_speaker": "user", "user_language": "English", "sessions": None, } @@ -88,7 +93,7 @@ RESULT_FIELDS = ( "turns", "user_language", - "persona_speaks_first", + "first_speaker", "sessions", "max_total_words", ) @@ -198,9 +203,7 @@ def _from_cli( ), max_concurrent=flag_value(args, "max_concurrent", defaults=DEFAULTS), max_total_words=flag_value(args, "max_total_words", defaults=DEFAULTS), - persona_speaks_first=not flag_value( - args, "provider_speaks_first", defaults=DEFAULTS - ), + first_speaker=flag_value(args, "first_speaker", defaults=DEFAULTS), sessions=_sessions(flag_value(args, "sessions", defaults=DEFAULTS)), user_language=flag_value(args, "user_language", defaults=DEFAULTS), ), @@ -220,6 +223,11 @@ def _from_config( existing config files. """ generation = config_section(config, "generation", command="generate") + if "persona_speaks_first" in generation: + raise ConfigError( + 'generation.persona_speaks_first was renamed: use "first_speaker": ' + '"user" in place of true, or "chatbot" in place of false' + ) chatbot = model_from_config( required(generation, "chatbot", section="generation config"), field="generation.chatbot", @@ -422,10 +430,11 @@ def register(subparsers: argparse._SubParsersAction) -> None: help="Maximum response words per conversation (default: unlimited)", ) parser.add_argument( - "--provider-speaks-first", - action="store_true", + "--first-speaker", + choices=FIRST_SPEAKERS, default=argparse.SUPPRESS, - help="Have the chatbot speak first (default: persona speaks first)", + help="Who opens the conversation: the user-side LLM or the chatbot " + "(default: user)", ) parser.add_argument( "--user-language", diff --git a/tests/integration/test_vera_pipeline_e2e.py b/tests/integration/test_vera_pipeline_e2e.py index 9feb8d16..f22ebd59 100644 --- a/tests/integration/test_vera_pipeline_e2e.py +++ b/tests/integration/test_vera_pipeline_e2e.py @@ -53,7 +53,7 @@ def _config( "output": str(output), "max_concurrent": 2, "max_total_words": None, - "persona_speaks_first": True, + "first_speaker": "user", "user_language": "English", "sessions": None, }, diff --git a/tests/safe_changes/replay.py b/tests/safe_changes/replay.py index 8e052c9d..52d885cc 100644 --- a/tests/safe_changes/replay.py +++ b/tests/safe_changes/replay.py @@ -229,7 +229,7 @@ def pipeline_config(output: Path) -> dict[str, Any]: "output": str(output), "max_concurrent": 1, "max_total_words": None, - "persona_speaks_first": True, + "first_speaker": "user", "user_language": "English", "sessions": None, }, diff --git a/tests/unit/test_vera_cli.py b/tests/unit/test_vera_cli.py index 00838a02..91bb6108 100644 --- a/tests/unit/test_vera_cli.py +++ b/tests/unit/test_vera_cli.py @@ -38,7 +38,7 @@ def _generation_config(**overrides: object) -> dict: "output": "output", "max_concurrent": None, "max_total_words": None, - "persona_speaks_first": True, + "first_speaker": "user", "user_language": "English", "sessions": None, "persona_context_template": "data/SI/persona_context_template.txt", @@ -242,7 +242,8 @@ def test_generation_delegates_every_value_to_generate_main() -> None: "2", "--max-total-words", "100", - "--provider-speaks-first", + "--first-speaker", + "chatbot", "--sessions", "intake,coaching", "--sample", @@ -327,7 +328,7 @@ def test_config_without_output_or_speed_fields_gets_the_cli_defaults( "user": [{"name": "claude-sonnet-5", "repeats": 1}], "turns": 30, "user_language": "English", - "persona_speaks_first": True, + "first_speaker": "user", "sessions": None, "max_total_words": None, }, @@ -396,7 +397,7 @@ def test_config_target_rejects_explicit_persona_component(tmp_path: Path) -> Non ["--output", "elsewhere"], ["--max-concurrent", "2"], ["--max-total-words", "10"], - ["--provider-speaks-first"], + ["--first-speaker", "chatbot"], ["--sessions", "intake"], ], ) @@ -672,3 +673,27 @@ def test_configs_without_into_still_resolve(tmp_path: Path) -> None: config = _write_config(tmp_path, _generation_config()) args = vera.build_parser().parse_args(["generate", "--config", str(config)]) assert generate.resolve_configs(args)[0].invocation.into is None + + +def test_config_with_the_old_speaking_order_field_says_it_was_renamed( + tmp_path: Path, +) -> None: + config_data = _generation_config() + del config_data["generation"]["first_speaker"] + config_data["generation"]["persona_speaks_first"] = True + config = _write_config(tmp_path, config_data) + parser = vera.build_parser() + + with pytest.raises(cli_config.ConfigError, match="renamed.*first_speaker"): + generate.resolve_configs( + parser.parse_args(["generate", "--config", str(config)]) + ) + + +def test_config_rejects_an_unknown_first_speaker(tmp_path: Path) -> None: + config = _write_config(tmp_path, _generation_config(first_speaker="persona")) + + with pytest.raises(SystemExit) as error: + vera.main(["generate", "--config", str(config)]) + + assert error.value.code == 2 diff --git a/tests/unit/test_vera_pipeline.py b/tests/unit/test_vera_pipeline.py index de5164a3..f65fe618 100644 --- a/tests/unit/test_vera_pipeline.py +++ b/tests/unit/test_vera_pipeline.py @@ -402,7 +402,7 @@ def test_config_without_behavior_fields_gets_the_cli_defaults( "user": [{"name": "gpt-5.2", "repeats": 1}], "turns": 30, "user_language": "English", - "persona_speaks_first": True, + "first_speaker": "user", "sessions": None, "max_total_words": None, }, diff --git a/tests/unit/utils/test_run_record.py b/tests/unit/utils/test_run_record.py index bc417c3c..27dff4ad 100644 --- a/tests/unit/utils/test_run_record.py +++ b/tests/unit/utils/test_run_record.py @@ -54,7 +54,7 @@ def _run_config(tmp_path: Path, **generation_overrides: object) -> RunConfig: "output": str(tmp_path / "output"), "max_concurrent": None, "max_total_words": None, - "persona_speaks_first": True, + "first_speaker": "user", "sessions": None, "persona_context_template": str(template), "user_language": "English",