diff --git a/CHANGELOG.md b/CHANGELOG.md index 32c5deba..420d1212 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - **`first_speaker` replaces `persona_speaks_first` and `--provider-speaks-first`**—Who opens a conversation is now one setting with one name: `"first_speaker": "user"` or `"chatbot"` in a config file, and `--first-speaker {user,chatbot}` on the CLI (default `user`). Update configs: `"persona_speaks_first": true` becomes `"first_speaker": "user"`, and `false` becomes `"chatbot"`. A config with the old field stops with an error that says so. The files in `configs/` are updated. - **Code moved under `src/vera_mh/`**—`generate`, `judge`, `score`, `utils`, `llm_clients`, and `vera_cli` are now subpackages of one package, so imports gain a prefix: `from judge.runner import X` becomes `from vera_mh.judge.runner import X`, and the same goes for `patch()` targets. `python -m score.compare` is now `python -m vera_mh.score.compare`. `uv sync` installs the package and a `vera` command (`vera = "vera_mh.cli:main"`), which replaces `python vera.py`: the root `vera.py` is gone, so run `uv run vera `. Design: [docs/design/src-layout.md](docs/design/src-layout.md) (#285). - **Legacy scripts removed**—`legacy/` is gone (`legacy.generate`, `legacy.judge`, `legacy.score`, `legacy.run_pipeline`, `legacy.pool_vera_scores`, `run_recommended_vera_pipeline.sh`), as announced in v2.0.4, along with `docs/legacy-scripts.md`, the tests that drove the scripts, and the `data/SI/rubric_manifest.json` and `data/HFO/rubric_manifest.json` symlinks only they read. Use the `vera` commands; the flag mapping is in [v2.0.4's `docs/legacy-scripts.md`](https://github.com/SpringCare/VERA-MH/blob/v2.0.4/docs/legacy-scripts.md). `distribute_files.py`, `scripts/run_combinations.sh`, and `scripts/score_publication_evaluations.sh` are removed too. To run several chatbots, loop over `vera generate --chatbot`; `--user` already takes several models. +- **Legacy leftovers removed from `judge/`**—`judge_single_conversation` (also exported from `vera_mh.judge`), `build_single_conversation_judge_run_key`, and `get_judge_logs_root` are gone; nothing called them once the legacy scripts left. `build_judge_task_log_path` now requires `output_folder` and drops `run_key` and `logs_root`: judge logs always go to `/logs/`, and `VERA_JUDGE_LOGS_ROOT` is no longer read. To judge one conversation, put it in a folder of its own and pass that to `vera judge --conversations`. ### Runtime, CLI, and pipeline diff --git a/src/vera_mh/generate/runner.py b/src/vera_mh/generate/runner.py index c62f04db..bb942e4f 100644 --- a/src/vera_mh/generate/runner.py +++ b/src/vera_mh/generate/runner.py @@ -262,36 +262,6 @@ def _job_result( out["skip_reason"] = skip_reason return out - def _create_conversation_jobs( - self, persona_names: Optional[List[str]] = None - ) -> List[Tuple[dict, int, int, int]]: - """Create job tuples for all persona/run combinations.""" - personas = load_prompts_from_csv( - persona_names, - prompt_path=self.persona_prompt_path, - persona_context_template_path=self.persona_context_template_path, - user_language=self.user_language, - max_personas=self.max_personas, - ) - jobs: List[Tuple[dict, int, int, int]] = [] - conversation_index = 1 - for persona in personas: - for run in range(1, self.runs_per_prompt + 1): - jobs.append( - ( - { - "prompt": persona["prompt"], - "name": persona["Name"], - "run": run, - }, - self.max_turns, - conversation_index, - run, - ) - ) - conversation_index += 1 - return jobs - async def _worker( self, worker_id: int, diff --git a/src/vera_mh/judge/__init__.py b/src/vera_mh/judge/__init__.py index c1fda329..8ba9f28c 100644 --- a/src/vera_mh/judge/__init__.py +++ b/src/vera_mh/judge/__init__.py @@ -2,14 +2,10 @@ from .llm_judge import LLMJudge from .run import run_judging -from .runner import ( - judge_conversations, - judge_single_conversation, -) +from .runner import judge_conversations __all__ = [ "LLMJudge", "judge_conversations", - "judge_single_conversation", "run_judging", ] diff --git a/src/vera_mh/judge/llm_judge.py b/src/vera_mh/judge/llm_judge.py index 499d97a6..7524d27f 100644 --- a/src/vera_mh/judge/llm_judge.py +++ b/src/vera_mh/judge/llm_judge.py @@ -43,7 +43,7 @@ def __init__( verbose: Whether to print verbose output during initialization """ - # Setup logger: batch judging and `judge.py` pass an explicit path from + # Setup logger: batch judging passes an explicit path from # build_judge_task_log_path (scoped to the run). Omitted log_file is only # for direct construction, tests, etc.—then judge_unscoped/ under # default_adhoc_parent(); UUID stem avoids concurrent overwrites. @@ -63,7 +63,7 @@ def __init__( # One log path per batch job: stem matches the evaluation TSV (see # build_judge_task_log_path). Different evaluation runs use different - # output folders, so run_key differs and logs do not collide across runs. + # output folders, so logs do not collide across runs. # mode="w" truncates on each job start; re-running the same job in the # same folder replaces the log (and TSV), which is intentional. file_handler = logging.FileHandler(log_file, mode="w", encoding="utf-8") diff --git a/src/vera_mh/judge/runner.py b/src/vera_mh/judge/runner.py index 89797878..803f3902 100644 --- a/src/vera_mh/judge/runner.py +++ b/src/vera_mh/judge/runner.py @@ -747,30 +747,3 @@ async def judge_conversations( print(f" ({skipped_error_n} skipped due to errors)") return results, output_folder - - -async def judge_single_conversation( - judge: LLMJudge, - conversation: ConversationData, - output_folder: str, -) -> Optional[Dict[str, Any]]: - """ - Judge a single conversation. - - Args: - judge: LLMJudge instance - conversation: ConversationData with content and metadata - output_folder: Output folder for results - - Returns: - Evaluation results or None if failed - """ - conversation_filename = conversation.metadata.get("filename", "unknown.txt") - print(f"📄 Judging: {conversation_filename}") - - result = await judge.evaluate_conversation_question_flow( - conversation, output_folder=output_folder, auto_save=True - ) - - print(f"🟢 Done: {conversation_filename}") - return result diff --git a/src/vera_mh/judge/utils.py b/src/vera_mh/judge/utils.py index 2c70639b..043ff63b 100644 --- a/src/vera_mh/judge/utils.py +++ b/src/vera_mh/judge/utils.py @@ -1,7 +1,6 @@ """Utility functions for the judge module.""" import os -from datetime import datetime from pathlib import Path from typing import Optional, Sequence @@ -78,65 +77,22 @@ def judge_evaluation_tsv_filename( return f"{conversation_name}_{judge_suffix}.tsv" -def build_single_conversation_judge_run_key( - conversation_path: str | Path, - *, - now: Optional[datetime] = None, -) -> str: - """ - Human-readable folder basename for ``judge.py`` single-file (``-c``) mode. - - Format: ``single___``. Batch judging uses the - evaluation folder basename instead; this gives a stable, unique folder name - per CLI run. Per-task logs use :func:`build_judge_task_log_path` with - ``output_folder`` set, so no ``run_key`` is involved. - """ - dt = datetime.now() if now is None else now - ts = dt.strftime("%Y%m%d_%H%M%S_%f")[:-3] - stem = Path(conversation_path).stem - return f"single_{ts}__{stem}" - - -def get_judge_logs_root() -> str: - """ - Root directory for legacy per-task judge LLM logs (when ``output_folder`` is - omitted from :func:`build_judge_task_log_path`). - - Override with env ``VERA_JUDGE_LOGS_ROOT`` (e.g. set by pytest to a temp dir). - Default: ``judge_logs`` in the current working directory. - """ - return os.environ.get("VERA_JUDGE_LOGS_ROOT", "judge_logs") - - def build_judge_task_log_path( conversation_filename: str, judge_model: str, judge_instance: Optional[int] = None, *, - run_key: Optional[str] = None, - logs_root: Optional[str] = None, - output_folder: Optional[str] = None, + output_folder: str, ) -> str: """ - Path to the per-task judge LLM log file, parallel to judge_evaluation_tsv_filename. - - With ``output_folder``: ``{output_folder}/logs/{stem}.log`` (``run_key`` unused). - - Legacy layout (no ``output_folder``): ``{logs_root}/{run_key}/{stem}.log`` - (``logs_root`` defaults to :func:`get_judge_logs_root`). ``run_key`` is required - for this branch. + Path to the per-task judge LLM log file, parallel to judge_evaluation_tsv_filename: + ``{output_folder}/logs/{stem}.log``. """ tsv_name = judge_evaluation_tsv_filename( conversation_filename, judge_model, judge_instance ) stem = Path(tsv_name).stem - if output_folder is not None: - return str(Path(output_folder) / "logs" / f"{stem}.log") - if not run_key: - raise ValueError("run_key is required when output_folder is not set") - # Legacy layout: {logs_root}/{run_key}/{stem}.log - root = logs_root if logs_root is not None else get_judge_logs_root() - return str(Path(root) / run_key / f"{stem}.log") + return str(Path(output_folder) / "logs" / f"{stem}.log") def default_adhoc_parent() -> str: diff --git a/tests/conftest.py b/tests/conftest.py index 339a7438..b693702f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,27 +8,20 @@ from tests.mocks.mock_llm import MockLLM from vera_mh.judge.rubric_config import RubricConfig -_judge_logs_temp: Path | None = None _adhoc_temp: Path | None = None def pytest_sessionstart(session: pytest.Session) -> None: - """Point judge file logs and adhoc outputs at temp dirs (cleaned in finish).""" - global _judge_logs_temp, _adhoc_temp - _judge_logs_temp = Path(tempfile.mkdtemp(prefix="vera_judge_logs_")) - os.environ["VERA_JUDGE_LOGS_ROOT"] = str(_judge_logs_temp) + """Point adhoc outputs, including unscoped judge logs, at a temp dir.""" + global _adhoc_temp _adhoc_temp = Path(tempfile.mkdtemp(prefix="vera_adhoc_")) os.environ["VERA_ADHOC_PARENT"] = str(_adhoc_temp) def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: - """Remove temp dirs; clear env so CLI uses normal defaults.""" - global _judge_logs_temp, _adhoc_temp - os.environ.pop("VERA_JUDGE_LOGS_ROOT", None) + """Remove the temp dir; clear env so CLI uses normal defaults.""" + global _adhoc_temp os.environ.pop("VERA_ADHOC_PARENT", None) - if _judge_logs_temp is not None and _judge_logs_temp.exists(): - shutil.rmtree(_judge_logs_temp, ignore_errors=True) - _judge_logs_temp = None if _adhoc_temp is not None and _adhoc_temp.exists(): shutil.rmtree(_adhoc_temp, ignore_errors=True) _adhoc_temp = None diff --git a/tests/integration/test_evaluation_runner.py b/tests/integration/test_evaluation_runner.py index 69690042..85e36e12 100644 --- a/tests/integration/test_evaluation_runner.py +++ b/tests/integration/test_evaluation_runner.py @@ -32,7 +32,6 @@ from vera_mh.judge.runner import ( batch_evaluate_with_individual_judges, judge_conversations, - judge_single_conversation, ) from vera_mh.utils.judge_results import build_dataframe_from_tsv_files @@ -167,56 +166,6 @@ def create_mock_judge(*args, **kwargs: Dict[str, Any]) -> MockLLM: yield -@pytest.mark.integration -@pytest.mark.asyncio -class TestJudgeSingleConversation: - """Tests for judge_single_conversation function. - - Note: judge_single_conversation calls evaluate_conversation method - which doesn't exist in LLMJudge (it uses evaluate_conversation_question_flow). - This appears to be dead/unused code. Tests focus on actual working paths. - """ - - async def test_judge_single_conversation_file_not_found( - self, - mock_rubric_files: str, - mock_llm_factory_for_judge, - tmp_path: Path, - ): - """Test handling of missing conversation file. - - Arrange: Create judge with valid rubric, use non-existent file - Act: Call judge_single_conversation with missing file - Assert: Raises FileNotFoundError when loading conversation - """ - from vera_mh.judge.llm_judge import LLMJudge - - output_folder = str(tmp_path / "evaluations") - missing_file = str(tmp_path / "nonexistent.txt") - - # Load rubric config - rubric_config = await RubricConfig.load( - rubric_folder=mock_rubric_files, - rubric_file="rubric.tsv", - rubric_prompt_beginning_file="rubric_prompt_beginning.txt", - question_prompt_file="question_prompt.txt", - ) - - judge = LLMJudge( - judge_model="mock-judge", - rubric_config=rubric_config, - ) - - # Should raise FileNotFoundError when trying to load missing file - with pytest.raises(FileNotFoundError): - conversation = await ConversationData.load(missing_file) - await judge_single_conversation( - judge=judge, - conversation=conversation, - output_folder=output_folder, - ) - - @pytest.mark.integration @pytest.mark.asyncio class TestBatchEvaluateWithIndividualJudges: diff --git a/tests/unit/judge/test_runner_extra_params.py b/tests/unit/judge/test_runner_extra_params.py index 13e4a491..a179593b 100644 --- a/tests/unit/judge/test_runner_extra_params.py +++ b/tests/unit/judge/test_runner_extra_params.py @@ -279,23 +279,6 @@ async def test_resume_rebuilds_results_csv_from_all_tsvs_when_batch_empty( class TestJudgeTaskLogPath: """Per-task log paths align with evaluation TSV basenames.""" - def test_build_judge_task_log_path_matches_tsv_stem(self, tmp_path: Path) -> None: - """Log file stem matches TSV basename from judge_evaluation_tsv_filename.""" - conv = "subdir/abc.txt" - model = "gpt-4o" - tsv_name = judge_evaluation_tsv_filename(conv, model, 3) - expected_stem = Path(tsv_name).stem - - log_path = build_judge_task_log_path( - conv, - model, - 3, - run_key="j_run__convfolder", - logs_root=str(tmp_path), - ) - - assert log_path == str(tmp_path / "j_run__convfolder" / f"{expected_stem}.log") - def test_build_judge_task_log_path_under_output_folder( self, tmp_path: Path ) -> None: @@ -314,13 +297,3 @@ def test_build_judge_task_log_path_under_output_folder( ) assert log_path == str(eval_dir / "logs" / f"{expected_stem}.log") - - def test_build_judge_task_log_path_requires_run_key_for_legacy_layout( - self, tmp_path: Path - ) -> None: - with pytest.raises(ValueError, match="run_key"): - build_judge_task_log_path( - "a.txt", - "gpt-4o", - logs_root=str(tmp_path), - ) diff --git a/tests/unit/judge/test_utils.py b/tests/unit/judge/test_utils.py index 6896a26a..a7d34fa6 100644 --- a/tests/unit/judge/test_utils.py +++ b/tests/unit/judge/test_utils.py @@ -1,12 +1,10 @@ """Unit tests for judge utility functions.""" import argparse -from datetime import datetime import pytest from vera_mh.judge.utils import ( - build_single_conversation_judge_run_key, parse_judge_models, ) @@ -129,17 +127,3 @@ def test_count_alphanumeric_raises(self): judge_model = _setup_judge_model_arg(["-j", "gpt-4o:2x"]) with pytest.raises(ValueError, match="must be an integer"): parse_judge_models(judge_model) - - -@pytest.mark.unit -class TestBuildSingleConversationJudgeRunKey: - """Tests for build_single_conversation_judge_run_key.""" - - def test_format_matches_cli_single_mode(self): - """Stem and truncated microsecond field match prior judge.py inline logic.""" - fixed = datetime(2026, 4, 20, 15, 30, 45, 123456) - key = build_single_conversation_judge_run_key( - "/tmp/foo/Amelia_conv.txt", - now=fixed, - ) - assert key == "single_20260420_153045_123__Amelia_conv" diff --git a/tests/unit/utils/test_run_root.py b/tests/unit/utils/test_run_root.py index e5d677ce..8eb3b3a9 100644 --- a/tests/unit/utils/test_run_root.py +++ b/tests/unit/utils/test_run_root.py @@ -44,7 +44,6 @@ ("judge/runner.py", "_worker"): "log line timestamps", ("judge/runner.py", "_run_workers_with_queue"): "log line timestamps", ("judge/runner.py", "judge_conversations"): "batch duration", - ("judge/utils.py", "build_single_conversation_judge_run_key"): "single mode", ("llm_clients/llm_interface.py", "_set_response_metadata"): "response time", ("utils/naming.py", "add_timestamp_to_path"): "comparison chart file names", ("utils/run_root.py", "run_started_at"): "the run's one clock read",