diff --git a/src/vera_mh/generate/conversation_turn.py b/src/vera_mh/generate/conversation_turn.py index f5b978e6..421a2ec9 100644 --- a/src/vera_mh/generate/conversation_turn.py +++ b/src/vera_mh/generate/conversation_turn.py @@ -5,7 +5,7 @@ from langchain_core.messages import AIMessage, BaseMessage, HumanMessage -from vera_mh.llm_clients.llm_interface import Role +from vera_mh.utils.role import Role @dataclass diff --git a/src/vera_mh/generate/runner.py b/src/vera_mh/generate/runner.py index c62f04db..d813e8f9 100644 --- a/src/vera_mh/generate/runner.py +++ b/src/vera_mh/generate/runner.py @@ -17,7 +17,7 @@ from typing import AbstractSet, Any, Dict, List, Optional, Tuple from vera_mh.llm_clients import LLMFactory -from vera_mh.llm_clients.llm_interface import LLMGenerationFailed, Role +from vera_mh.llm_clients.llm_interface import LLMGenerationFailed from vera_mh.utils.conversation_layout import resolve_conversation_input from vera_mh.utils.logging_utils import ( cleanup_logger, @@ -31,6 +31,7 @@ TRANSCRIPT_RUN_SUFFIX_RE, persona_token_for_transcript_stem, ) +from vera_mh.utils.role import Role from .conversation_simulator import ConversationSimulator from .utils import DEFAULT_USER_LANGUAGE, load_prompts_from_csv diff --git a/src/vera_mh/judge/llm_judge.py b/src/vera_mh/judge/llm_judge.py index 499d97a6..eb544343 100644 --- a/src/vera_mh/judge/llm_judge.py +++ b/src/vera_mh/judge/llm_judge.py @@ -15,9 +15,10 @@ log_llm_failure_metadata, log_llm_response_metadata, ) -from vera_mh.llm_clients import LLMFactory, Role +from vera_mh.llm_clients import LLMFactory from vera_mh.llm_clients.llm_interface import JudgeLLM, LLMGenerationFailed from vera_mh.utils.ratings import BEST_PRACTICE, DAMAGING, NEUTRAL +from vera_mh.utils.role import Role class LLMJudge: diff --git a/src/vera_mh/llm_clients/__init__.py b/src/vera_mh/llm_clients/__init__.py index c3594b10..5ab4d67b 100644 --- a/src/vera_mh/llm_clients/__init__.py +++ b/src/vera_mh/llm_clients/__init__.py @@ -11,13 +11,12 @@ from .config import Config from .llm_factory import LLMFactory -from .llm_interface import JudgeLLM, LLMGenerationFailed, LLMInterface, Role +from .llm_interface import JudgeLLM, LLMGenerationFailed, LLMInterface __all__ = [ "LLMInterface", "JudgeLLM", "LLMFactory", "Config", - "Role", "LLMGenerationFailed", ] diff --git a/src/vera_mh/llm_clients/azure_llm.py b/src/vera_mh/llm_clients/azure_llm.py index 58a6c1a4..eadadd29 100644 --- a/src/vera_mh/llm_clients/azure_llm.py +++ b/src/vera_mh/llm_clients/azure_llm.py @@ -7,9 +7,9 @@ from langchain_core.messages import HumanMessage, SystemMessage from pydantic import BaseModel -from vera_mh.llm_clients import Role from vera_mh.utils.conversation_utils import build_langchain_messages from vera_mh.utils.debug import debug_print +from vera_mh.utils.role import Role from .config import Config from .llm_interface import ( diff --git a/src/vera_mh/llm_clients/claude_llm.py b/src/vera_mh/llm_clients/claude_llm.py index 3cc2ae70..33d96385 100644 --- a/src/vera_mh/llm_clients/claude_llm.py +++ b/src/vera_mh/llm_clients/claude_llm.py @@ -7,11 +7,11 @@ from vera_mh.utils.conversation_utils import build_langchain_messages from vera_mh.utils.debug import debug_print +from vera_mh.utils.role import Role from .config import Config from .llm_interface import ( JudgeLLM, - Role, ensure_pydantic_response, extract_structured_invoke_result, ) diff --git a/src/vera_mh/llm_clients/endpoint_llm.py b/src/vera_mh/llm_clients/endpoint_llm.py index 320b40b0..f283c0b1 100644 --- a/src/vera_mh/llm_clients/endpoint_llm.py +++ b/src/vera_mh/llm_clients/endpoint_llm.py @@ -4,9 +4,10 @@ import aiohttp from vera_mh.utils.conversation_utils import build_langchain_messages +from vera_mh.utils.role import Role from .config import Config -from .llm_interface import LLMInterface, Role +from .llm_interface import LLMInterface class EndpointLLM(LLMInterface): diff --git a/src/vera_mh/llm_clients/gemini_llm.py b/src/vera_mh/llm_clients/gemini_llm.py index d2567958..7a0a9690 100644 --- a/src/vera_mh/llm_clients/gemini_llm.py +++ b/src/vera_mh/llm_clients/gemini_llm.py @@ -9,11 +9,11 @@ from vera_mh.utils.conversation_utils import build_langchain_messages from vera_mh.utils.debug import debug_print +from vera_mh.utils.role import Role from .config import Config from .llm_interface import ( JudgeLLM, - Role, ensure_pydantic_response, extract_structured_invoke_result, ) diff --git a/src/vera_mh/llm_clients/llm_factory.py b/src/vera_mh/llm_clients/llm_factory.py index 680792e7..b5450d51 100644 --- a/src/vera_mh/llm_clients/llm_factory.py +++ b/src/vera_mh/llm_clients/llm_factory.py @@ -1,6 +1,8 @@ from typing import Optional -from .llm_interface import JudgeLLM, LLMInterface, Role +from vera_mh.utils.role import Role + +from .llm_interface import JudgeLLM, LLMInterface class LLMFactory: diff --git a/src/vera_mh/llm_clients/llm_interface.py b/src/vera_mh/llm_clients/llm_interface.py index 0b2b9ac0..bb2dd762 100644 --- a/src/vera_mh/llm_clients/llm_interface.py +++ b/src/vera_mh/llm_clients/llm_interface.py @@ -5,11 +5,12 @@ from abc import ABC, abstractmethod from collections.abc import Awaitable, Callable from datetime import datetime -from enum import Enum from typing import Any, Dict, List, Optional, Protocol, Type, TypeVar from pydantic import BaseModel, ValidationError +from vera_mh.utils.role import Role + T = TypeVar("T", bound=BaseModel) R = TypeVar("R") @@ -113,14 +114,6 @@ class LLMGenerationFailed(Exception): """Raised when an LLM call fails after all retry attempts are exhausted.""" -class Role(Enum): - """Role of the LLM in a conversation.""" - - PERSONA = "persona" # User roleplaying as a human seeking help - PROVIDER = "provider" # Chatbot providing support - JUDGE = "judge" # Judge role, used for judge operations - - # Default prompt sent to the LLM when starting a conversation (no first_message set). DEFAULT_START_PROMPT = "Start the conversation based on the system prompt" diff --git a/src/vera_mh/llm_clients/ollama_llm.py b/src/vera_mh/llm_clients/ollama_llm.py index e00a058a..3f9e29a3 100644 --- a/src/vera_mh/llm_clients/ollama_llm.py +++ b/src/vera_mh/llm_clients/ollama_llm.py @@ -5,9 +5,10 @@ from vera_mh.utils.conversation_utils import format_conversation_as_string from vera_mh.utils.debug import debug_print +from vera_mh.utils.role import Role from .config import Config -from .llm_interface import LLMInterface, Role +from .llm_interface import LLMInterface class OllamaLLM(LLMInterface): diff --git a/src/vera_mh/llm_clients/openai_llm.py b/src/vera_mh/llm_clients/openai_llm.py index 4f6132f5..a6d5426d 100644 --- a/src/vera_mh/llm_clients/openai_llm.py +++ b/src/vera_mh/llm_clients/openai_llm.py @@ -8,11 +8,11 @@ from vera_mh.utils.conversation_utils import build_langchain_messages from vera_mh.utils.debug import debug_print +from vera_mh.utils.role import Role from .config import Config from .llm_interface import ( JudgeLLM, - Role, ensure_pydantic_response, extract_structured_invoke_result, ) diff --git a/src/vera_mh/utils/conversation_utils.py b/src/vera_mh/utils/conversation_utils.py index 75d01e0d..b9ccc3e8 100644 --- a/src/vera_mh/utils/conversation_utils.py +++ b/src/vera_mh/utils/conversation_utils.py @@ -6,8 +6,8 @@ from langchain_core.messages import AIMessage, BaseMessage, HumanMessage -from vera_mh.llm_clients.llm_interface import Role from vera_mh.utils.naming import add_timestamp_to_path +from vera_mh.utils.role import Role from .debug import debug_print diff --git a/src/vera_mh/utils/role.py b/src/vera_mh/utils/role.py new file mode 100644 index 00000000..e8aaa199 --- /dev/null +++ b/src/vera_mh/utils/role.py @@ -0,0 +1,11 @@ +"""Who is speaking in a conversation, or judging one.""" + +from enum import Enum + + +class Role(Enum): + """Role of the LLM in a conversation.""" + + PERSONA = "persona" # User roleplaying as a human seeking help + PROVIDER = "provider" # Chatbot providing support + JUDGE = "judge" # Judge role, used for judge operations diff --git a/tests/conftest.py b/tests/conftest.py index 339a7438..d8c1e11c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -140,7 +140,7 @@ def mock_llm() -> MockLLM: @pytest.fixture def mock_persona() -> MockLLM: """Mock LLM configured as a persona.""" - from vera_mh.llm_clients.llm_interface import Role + from vera_mh.utils.role import Role return MockLLM( name="mock-persona", @@ -152,7 +152,7 @@ def mock_persona() -> MockLLM: @pytest.fixture def mock_agent() -> MockLLM: """Mock LLM configured as a chatbot agent.""" - from vera_mh.llm_clients.llm_interface import Role + from vera_mh.utils.role import Role return MockLLM( name="mock-agent", diff --git a/tests/integration/test_conversation_runner.py b/tests/integration/test_conversation_runner.py index 3e243756..67944942 100644 --- a/tests/integration/test_conversation_runner.py +++ b/tests/integration/test_conversation_runner.py @@ -17,7 +17,7 @@ from tests.mocks.mock_llm import MockLLM from vera_mh.generate.runner import ConversationRunner -from vera_mh.llm_clients.llm_interface import Role +from vera_mh.utils.role import Role def create_test_runner( diff --git a/tests/integration/test_vera_pipeline_e2e.py b/tests/integration/test_vera_pipeline_e2e.py index f22ebd59..f7e866a1 100644 --- a/tests/integration/test_vera_pipeline_e2e.py +++ b/tests/integration/test_vera_pipeline_e2e.py @@ -26,9 +26,9 @@ from tests.mocks.mock_llm import MockLLM from vera_mh import cli as vera -from vera_mh.llm_clients import Role from vera_mh.llm_clients.config import Config from vera_mh.llm_clients.llm_factory import LLMFactory +from vera_mh.utils.role import Role from vera_mh.vera_cli import config as cli_config # Read from Config so tests/integration/test_model_availability.py catches a diff --git a/tests/mocks/mock_llm.py b/tests/mocks/mock_llm.py index 33616ffe..c19dc41b 100644 --- a/tests/mocks/mock_llm.py +++ b/tests/mocks/mock_llm.py @@ -1,6 +1,7 @@ from typing import Any, Dict, List, Optional, Type, TypeVar -from vera_mh.llm_clients.llm_interface import JudgeLLM, LLMGenerationFailed, Role +from vera_mh.llm_clients.llm_interface import JudgeLLM, LLMGenerationFailed +from vera_mh.utils.role import Role T = TypeVar("T") diff --git a/tests/safe_changes/full_run.py b/tests/safe_changes/full_run.py index 4b47fbcc..4d2db554 100644 --- a/tests/safe_changes/full_run.py +++ b/tests/safe_changes/full_run.py @@ -87,8 +87,9 @@ ) from vera_mh.judge.rubric_config import RubricConfig from vera_mh.llm_clients.llm_factory import LLMFactory -from vera_mh.llm_clients.llm_interface import JudgeLLM, Role +from vera_mh.llm_clients.llm_interface import JudgeLLM from vera_mh.utils.conversation_utils import format_conversation_summary +from vera_mh.utils.role import Role from vera_mh.vera_cli import config as cli_config REPO_ROOT = Path(__file__).resolve().parents[2] diff --git a/tests/safe_changes/replay.py b/tests/safe_changes/replay.py index 52d885cc..534b1def 100644 --- a/tests/safe_changes/replay.py +++ b/tests/safe_changes/replay.py @@ -29,7 +29,8 @@ from vera_mh import cli as vera from vera_mh.llm_clients.llm_factory import LLMFactory -from vera_mh.llm_clients.llm_interface import JudgeLLM, Role +from vera_mh.llm_clients.llm_interface import JudgeLLM +from vera_mh.utils.role import Role from vera_mh.vera_cli import config as cli_config REPO_ROOT = Path(__file__).resolve().parents[2] diff --git a/tests/unit/generate/test_conversation_simulator.py b/tests/unit/generate/test_conversation_simulator.py index bd02e12e..138a7bfe 100644 --- a/tests/unit/generate/test_conversation_simulator.py +++ b/tests/unit/generate/test_conversation_simulator.py @@ -9,8 +9,8 @@ from vera_mh.llm_clients.llm_interface import ( DEFAULT_START_PROMPT, LLMGenerationFailed, - Role, ) +from vera_mh.utils.role import Role @pytest.mark.unit diff --git a/tests/unit/generate/test_conversation_turn.py b/tests/unit/generate/test_conversation_turn.py index 378e1062..cd780926 100644 --- a/tests/unit/generate/test_conversation_turn.py +++ b/tests/unit/generate/test_conversation_turn.py @@ -3,7 +3,7 @@ from langchain_core.messages import AIMessage, HumanMessage from vera_mh.generate.conversation_turn import ConversationTurn -from vera_mh.llm_clients import Role +from vera_mh.utils.role import Role class TestConversationTurnCreation: diff --git a/tests/unit/llm_clients/README.md b/tests/unit/llm_clients/README.md index 7fe1e169..5c7ddcd1 100644 --- a/tests/unit/llm_clients/README.md +++ b/tests/unit/llm_clients/README.md @@ -67,8 +67,8 @@ from unittest.mock import patch import pytest -from vera_mh.llm_clients import Role from vera_mh.llm_clients.my_provider_llm import MyProviderLLM +from vera_mh.utils.role import Role from .test_base_llm import TestJudgeLLMBase from .test_helpers import ( diff --git a/tests/unit/llm_clients/conftest.py b/tests/unit/llm_clients/conftest.py index 7b66a4c7..e76fd352 100644 --- a/tests/unit/llm_clients/conftest.py +++ b/tests/unit/llm_clients/conftest.py @@ -14,8 +14,8 @@ import pytest -from vera_mh.llm_clients import Role from vera_mh.llm_clients.config import Config +from vera_mh.utils.role import Role # ============================================================================ # Mock Response Factories diff --git a/tests/unit/llm_clients/test_azure_llm.py b/tests/unit/llm_clients/test_azure_llm.py index 1d382a0c..caa07738 100644 --- a/tests/unit/llm_clients/test_azure_llm.py +++ b/tests/unit/llm_clients/test_azure_llm.py @@ -6,10 +6,10 @@ import pytest from pydantic import BaseModel, Field -from vera_mh.llm_clients import Role from vera_mh.llm_clients.azure_llm import AzureLLM from vera_mh.llm_clients.config import Config from vera_mh.llm_clients.llm_interface import LLMGenerationFailed +from vera_mh.utils.role import Role from .test_base_llm import TestJudgeLLMBase from .test_helpers import ( diff --git a/tests/unit/llm_clients/test_base_llm.py b/tests/unit/llm_clients/test_base_llm.py index d1a1bf5c..e24960c0 100644 --- a/tests/unit/llm_clients/test_base_llm.py +++ b/tests/unit/llm_clients/test_base_llm.py @@ -32,12 +32,12 @@ def get_mock_patches(self): import pytest from pydantic import BaseModel, Field -from vera_mh.llm_clients import Role from vera_mh.llm_clients.llm_interface import ( JudgeLLM, LLMGenerationFailed, LLMInterface, ) +from vera_mh.utils.role import Role from .test_helpers import ( assert_iso_timestamp, diff --git a/tests/unit/llm_clients/test_claude_llm.py b/tests/unit/llm_clients/test_claude_llm.py index 5b09c68b..a470b3e6 100644 --- a/tests/unit/llm_clients/test_claude_llm.py +++ b/tests/unit/llm_clients/test_claude_llm.py @@ -5,10 +5,10 @@ import pytest -from vera_mh.llm_clients import Role from vera_mh.llm_clients.claude_llm import ClaudeLLM from vera_mh.llm_clients.config import Config from vera_mh.llm_clients.llm_interface import LLMGenerationFailed +from vera_mh.utils.role import Role from .test_base_llm import TestJudgeLLMBase from .test_helpers import ( diff --git a/tests/unit/llm_clients/test_endpoint_llm.py b/tests/unit/llm_clients/test_endpoint_llm.py index 9cc82f90..d13a9bb2 100644 --- a/tests/unit/llm_clients/test_endpoint_llm.py +++ b/tests/unit/llm_clients/test_endpoint_llm.py @@ -5,9 +5,9 @@ import pytest -from vera_mh.llm_clients import Role from vera_mh.llm_clients.endpoint_llm import EndpointLLM from vera_mh.llm_clients.llm_interface import DEFAULT_START_PROMPT, LLMGenerationFailed +from vera_mh.utils.role import Role from .test_base_llm import TestLLMBase from .test_helpers import ( diff --git a/tests/unit/llm_clients/test_gemini_llm.py b/tests/unit/llm_clients/test_gemini_llm.py index 7cd7d1fd..281912c5 100644 --- a/tests/unit/llm_clients/test_gemini_llm.py +++ b/tests/unit/llm_clients/test_gemini_llm.py @@ -6,13 +6,13 @@ import pytest from langchain_core.messages import AIMessage -from vera_mh.llm_clients import Role from vera_mh.llm_clients.config import Config from vera_mh.llm_clients.gemini_llm import ( _STRUCTURED_OUTPUT_MAX_LOG_CHARS, GeminiLLM, ) from vera_mh.llm_clients.llm_interface import LLMGenerationFailed +from vera_mh.utils.role import Role from .test_base_llm import TestJudgeLLMBase from .test_helpers import ( diff --git a/tests/unit/llm_clients/test_helpers.py b/tests/unit/llm_clients/test_helpers.py index 3c42996d..e1ab9242 100644 --- a/tests/unit/llm_clients/test_helpers.py +++ b/tests/unit/llm_clients/test_helpers.py @@ -23,8 +23,8 @@ from datetime import datetime from typing import Any, Dict, Optional -from vera_mh.llm_clients import Role from vera_mh.llm_clients.llm_interface import LLMGenerationFailed, LLMInterface +from vera_mh.utils.role import Role # ============================================================================ # Metadata Assertions diff --git a/tests/unit/llm_clients/test_llm_factory.py b/tests/unit/llm_clients/test_llm_factory.py index 445b20ce..ac4e3d7a 100644 --- a/tests/unit/llm_clients/test_llm_factory.py +++ b/tests/unit/llm_clients/test_llm_factory.py @@ -2,13 +2,13 @@ import pytest -from vera_mh.llm_clients import Role from vera_mh.llm_clients.azure_llm import AzureLLM from vera_mh.llm_clients.claude_llm import ClaudeLLM from vera_mh.llm_clients.gemini_llm import GeminiLLM from vera_mh.llm_clients.llm_factory import LLMFactory from vera_mh.llm_clients.ollama_llm import OllamaLLM from vera_mh.llm_clients.openai_llm import OpenAILLM +from vera_mh.utils.role import Role @pytest.fixture diff --git a/tests/unit/llm_clients/test_llm_interface.py b/tests/unit/llm_clients/test_llm_interface.py index bc583199..84a0a461 100644 --- a/tests/unit/llm_clients/test_llm_interface.py +++ b/tests/unit/llm_clients/test_llm_interface.py @@ -5,13 +5,13 @@ import pytest from pydantic import BaseModel -from vera_mh.llm_clients import Role from vera_mh.llm_clients.llm_interface import ( LLMGenerationFailed, LLMInterface, ensure_pydantic_response, extract_structured_invoke_result, ) +from vera_mh.utils.role import Role class ConcreteLLM(LLMInterface): diff --git a/tests/unit/llm_clients/test_ollama_llm.py b/tests/unit/llm_clients/test_ollama_llm.py index 55590442..5672f22d 100644 --- a/tests/unit/llm_clients/test_ollama_llm.py +++ b/tests/unit/llm_clients/test_ollama_llm.py @@ -5,7 +5,8 @@ import pytest -from vera_mh.llm_clients.llm_interface import LLMGenerationFailed, Role +from vera_mh.llm_clients.llm_interface import LLMGenerationFailed +from vera_mh.utils.role import Role from .test_base_llm import TestLLMBase from .test_helpers import ( diff --git a/tests/unit/llm_clients/test_openai_llm.py b/tests/unit/llm_clients/test_openai_llm.py index 741ddfcf..411f924f 100644 --- a/tests/unit/llm_clients/test_openai_llm.py +++ b/tests/unit/llm_clients/test_openai_llm.py @@ -5,10 +5,10 @@ import pytest -from vera_mh.llm_clients import Role from vera_mh.llm_clients.config import Config from vera_mh.llm_clients.llm_interface import LLMGenerationFailed from vera_mh.llm_clients.openai_llm import OpenAILLM +from vera_mh.utils.role import Role from .test_base_llm import TestJudgeLLMBase from .test_helpers import ( diff --git a/tests/unit/utils/test_conversation_utils.py b/tests/unit/utils/test_conversation_utils.py index d377d962..53844b00 100644 --- a/tests/unit/utils/test_conversation_utils.py +++ b/tests/unit/utils/test_conversation_utils.py @@ -5,7 +5,6 @@ import pytest from langchain_core.messages import AIMessage, HumanMessage -from vera_mh.llm_clients.llm_interface import Role from vera_mh.utils.conversation_utils import ( add_timestamp_to_path, build_langchain_messages, @@ -14,6 +13,7 @@ format_conversation_summary, load_persona_role_reminder, ) +from vera_mh.utils.role import Role class TestEnsureProviderHasLastTurn: diff --git a/tests/unit/utils/test_logging_utils.py b/tests/unit/utils/test_logging_utils.py index c0a7b72a..3a077c3c 100644 --- a/tests/unit/utils/test_logging_utils.py +++ b/tests/unit/utils/test_logging_utils.py @@ -6,7 +6,6 @@ import pytest from tests.mocks.mock_llm import MockLLM -from vera_mh.llm_clients import Role from vera_mh.utils.debug import set_debug from vera_mh.utils.logging_utils import ( cleanup_logger, @@ -19,6 +18,7 @@ log_llm_response_metadata, setup_conversation_logger, ) +from vera_mh.utils.role import Role @pytest.mark.unit