From 53a03581f42202afa0c8554f6c50be10fca1ebbf Mon Sep 17 00:00:00 2001 From: Anselme Irumva Date: Thu, 25 Jun 2026 10:25:18 +0200 Subject: [PATCH] CORE-507 Troubleshoot: Attempt 3 failed to extract JSON caused by: Failed to construct model: ActionExtractionResult --- .../agent/recommender_advisor_agent/agent.py | 9 +- .../phase_handlers/action_handler.py | 54 ++-- backend/common_libs/llm/models_utils.py | 11 +- .../test_action_handler_eval.py | 244 ++++++++++++++++++ .../test_action_planning_intent_fix.py | 8 + .../test_blanket_rejection_fix.py | 8 + .../evaluation_tests/test_schema_builder.py | 2 + ...reproduce_recommender_early_termination.py | 10 +- .../test_recommender_agent_interactive.py | 8 + 9 files changed, 324 insertions(+), 30 deletions(-) create mode 100644 backend/evaluation_tests/recommender_advisor_agent/test_action_handler_eval.py diff --git a/backend/app/agent/recommender_advisor_agent/agent.py b/backend/app/agent/recommender_advisor_agent/agent.py index 2d84352f3..d04e94cef 100644 --- a/backend/app/agent/recommender_advisor_agent/agent.py +++ b/backend/app/agent/recommender_advisor_agent/agent.py @@ -51,10 +51,10 @@ from common_libs.llm.generative_models import GeminiGenerativeLLM from common_libs.llm.models_utils import ( LLMConfig, - LOW_TEMPERATURE_GENERATION_CONFIG, MEDIUM_TEMPERATURE_GENERATION_CONFIG, JSON_GENERATION_CONFIG ) +from common_libs.llm.schema_builder import with_response_schema # Import PreferenceVector from Epic 2 from app.agent.preference_elicitation_agent.types import PreferenceVector @@ -120,6 +120,12 @@ def __init__( config=llm_config ) + self._action_llm = GeminiGenerativeLLM( + config=LLMConfig( + generation_config=MEDIUM_TEMPERATURE_GENERATION_CONFIG | with_response_schema(ActionExtractionResult) + ) + ) + # Initialize LLM callers self._conversation_caller: LLMCaller[ConversationResponse] = LLMCaller[ConversationResponse]( model_response_type=ConversationResponse @@ -162,6 +168,7 @@ def _init_phase_handlers(self) -> None: conversation_llm=self._conversation_llm, conversation_caller=self._conversation_caller, action_caller=self._action_caller, + action_llm=self._action_llm, intent_classifier=self._intent_classifier, logger=self.logger ) diff --git a/backend/app/agent/recommender_advisor_agent/phase_handlers/action_handler.py b/backend/app/agent/recommender_advisor_agent/phase_handlers/action_handler.py index d954dc64a..f9c998d35 100644 --- a/backend/app/agent/recommender_advisor_agent/phase_handlers/action_handler.py +++ b/backend/app/agent/recommender_advisor_agent/phase_handlers/action_handler.py @@ -4,6 +4,7 @@ Handles the ACTION_PLANNING phase where we guide users toward concrete next steps. """ +from textwrap import dedent from app.agent.agent_types import LLMStats from app.agent.llm_caller import LLMCaller @@ -50,6 +51,7 @@ def __init__( conversation_llm: GeminiGenerativeLLM, conversation_caller: LLMCaller[ConversationResponse], action_caller: LLMCaller[ActionExtractionResult], + action_llm: GeminiGenerativeLLM = None, intent_classifier: IntentClassifier = None, present_handler: 'PresentPhaseHandler' = None, concerns_handler: 'ConcernsPhaseHandler' = None, @@ -61,6 +63,7 @@ def __init__( Args: action_caller: LLM caller for action extraction + action_llm: LLM configured with build_request_schema(ActionExtractionResult) for structured extraction intent_classifier: Optional intent classifier for routing user choices present_handler: Optional present handler for immediate delegation concerns_handler: Optional concerns handler for immediate delegation @@ -68,6 +71,7 @@ def __init__( """ super().__init__(conversation_llm, conversation_caller, **kwargs) self._action_caller = action_caller + self._action_llm = action_llm self._intent_classifier = intent_classifier self._present_handler = present_handler self._concerns_handler = concerns_handler @@ -285,35 +289,33 @@ async def _extract_action( ) ] - prompt = f""" -Analyze this user response to determine if they're committing to an action: - -User said: "{user_input}" - -Context: They've been exploring career/occupation recommendations. - -Determine: -1. has_commitment: Did they make a clear commitment to take action? (true/false) -2. action_type: What type of action? Must be one of: apply_to_job, enroll_in_training, explore_occupation, research_employer, network -3. commitment_level: How strong is their commitment? Must be one of: will_do_this_week, will_do_this_month, interested, maybe_later, not_interested -4. barriers_mentioned: List any barriers or concerns mentioned (empty list if none) -5. plan_when: When they'll act, in their words (e.g., "Thursday morning"). Null if not stated. -6. plan_where: Where / the specific target (e.g., "the depot", "the Brookside posting"). Null if not stated. -7. plan_how: How - transport, what to bring or say (e.g., "matatu, ask the supervisor"). Null if not stated. -8. plan_backup: The if-then backup for the most likely obstacle (e.g., "Saturday if the fare is short"). Null if not stated. - -IMPORTANT: -- Use exact field names: has_commitment, action_type, commitment_level, barriers_mentioned, plan_when, plan_where, plan_how, plan_backup -- action_type and commitment_level must use exact enum values listed above -- If has_commitment is false, set action_type and commitment_level to null -- Only fill a plan_* field if the user actually stated it THIS turn - never invent details. Leave it null otherwise. - -{get_json_response_instructions(examples=examples)} -""" + prompt = dedent(f""" + Analyze this user response to determine if they're committing to an action: + + User said: "{user_input}" + + Context: They've been exploring career/occupation recommendations. + + Determine: + 1. has_commitment: Did they make a clear commitment to take action? (true/false) + 2. action_type: What type of action? Must be one of: apply_to_job, enroll_in_training, explore_occupation, research_employer, network + 3. commitment_level: How strong is their commitment? Must be one of: will_do_this_week, will_do_this_month, interested, maybe_later, not_interested + 4. barriers_mentioned: List any barriers or concerns mentioned (empty list if none) + 5. plan_when: When they'll act, in their words (e.g., "Thursday morning"). Null if not stated. + 6. plan_where: Where / the specific target (e.g., "the depot", "the Brookside posting"). Null if not stated. + 7. plan_how: How - transport, what to bring or say (e.g., "matatu, ask the supervisor"). Null if not stated. + 8. plan_backup: The if-then backup for the most likely obstacle (e.g., "Saturday if the fare is short"). Null if not stated. + + IMPORTANT: + - Use exact field names: has_commitment, action_type, commitment_level, barriers_mentioned, plan_when, plan_where, plan_how, plan_backup + - action_type and commitment_level must use exact enum values listed above + - If has_commitment is false, set action_type and commitment_level to null + - Only fill a plan_* field if the user actually stated it THIS turn - never invent details. Leave it null otherwise. + """) try: result = await self._action_caller.call_llm( - llm=self._conversation_llm, + llm=self._action_llm or self._conversation_llm, llm_input=ConversationHistoryFormatter.format_for_agent_generative_prompt( model_response_instructions=prompt, context=context, diff --git a/backend/common_libs/llm/models_utils.py b/backend/common_libs/llm/models_utils.py index 989c1e841..ab40411e8 100644 --- a/backend/common_libs/llm/models_utils.py +++ b/backend/common_libs/llm/models_utils.py @@ -207,13 +207,20 @@ async def generate_content(self, llm_input: LLMInput | str) -> LLMResponse: raise NotImplementedError() +vertex_ai_initialized: bool = False +def init_vertex_ai_once(_logger: logging.Logger, location: str): + global vertex_ai_initialized + if not vertex_ai_initialized: + _logger.info("Initializing VertexAI client with location: %s", location) + vertexai.init(location=location) + vertex_ai_initialized = True + class BasicLLM(LLM): def __init__(self, *, config: LLMConfig = LLMConfig()): # Before constructing the llm model, we need to initialize the VertexAI client # as the init function may have been called in another module with different parameters self.logger = logging.getLogger(self.__class__.__name__) - self.logger.info("Initializing VertexAI client with location: %s", config.location) - vertexai.init(location=config.location) + init_vertex_ai_once(self.logger, config.location) self._retry_config = config.retry_config self._model = None self._chat = None diff --git a/backend/evaluation_tests/recommender_advisor_agent/test_action_handler_eval.py b/backend/evaluation_tests/recommender_advisor_agent/test_action_handler_eval.py new file mode 100644 index 000000000..0560c968b --- /dev/null +++ b/backend/evaluation_tests/recommender_advisor_agent/test_action_handler_eval.py @@ -0,0 +1,244 @@ +import asyncio +import logging +from pydantic import BaseModel +from pathlib import Path + +import pytest + +from app.agent.llm_caller import LLMCaller +from app.agent.recommender_advisor_agent.llm_response_models import ( + ActionExtractionResult, + ConversationResponse, +) +from app.agent.recommender_advisor_agent.phase_handlers.action_handler import ActionPhaseHandler +from app.conversation_memory.conversation_memory_types import ( + ConversationContext, + ConversationHistory, +) +from common_libs.llm.generative_models import GeminiGenerativeLLM +from common_libs.llm.models_utils import ( + JSON_GENERATION_CONFIG, + LOW_TEMPERATURE_GENERATION_CONFIG, + ZERO_TEMPERATURE_GENERATION_CONFIG, + LLMConfig, +) +from common_libs.llm.schema_builder import with_response_schema +from evaluation_tests.compass_test_case import CompassTestCase + +logger = logging.getLogger(__name__) +LOG_DIR = Path(__file__).parent / "logs" + + + +class ActionExtractionTest(CompassTestCase): + user_input: str + expected_action_result: ActionExtractionResult + + +# Non-None value in a plan slot means "must be present in actual output". +# The exact text is not compared — only presence (not None) is asserted. +_PRESENT = "present" + +TESTS: list[ActionExtractionTest] = [ + ActionExtractionTest( + name="concrete commitment with when and how", + user_input=( + "Yes, I'll go to the Nyali construction site Thursday morning by matatu " + "and ask the foreman directly about the apprenticeship opening." + ), + expected_action_result=ActionExtractionResult( + reasoning="", + has_commitment=True, + action_type="apply_to_job", + commitment_level="will_do_this_week", + barriers_mentioned=[], + plan_when=_PRESENT, + plan_how=_PRESENT, + ), + ), + ActionExtractionTest( + name="vague strong commitment, no plan details stated", + user_input="Yes I want to apply for the electrical apprenticeship, I'll do it this week.", + expected_action_result=ActionExtractionResult( + reasoning="", + has_commitment=True, + action_type="apply_to_job", + commitment_level="will_do_this_week", + barriers_mentioned=[], + ), + ), + ActionExtractionTest( + name="training enrollment commitment next month", + user_input=( + "I want to register for the electrician grade test course. " + "I'll sign up at Mombasa Technical next month." + ), + expected_action_result=ActionExtractionResult( + reasoning="", + has_commitment=True, + action_type="enroll_in_training", + commitment_level="will_do_this_month", + barriers_mentioned=[], + ), + ), + ActionExtractionTest( + name="barriers only, no commitment made", + user_input=( + "I'm not sure I can afford the bus fare to get there, " + "and I don't know if they're still hiring for that position." + ), + expected_action_result=ActionExtractionResult( + reasoning="", + has_commitment=False, + action_type=None, + commitment_level=None, + barriers_mentioned=["placeholder"], # non-empty → assert actual has barriers + ), + ), + ActionExtractionTest( + name="full concrete plan including backup", + user_input=( + "I'll go Monday morning, take the 7am Coast Bus from Mwembe Tayari, " + "bring my ID and ask for the site manager. " + "If he's not there I'll come back Wednesday." + ), + expected_action_result=ActionExtractionResult( + reasoning="", + has_commitment=True, + action_type="apply_to_job", + commitment_level="will_do_this_week", + barriers_mentioned=[], + plan_when=_PRESENT, + plan_how=_PRESENT, + plan_backup=_PRESENT, + ), + ), + ActionExtractionTest( + name="networking commitment this week", + user_input=( + "I'll ask my uncle to introduce me to his contacts at the construction " + "company this weekend — he knows the site supervisor." + ), + expected_action_result=ActionExtractionResult( + reasoning="", + has_commitment=True, + action_type="network", + commitment_level="will_do_this_week", + barriers_mentioned=[], + ), + ), + ActionExtractionTest( + name="explicit refusal, no commitment", + user_input="No, I don't think any of these are right for me right now.", + expected_action_result=ActionExtractionResult( + reasoning="", + has_commitment=False, + action_type=None, + commitment_level=None, + barriers_mentioned=[], + ), + ), + ActionExtractionTest( + name="barrier plus weak interest, no firm commitment", + user_input=( + "The pay sounds okay but I'm worried about the cost of the training. " + "Maybe I could look into it at some point." + ), + expected_action_result=ActionExtractionResult( + reasoning="", + has_commitment=False, + action_type=None, + commitment_level=None, + barriers_mentioned=["placeholder"], # expects barriers (cost concern) + ), + ), +] + +def _assert_extraction( + actual: ActionExtractionResult, + expected: ActionExtractionResult, + description: str, +) -> None: + assert actual.has_commitment == expected.has_commitment, ( + f"[{description}] has_commitment: got {actual.has_commitment!r}, " + f"expected {expected.has_commitment!r}. reasoning='{actual.reasoning}'" + ) + + if expected.action_type is not None: + assert actual.action_type == expected.action_type, ( + f"[{description}] action_type: got {actual.action_type!r}, " + f"expected {expected.action_type!r}. reasoning='{actual.reasoning}'" + ) + + if expected.commitment_level is not None: + assert actual.commitment_level == expected.commitment_level, ( + f"[{description}] commitment_level: got {actual.commitment_level!r}, " + f"expected {expected.commitment_level!r}. reasoning='{actual.reasoning}'" + ) + + if expected.barriers_mentioned: + assert actual.barriers_mentioned, ( + f"[{description}] expected barriers to be mentioned but got none. " + f"reasoning='{actual.reasoning}'" + ) + else: + assert not actual.barriers_mentioned, ( + f"[{description}] expected no barriers but got {actual.barriers_mentioned!r}. " + f"reasoning='{actual.reasoning}'" + ) + + for slot in ("plan_when", "plan_how", "plan_backup"): + if getattr(expected, slot) is not None: + assert getattr(actual, slot) is not None, ( + f"[{description}] {slot} should be present but was None. " + f"reasoning='{actual.reasoning}'" + ) + +async def _build_extraction_handler() -> ActionPhaseHandler: + llm = GeminiGenerativeLLM( + system_instructions="", + config=LLMConfig(generation_config=LOW_TEMPERATURE_GENERATION_CONFIG | JSON_GENERATION_CONFIG), + ) + action_llm = GeminiGenerativeLLM( + config=LLMConfig( + generation_config=ZERO_TEMPERATURE_GENERATION_CONFIG | with_response_schema(ActionExtractionResult) + ) + ) + return ActionPhaseHandler( + conversation_llm=llm, + conversation_caller=LLMCaller[ConversationResponse](model_response_type=ConversationResponse), + action_caller=LLMCaller[ActionExtractionResult](model_response_type=ActionExtractionResult), + action_llm=action_llm, + ) + + +def _empty_context() -> ConversationContext: + h = ConversationHistory() + return ConversationContext(all_history=h, history=h, summary="") + + +@pytest.mark.asyncio +@pytest.mark.evaluation_test +@pytest.mark.parametrize( + "test_case", + TESTS, + ids=[t.name for t in TESTS], +) +async def test_action_extraction(test_case: ActionExtractionTest): + """Call _extract_action and assert the LLM produces the expected result.""" + handler = await _build_extraction_handler() + context = _empty_context() + + result, _stats = await handler._extract_action(test_case.user_input, context) + + LOG_DIR.mkdir(exist_ok=True) + safe_name = test_case.name.replace(" ", "_").replace(",", "") + with open(LOG_DIR / f"action_extraction_{safe_name}.log", "w") as f: + f.write(f"user_input : {test_case.user_input}\n") + f.write(f"result : {result}\n") + + assert result is not None, ( + f"[{test_case.name}] _extract_action returned None (LLM call failed)" + ) + + _assert_extraction(result, test_case.expected_action_result, test_case.name) diff --git a/backend/evaluation_tests/recommender_advisor_agent/test_action_planning_intent_fix.py b/backend/evaluation_tests/recommender_advisor_agent/test_action_planning_intent_fix.py index fd5a8b28e..e3f41078b 100644 --- a/backend/evaluation_tests/recommender_advisor_agent/test_action_planning_intent_fix.py +++ b/backend/evaluation_tests/recommender_advisor_agent/test_action_planning_intent_fix.py @@ -49,6 +49,7 @@ from app.agent.agent_types import AgentInput, AgentOutput from common_libs.llm.generative_models import GeminiGenerativeLLM from common_libs.llm.models_utils import LLMConfig, LOW_TEMPERATURE_GENERATION_CONFIG, JSON_GENERATION_CONFIG +from common_libs.llm.schema_builder import with_response_schema from app.countries import Country from evaluation_tests.recommender_advisor_agent.sample_data import ( create_sample_recommendations, @@ -145,6 +146,12 @@ async def initialize_handlers(): model_response_type=ActionExtractionResult ) + action_llm = GeminiGenerativeLLM( + config=LLMConfig( + generation_config=LOW_TEMPERATURE_GENERATION_CONFIG | with_response_schema(ActionExtractionResult) + ) + ) + recommendation_interface = RecommendationInterface(node2vec_client=None) intent_classifier = IntentClassifier(intent_caller=intent_caller) @@ -209,6 +216,7 @@ async def initialize_handlers(): conversation_llm=llm, conversation_caller=conversation_caller, action_caller=action_caller, + action_llm=action_llm, intent_classifier=intent_classifier ) diff --git a/backend/evaluation_tests/recommender_advisor_agent/test_blanket_rejection_fix.py b/backend/evaluation_tests/recommender_advisor_agent/test_blanket_rejection_fix.py index b3bf67daa..67851ea76 100644 --- a/backend/evaluation_tests/recommender_advisor_agent/test_blanket_rejection_fix.py +++ b/backend/evaluation_tests/recommender_advisor_agent/test_blanket_rejection_fix.py @@ -54,6 +54,7 @@ from app.agent.preference_elicitation_agent.types import PreferenceVector from common_libs.llm.generative_models import GeminiGenerativeLLM from common_libs.llm.models_utils import LLMConfig, LOW_TEMPERATURE_GENERATION_CONFIG, JSON_GENERATION_CONFIG +from common_libs.llm.schema_builder import with_response_schema from app.countries import Country logger = logging.getLogger(__name__) @@ -120,6 +121,12 @@ async def _build_handlers(llm: GeminiGenerativeLLM) -> dict: intent_caller = LLMCaller[UserIntentClassification](model_response_type=UserIntentClassification) action_caller = LLMCaller[ActionExtractionResult](model_response_type=ActionExtractionResult) + action_llm = GeminiGenerativeLLM( + config=LLMConfig( + generation_config=LOW_TEMPERATURE_GENERATION_CONFIG | with_response_schema(ActionExtractionResult) + ) + ) + intent_classifier = IntentClassifier(intent_caller=intent_caller) recommendation_interface = RecommendationInterface(node2vec_client=None) @@ -127,6 +134,7 @@ async def _build_handlers(llm: GeminiGenerativeLLM) -> dict: conversation_llm=llm, conversation_caller=conversation_caller, action_caller=action_caller, + action_llm=action_llm, intent_classifier=intent_classifier, ) concerns_handler = ConcernsPhaseHandler( diff --git a/backend/evaluation_tests/test_schema_builder.py b/backend/evaluation_tests/test_schema_builder.py index a9108f5bc..7eb858e6c 100644 --- a/backend/evaluation_tests/test_schema_builder.py +++ b/backend/evaluation_tests/test_schema_builder.py @@ -20,6 +20,7 @@ _ContextualizationLLMOutput from app.agent.linking_and_ranking_pipeline.pick_top_skills_tool import _PickTopSkillsLLMOutput from app.agent.linking_and_ranking_pipeline.relevant_entities_classifier_llm import _RelevantEntityClassifierLLMOutput +from app.agent.recommender_advisor_agent.llm_response_models import ActionExtractionResult from app.agent.simple_llm_agent.llm_response import ModelResponse from app.agent.skill_explorer_agent._responsibilities_extraction_llm import ResponsibilitiesExtractionResponse from app.agent.skill_explorer_agent._sentence_decomposition_llm import _SentenceDecompositionResponse, \ @@ -265,6 +266,7 @@ class ListModel(BaseModel): ModelResponse, ResponsibilitiesExtractionResponse, _SentenceDecompositionFirstPassResponse, + ActionExtractionResult, _SentenceDecompositionResponse, CVExtractionResponse ] diff --git a/backend/scripts/reproduce_recommender_early_termination.py b/backend/scripts/reproduce_recommender_early_termination.py index ab9636122..b31662418 100644 --- a/backend/scripts/reproduce_recommender_early_termination.py +++ b/backend/scripts/reproduce_recommender_early_termination.py @@ -70,7 +70,8 @@ ) from app.agent.agent_types import AgentInput, AgentOutput from common_libs.llm.generative_models import GeminiGenerativeLLM -from common_libs.llm.models_utils import LLMConfig, MEDIUM_TEMPERATURE_GENERATION_CONFIG, JSON_GENERATION_CONFIG +from common_libs.llm.models_utils import LLMConfig, MEDIUM_TEMPERATURE_GENERATION_CONFIG, ZERO_TEMPERATURE_GENERATION_CONFIG, JSON_GENERATION_CONFIG +from common_libs.llm.schema_builder import with_response_schema from app.countries import Country console = Console() @@ -213,6 +214,12 @@ async def build_handlers(llm: GeminiGenerativeLLM): intent_caller = LLMCaller[UserIntentClassification](model_response_type=UserIntentClassification) action_caller = LLMCaller[ActionExtractionResult](model_response_type=ActionExtractionResult) + action_llm = GeminiGenerativeLLM( + config=LLMConfig( + generation_config=ZERO_TEMPERATURE_GENERATION_CONFIG | with_response_schema(ActionExtractionResult) + ) + ) + intent_classifier = IntentClassifier(intent_caller=intent_caller) recommendation_interface = RecommendationInterface(node2vec_client=None) @@ -220,6 +227,7 @@ async def build_handlers(llm: GeminiGenerativeLLM): conversation_llm=llm, conversation_caller=conversation_caller, action_caller=action_caller, + action_llm=action_llm, intent_classifier=intent_classifier, ) concerns_handler = ConcernsPhaseHandler( diff --git a/backend/scripts/test_recommender_agent_interactive.py b/backend/scripts/test_recommender_agent_interactive.py index c6e04c96e..e67ac1417 100644 --- a/backend/scripts/test_recommender_agent_interactive.py +++ b/backend/scripts/test_recommender_agent_interactive.py @@ -343,6 +343,7 @@ async def initialize_handlers(): # Initialize LLM exactly like the agent does from common_libs.llm.models_utils import LLMConfig, LOW_TEMPERATURE_GENERATION_CONFIG, JSON_GENERATION_CONFIG + from common_libs.llm.schema_builder import with_response_schema from app.agent.recommender_advisor_agent.phase_handlers.tradeoffs_handler import TradeoffsPhaseHandler from app.agent.recommender_advisor_agent.phase_handlers.followup_handler import FollowupPhaseHandler from app.agent.recommender_advisor_agent.phase_handlers.skills_pivot_handler import SkillsPivotPhaseHandler @@ -374,6 +375,12 @@ async def initialize_handlers(): model_response_type=ActionExtractionResult ) + action_llm = GeminiGenerativeLLM( + config=LLMConfig( + generation_config=LOW_TEMPERATURE_GENERATION_CONFIG | with_response_schema(ActionExtractionResult) + ) + ) + # Initialize recommendation interface recommendation_interface = RecommendationInterface(node2vec_client=None) @@ -466,6 +473,7 @@ async def initialize_handlers(): conversation_llm=llm, conversation_caller=conversation_caller, action_caller=action_caller, + action_llm=action_llm, intent_classifier=intent_classifier )