CORE-507 Troubleshoot: Attempt 3 failed to extract JSON caused by: Failed to construct model: ActionExtractionResult

backend/app/agent/recommender_advisor_agent/agent.py

@@ -51,10 +51,10 @@
 from common_libs.llm.generative_models import GeminiGenerativeLLM
 from common_libs.llm.models_utils import (
     LLMConfig,
-    LOW_TEMPERATURE_GENERATION_CONFIG,
     MEDIUM_TEMPERATURE_GENERATION_CONFIG,
     JSON_GENERATION_CONFIG
 )
+from common_libs.llm.schema_builder import with_response_schema
 
 # Import PreferenceVector from Epic 2
 from app.agent.preference_elicitation_agent.types import PreferenceVector
@@ -120,6 +120,12 @@ def __init__(
             config=llm_config
         )
 
+        self._action_llm = GeminiGenerativeLLM(
+            config=LLMConfig(
+                generation_config=MEDIUM_TEMPERATURE_GENERATION_CONFIG | with_response_schema(ActionExtractionResult)
+            )
+        )
+
         # Initialize LLM callers
         self._conversation_caller: LLMCaller[ConversationResponse] = LLMCaller[ConversationResponse](
             model_response_type=ConversationResponse
@@ -162,6 +168,7 @@ def _init_phase_handlers(self) -> None:
             conversation_llm=self._conversation_llm,
             conversation_caller=self._conversation_caller,
             action_caller=self._action_caller,
+            action_llm=self._action_llm,
             intent_classifier=self._intent_classifier,
             logger=self.logger
         )

backend/app/agent/recommender_advisor_agent/phase_handlers/action_handler.py

@@ -4,6 +4,7 @@
 Handles the ACTION_PLANNING phase where we guide users
 toward concrete next steps.
 """
+from textwrap import dedent
 
 from app.agent.agent_types import LLMStats
 from app.agent.llm_caller import LLMCaller
@@ -50,6 +51,7 @@ def __init__(
         conversation_llm: GeminiGenerativeLLM,
         conversation_caller: LLMCaller[ConversationResponse],
         action_caller: LLMCaller[ActionExtractionResult],
+        action_llm: GeminiGenerativeLLM = None,
         intent_classifier: IntentClassifier = None,
         present_handler: 'PresentPhaseHandler' = None,
         concerns_handler: 'ConcernsPhaseHandler' = None,
@@ -61,13 +63,15 @@ def __init__(
 
         Args:
             action_caller: LLM caller for action extraction
+            action_llm: LLM configured with build_request_schema(ActionExtractionResult) for structured extraction
             intent_classifier: Optional intent classifier for routing user choices
             present_handler: Optional present handler for immediate delegation
             concerns_handler: Optional concerns handler for immediate delegation
             wrapup_handler: Optional wrapup handler for immediate delegation
         """
         super().__init__(conversation_llm, conversation_caller, **kwargs)
         self._action_caller = action_caller
+        self._action_llm = action_llm
         self._intent_classifier = intent_classifier
         self._present_handler = present_handler
         self._concerns_handler = concerns_handler
@@ -285,35 +289,33 @@ async def _extract_action(
             )
         ]
 
-        prompt = f"""
-Analyze this user response to determine if they're committing to an action:
-
-User said: "{user_input}"
-
-Context: They've been exploring career/occupation recommendations.
-
-Determine:
-1. has_commitment: Did they make a clear commitment to take action? (true/false)
-2. action_type: What type of action? Must be one of: apply_to_job, enroll_in_training, explore_occupation, research_employer, network
-3. commitment_level: How strong is their commitment? Must be one of: will_do_this_week, will_do_this_month, interested, maybe_later, not_interested
-4. barriers_mentioned: List any barriers or concerns mentioned (empty list if none)
-5. plan_when: When they'll act, in their words (e.g., "Thursday morning"). Null if not stated.
-6. plan_where: Where / the specific target (e.g., "the depot", "the Brookside posting"). Null if not stated.
-7. plan_how: How - transport, what to bring or say (e.g., "matatu, ask the supervisor"). Null if not stated.
-8. plan_backup: The if-then backup for the most likely obstacle (e.g., "Saturday if the fare is short"). Null if not stated.
-
-IMPORTANT:
-- Use exact field names: has_commitment, action_type, commitment_level, barriers_mentioned, plan_when, plan_where, plan_how, plan_backup
-- action_type and commitment_level must use exact enum values listed above
-- If has_commitment is false, set action_type and commitment_level to null
-- Only fill a plan_* field if the user actually stated it THIS turn - never invent details. Leave it null otherwise.
-
-{get_json_response_instructions(examples=examples)}
-"""
+        prompt = dedent(f"""
+            Analyze this user response to determine if they're committing to an action:
+
+            User said: "{user_input}"
+
+            Context: They've been exploring career/occupation recommendations.
+
+            Determine:
+            1. has_commitment: Did they make a clear commitment to take action? (true/false)
+            2. action_type: What type of action? Must be one of: apply_to_job, enroll_in_training, explore_occupation, research_employer, network
+            3. commitment_level: How strong is their commitment? Must be one of: will_do_this_week, will_do_this_month, interested, maybe_later, not_interested
+            4. barriers_mentioned: List any barriers or concerns mentioned (empty list if none)
+            5. plan_when: When they'll act, in their words (e.g., "Thursday morning"). Null if not stated.
+            6. plan_where: Where / the specific target (e.g., "the depot", "the Brookside posting"). Null if not stated.
+            7. plan_how: How - transport, what to bring or say (e.g., "matatu, ask the supervisor"). Null if not stated.
+            8. plan_backup: The if-then backup for the most likely obstacle (e.g., "Saturday if the fare is short"). Null if not stated.
+
+            IMPORTANT:
+            - Use exact field names: has_commitment, action_type, commitment_level, barriers_mentioned, plan_when, plan_where, plan_how, plan_backup
+            - action_type and commitment_level must use exact enum values listed above
+            - If has_commitment is false, set action_type and commitment_level to null
+            - Only fill a plan_* field if the user actually stated it THIS turn - never invent details. Leave it null otherwise.
+            """)
 
         try:
             result = await self._action_caller.call_llm(
-                llm=self._conversation_llm,
+                llm=self._action_llm or self._conversation_llm,
                 llm_input=ConversationHistoryFormatter.format_for_agent_generative_prompt(
                     model_response_instructions=prompt,
                     context=context,

backend/common_libs/llm/models_utils.py

@@ -207,13 +207,20 @@ async def generate_content(self, llm_input: LLMInput | str) -> LLMResponse:
         raise NotImplementedError()
 
 
+vertex_ai_initialized: bool = False
+def init_vertex_ai_once(_logger: logging.Logger, location: str):
+    global vertex_ai_initialized
+    if not vertex_ai_initialized:
+        _logger.info("Initializing VertexAI client with location: %s", location)
+        vertexai.init(location=location)
+        vertex_ai_initialized = True
+
 class BasicLLM(LLM):
     def __init__(self, *, config: LLMConfig = LLMConfig()):
         # Before constructing the llm model, we need to initialize the VertexAI client
         # as the init function may have been called in another module with different parameters
         self.logger = logging.getLogger(self.__class__.__name__)
-        self.logger.info("Initializing VertexAI client with location: %s", config.location)
-        vertexai.init(location=config.location)
+        init_vertex_ai_once(self.logger, config.location)
         self._retry_config = config.retry_config
         self._model = None
         self._chat = None

backend/evaluation_tests/recommender_advisor_agent/test_action_handler_eval.py

@@ -0,0 +1,244 @@
+import asyncio
+import logging
+from pydantic import BaseModel
+from pathlib import Path
+
+import pytest
+
+from app.agent.llm_caller import LLMCaller
+from app.agent.recommender_advisor_agent.llm_response_models import (
+    ActionExtractionResult,
+    ConversationResponse,
+)
+from app.agent.recommender_advisor_agent.phase_handlers.action_handler import ActionPhaseHandler
+from app.conversation_memory.conversation_memory_types import (
+    ConversationContext,
+    ConversationHistory,
+)
+from common_libs.llm.generative_models import GeminiGenerativeLLM
+from common_libs.llm.models_utils import (
+    JSON_GENERATION_CONFIG,
+    LOW_TEMPERATURE_GENERATION_CONFIG,
+    ZERO_TEMPERATURE_GENERATION_CONFIG,
+    LLMConfig,
+)
+from common_libs.llm.schema_builder import with_response_schema
+from evaluation_tests.compass_test_case import CompassTestCase
+
+logger = logging.getLogger(__name__)
+LOG_DIR = Path(__file__).parent / "logs"
+
+
+
+class ActionExtractionTest(CompassTestCase):
+    user_input: str
+    expected_action_result: ActionExtractionResult
+
+
+# Non-None value in a plan slot means "must be present in actual output".
+# The exact text is not compared — only presence (not None) is asserted.
+_PRESENT = "present"
+
+TESTS: list[ActionExtractionTest] = [
+    ActionExtractionTest(
+        name="concrete commitment with when and how",
+        user_input=(
+            "Yes, I'll go to the Nyali construction site Thursday morning by matatu "
+            "and ask the foreman directly about the apprenticeship opening."
+        ),
+        expected_action_result=ActionExtractionResult(
+            reasoning="",
+            has_commitment=True,
+            action_type="apply_to_job",
+            commitment_level="will_do_this_week",
+            barriers_mentioned=[],
+            plan_when=_PRESENT,
+            plan_how=_PRESENT,
+        ),
+    ),
+    ActionExtractionTest(
+        name="vague strong commitment, no plan details stated",
+        user_input="Yes I want to apply for the electrical apprenticeship, I'll do it this week.",
+        expected_action_result=ActionExtractionResult(
+            reasoning="",
+            has_commitment=True,
+            action_type="apply_to_job",
+            commitment_level="will_do_this_week",
+            barriers_mentioned=[],
+        ),
+    ),
+    ActionExtractionTest(
+        name="training enrollment commitment next month",
+        user_input=(
+            "I want to register for the electrician grade test course. "
+            "I'll sign up at Mombasa Technical next month."
+        ),
+        expected_action_result=ActionExtractionResult(
+            reasoning="",
+            has_commitment=True,
+            action_type="enroll_in_training",
+            commitment_level="will_do_this_month",
+            barriers_mentioned=[],
+        ),
+    ),
+    ActionExtractionTest(
+        name="barriers only, no commitment made",
+        user_input=(
+            "I'm not sure I can afford the bus fare to get there, "
+            "and I don't know if they're still hiring for that position."
+        ),
+        expected_action_result=ActionExtractionResult(
+            reasoning="",
+            has_commitment=False,
+            action_type=None,
+            commitment_level=None,
+            barriers_mentioned=["placeholder"],  # non-empty → assert actual has barriers
+        ),
+    ),
+    ActionExtractionTest(
+        name="full concrete plan including backup",
+        user_input=(
+            "I'll go Monday morning, take the 7am Coast Bus from Mwembe Tayari, "
+            "bring my ID and ask for the site manager. "
+            "If he's not there I'll come back Wednesday."
+        ),
+        expected_action_result=ActionExtractionResult(
+            reasoning="",
+            has_commitment=True,
+            action_type="apply_to_job",
+            commitment_level="will_do_this_week",
+            barriers_mentioned=[],
+            plan_when=_PRESENT,
+            plan_how=_PRESENT,
+            plan_backup=_PRESENT,
+        ),
+    ),
+    ActionExtractionTest(
+        name="networking commitment this week",
+        user_input=(
+            "I'll ask my uncle to introduce me to his contacts at the construction "
+            "company this weekend — he knows the site supervisor."
+        ),
+        expected_action_result=ActionExtractionResult(
+            reasoning="",
+            has_commitment=True,
+            action_type="network",
+            commitment_level="will_do_this_week",
+            barriers_mentioned=[],
+        ),
+    ),
+    ActionExtractionTest(
+        name="explicit refusal, no commitment",
+        user_input="No, I don't think any of these are right for me right now.",
+        expected_action_result=ActionExtractionResult(
+            reasoning="",
+            has_commitment=False,
+            action_type=None,
+            commitment_level=None,
+            barriers_mentioned=[],
+        ),
+    ),
+    ActionExtractionTest(
+        name="barrier plus weak interest, no firm commitment",
+        user_input=(
+            "The pay sounds okay but I'm worried about the cost of the training. "
+            "Maybe I could look into it at some point."
+        ),
+        expected_action_result=ActionExtractionResult(
+            reasoning="",
+            has_commitment=False,
+            action_type=None,
+            commitment_level=None,
+            barriers_mentioned=["placeholder"],  # expects barriers (cost concern)
+        ),
+    ),
+]
+
+def _assert_extraction(
+    actual: ActionExtractionResult,
+    expected: ActionExtractionResult,
+    description: str,
+) -> None:
+    assert actual.has_commitment == expected.has_commitment, (
+        f"[{description}] has_commitment: got {actual.has_commitment!r}, "
+        f"expected {expected.has_commitment!r}. reasoning='{actual.reasoning}'"
+    )
+
+    if expected.action_type is not None:
+        assert actual.action_type == expected.action_type, (
+            f"[{description}] action_type: got {actual.action_type!r}, "
+            f"expected {expected.action_type!r}. reasoning='{actual.reasoning}'"
+        )
+
+    if expected.commitment_level is not None:
+        assert actual.commitment_level == expected.commitment_level, (
+            f"[{description}] commitment_level: got {actual.commitment_level!r}, "
+            f"expected {expected.commitment_level!r}. reasoning='{actual.reasoning}'"
+        )
+
+    if expected.barriers_mentioned:
+        assert actual.barriers_mentioned, (
+            f"[{description}] expected barriers to be mentioned but got none. "
+            f"reasoning='{actual.reasoning}'"
+        )
+    else:
+        assert not actual.barriers_mentioned, (
+            f"[{description}] expected no barriers but got {actual.barriers_mentioned!r}. "
+            f"reasoning='{actual.reasoning}'"
+        )
+
+    for slot in ("plan_when", "plan_how", "plan_backup"):
+        if getattr(expected, slot) is not None:
+            assert getattr(actual, slot) is not None, (
+                f"[{description}] {slot} should be present but was None. "
+                f"reasoning='{actual.reasoning}'"
+            )
+
+async def _build_extraction_handler() -> ActionPhaseHandler:
+    llm = GeminiGenerativeLLM(
+        system_instructions="",
+        config=LLMConfig(generation_config=LOW_TEMPERATURE_GENERATION_CONFIG | JSON_GENERATION_CONFIG),
+    )
+    action_llm = GeminiGenerativeLLM(
+        config=LLMConfig(
+            generation_config=ZERO_TEMPERATURE_GENERATION_CONFIG | with_response_schema(ActionExtractionResult)
+        )
+    )
+    return ActionPhaseHandler(
+        conversation_llm=llm,
+        conversation_caller=LLMCaller[ConversationResponse](model_response_type=ConversationResponse),
+        action_caller=LLMCaller[ActionExtractionResult](model_response_type=ActionExtractionResult),
+        action_llm=action_llm,
+    )
+
+
+def _empty_context() -> ConversationContext:
+    h = ConversationHistory()
+    return ConversationContext(all_history=h, history=h, summary="")
+
+
+@pytest.mark.asyncio
+@pytest.mark.evaluation_test
+@pytest.mark.parametrize(
+    "test_case",
+    TESTS,
+    ids=[t.name for t in TESTS],
+)
+async def test_action_extraction(test_case: ActionExtractionTest):
+    """Call _extract_action and assert the LLM produces the expected result."""
+    handler = await _build_extraction_handler()
+    context = _empty_context()
+
+    result, _stats = await handler._extract_action(test_case.user_input, context)
+
+    LOG_DIR.mkdir(exist_ok=True)
+    safe_name = test_case.name.replace(" ", "_").replace(",", "")
+    with open(LOG_DIR / f"action_extraction_{safe_name}.log", "w") as f:
+        f.write(f"user_input : {test_case.user_input}\n")
+        f.write(f"result     : {result}\n")
+
+    assert result is not None, (
+        f"[{test_case.name}] _extract_action returned None (LLM call failed)"
+    )
+
+    _assert_extraction(result, test_case.expected_action_result, test_case.name)